<commit_before>package server

import (
	"fmt"

	"net"

	"sync"

	"github.com/xiaonanln/goSyncQueue"
	"github.com/xiaonanln/goworld/common"
	"github.com/xiaonanln/goworld/components/dispatcher/dispatcher_client"
	"github.com/xiaonanln/goworld/config"
	"github.com/xiaonanln/goworld/consts"
	"github.com/xiaonanln/goworld/gwlog"
	"github.com/xiaonanln/goworld/netutil"
	"github.com/xiaonanln/goworld/proto"
)

type GateService struct {
	listenAddr        string
	clientProxies     map[common.ClientID]*ClientProxy
	clientProxiesLock sync.RWMutex
	packetQueue       sync_queue.SyncQueue
}

func newGateService() *GateService {
	return &GateService{
		//packetQueue: make(chan packetQueueItem, consts.DISPATCHER_CLIENT_PACKET_QUEUE_SIZE),
		clientProxies: map[common.ClientID]*ClientProxy{},
		packetQueue:   sync_queue.NewSyncQueue(),
	}
}

func (gs *GateService) run() {
	cfg := config.GetServer(serverid)
	gs.listenAddr = fmt.Sprintf("%s:%d", cfg.Ip, cfg.Port)
	go netutil.ServeForever(gs.handlePacketRoutine)
	netutil.ServeTCPForever(gs.listenAddr, gs)
}

func (gs *GateService) String() string {
	return fmt.Sprintf("GateService<%s>", gs.listenAddr)
}

func (gs *GateService) ServeTCPConnection(conn net.Conn) {
	cp := newClientProxy(conn)
	gs.clientProxiesLock.Lock()
	gs.clientProxies[cp.clientid] = cp
	gs.clientProxiesLock.Unlock()
	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientConnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.ServeTCPConnection: client %s connected", gs, cp)
	}
	cp.serve()
}

func (gs *GateService) onClientProxyClose(cp *ClientProxy) {
	gs.clientProxiesLock.Lock()
	delete(gs.clientProxies, cp.clientid)
	gs.clientProxiesLock.Unlock()

	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.onClientProxyClose: client %s disconnected", gs, cp)
	}
}

func (gs *GateService) HandleDispatcherClientPacket(msgtype proto.MsgType_t, packet *netutil.Packet) {
	if consts.DEBUG_PACKETS {
		gwlog.Debug("%s.HandleDispatcherClientPacket: msgtype=%v, packet=%v", gs, msgtype, packet.Payload())
	}
	_ = packet.ReadUint16() // sid
	clientid := packet.ReadClientID()
	gs.clientProxiesLock.RLock()
	clientproxy := gs.clientProxies[clientid]
	gs.clientProxiesLock.RUnlock()

	if clientproxy != nil {
		clientproxy.SendPacket(packet)
	}

	packet.Release()
	//typeName := packet.ReadVarStr()
	//entityid := packet.ReadEntityID()

}

func (gs *GateService) handlePacketRoutine() {
	for {
		item := gs.packetQueue.Pop().(packetQueueItem)
		gs.HandleDispatcherClientPacket(item.msgtype, item.packet)
	}
}
<commit_msg>notify clietn disconnecte if game is sending packets to client<commit_after>package server

import (
	"fmt"

	"net"

	"sync"

	"github.com/xiaonanln/goSyncQueue"
	"github.com/xiaonanln/goworld/common"
	"github.com/xiaonanln/goworld/components/dispatcher/dispatcher_client"
	"github.com/xiaonanln/goworld/config"
	"github.com/xiaonanln/goworld/consts"
	"github.com/xiaonanln/goworld/gwlog"
	"github.com/xiaonanln/goworld/netutil"
	"github.com/xiaonanln/goworld/proto"
)

type GateService struct {
	listenAddr        string
	clientProxies     map[common.ClientID]*ClientProxy
	clientProxiesLock sync.RWMutex
	packetQueue       sync_queue.SyncQueue
}

func newGateService() *GateService {
	return &GateService{
		//packetQueue: make(chan packetQueueItem, consts.DISPATCHER_CLIENT_PACKET_QUEUE_SIZE),
		clientProxies: map[common.ClientID]*ClientProxy{},
		packetQueue:   sync_queue.NewSyncQueue(),
	}
}

func (gs *GateService) run() {
	cfg := config.GetServer(serverid)
	gs.listenAddr = fmt.Sprintf("%s:%d", cfg.Ip, cfg.Port)
	go netutil.ServeForever(gs.handlePacketRoutine)
	netutil.ServeTCPForever(gs.listenAddr, gs)
}

func (gs *GateService) String() string {
	return fmt.Sprintf("GateService<%s>", gs.listenAddr)
}

func (gs *GateService) ServeTCPConnection(conn net.Conn) {
	cp := newClientProxy(conn)
	gs.clientProxiesLock.Lock()
	gs.clientProxies[cp.clientid] = cp
	gs.clientProxiesLock.Unlock()
	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientConnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.ServeTCPConnection: client %s connected", gs, cp)
	}
	cp.serve()
}

func (gs *GateService) onClientProxyClose(cp *ClientProxy) {
	gs.clientProxiesLock.Lock()
	delete(gs.clientProxies, cp.clientid)
	gs.clientProxiesLock.Unlock()

	dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(cp.clientid)
	if consts.DEBUG_CLIENTS {
		gwlog.Debug("%s.onClientProxyClose: client %s disconnected", gs, cp)
	}
}

func (gs *GateService) HandleDispatcherClientPacket(msgtype proto.MsgType_t, packet *netutil.Packet) {
	if consts.DEBUG_PACKETS {
		gwlog.Debug("%s.HandleDispatcherClientPacket: msgtype=%v, packet=%v", gs, msgtype, packet.Payload())
	}
	_ = packet.ReadUint16() // sid
	clientid := packet.ReadClientID()
	gs.clientProxiesLock.RLock()
	clientproxy := gs.clientProxies[clientid]
	gs.clientProxiesLock.RUnlock()

	if clientproxy != nil {
		clientproxy.SendPacket(packet)
	} else {
		// client already disconnected, but the game service seems not knowing it, so tell it
		dispatcher_client.GetDispatcherClientForSend().SendNotifyClientDisconnected(clientid)
	}

	packet.Release()
	//typeName := packet.ReadVarStr()
	//entityid := packet.ReadEntityID()

}

func (gs *GateService) handlePacketRoutine() {
	for {
		item := gs.packetQueue.Pop().(packetQueueItem)
		gs.HandleDispatcherClientPacket(item.msgtype, item.packet)
	}
}
<|endoftext|>


<commit_before><commit_msg>Create ConvertaNumbertoHexadecimal.go<commit_after><|endoftext|>


<commit_before>package bxmpp

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/jpillora/backoff"
	"github.com/matterbridge/go-xmpp"
	"github.com/rs/xid"
)

type Bxmpp struct {
	*bridge.Config

	startTime time.Time
	xc        *xmpp.Client
	xmppMap   map[string]string
	connected bool
	sync.RWMutex

	avatarAvailability map[string]bool
	avatarMap          map[string]string
}

func New(cfg *bridge.Config) bridge.Bridger {
	return &Bxmpp{
		Config:             cfg,
		xmppMap:            make(map[string]string),
		avatarAvailability: make(map[string]bool),
		avatarMap:          make(map[string]string),
	}
}

func (b *Bxmpp) Connect() error {
	b.Log.Infof("Connecting %s", b.GetString("Server"))
	if err := b.createXMPP(); err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}

	b.Log.Info("Connection succeeded")
	go b.manageConnection()
	return nil
}

func (b *Bxmpp) Disconnect() error {
	return nil
}

func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
	if channel.Options.Key != "" {
		b.Log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
		b.xc.JoinProtectedMUC(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"), channel.Options.Key, xmpp.NoHistory, 0, nil)
	} else {
		b.xc.JoinMUCNoHistory(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"))
	}
	return nil
}

func (b *Bxmpp) Send(msg config.Message) (string, error) {
	// should be fixed by using a cache instead of dropping
	if !b.Connected() {
		return "", fmt.Errorf("bridge %s not connected, dropping message %#v to bridge", b.Account, msg)
	}
	// ignore delete messages
	if msg.Event == config.EventMsgDelete {
		return "", nil
	}

	b.Log.Debugf("=> Receiving %#v", msg)

	if msg.Event == config.EventAvatarDownload {
		return b.cacheAvatar(&msg), nil
	}

	// Make a action /me of the message, prepend the username with it.
	// https://xmpp.org/extensions/xep-0245.html
	if msg.Event == config.EventUserAction {
		msg.Username = "/me " + msg.Username
	}

	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
	var err error
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
			if b.GetString("WebhookURL") != "" {
				err = b.postSlackCompatibleWebhook(msg)
			} else {
				_, err = b.xc.Send(xmpp.Chat{
					Type:   "groupchat",
					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
					Text:   rmsg.Username + rmsg.Text,
				})
			}

			if err != nil {
				b.Log.WithError(err).Error("Unable to send message with share URL.")
			}
		}
		if len(msg.Extra["file"]) > 0 {
			return "", b.handleUploadFile(&msg)
		}
	}

	if b.GetString("WebhookURL") != "" {
		b.Log.Debugf("Sending message using Webhook")
		err := b.postSlackCompatibleWebhook(msg)
		if err != nil {
			b.Log.Errorf("Failed to send message using webhook: %s", err)
			return "", err
		}

		return "", nil
	}

	// Post normal message.
	var msgReplaceID string
	msgID := xid.New().String()
	if msg.ID != "" {
		msgReplaceID = msg.ID
	}
	b.Log.Debugf("=> Sending message %#v", msg)
	if _, err := b.xc.Send(xmpp.Chat{
		Type:      "groupchat",
		Remote:    msg.Channel + "@" + b.GetString("Muc"),
		Text:      msg.Username + msg.Text,
		ID:        msgID,
		ReplaceID: msgReplaceID,
	}); err != nil {
		return "", err
	}
	return msgID, nil
}

func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
	type XMPPWebhook struct {
		Username string `json:"username"`
		Text     string `json:"text"`
	}
	webhookBody, err := json.Marshal(XMPPWebhook{
		Username: msg.Username,
		Text:     msg.Text,
	})
	if err != nil {
		b.Log.Errorf("Failed to marshal webhook: %s", err)
		return err
	}

	resp, err := http.Post(b.GetString("WebhookURL")+"/"+url.QueryEscape(msg.Channel), "application/json", bytes.NewReader(webhookBody))
	if err != nil {
		b.Log.Errorf("Failed to POST webhook: %s", err)
		return err
	}

	resp.Body.Close()
	return nil
}

func (b *Bxmpp) createXMPP() error {
	var serverName string
	switch {
	case !b.GetBool("Anonymous"):
		if !strings.Contains(b.GetString("Jid"), "@") {
			return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
		}
		serverName = strings.Split(b.GetString("Jid"), "@")[1]
	case !strings.Contains(b.GetString("Server"), ":"):
		serverName = strings.Split(b.GetString("Server"), ":")[0]
	default:
		serverName = b.GetString("Server")
	}

	tc := &tls.Config{
		ServerName:         serverName,
		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
	}

	xmpp.DebugWriter = b.Log.Writer()

	options := xmpp.Options{
		Host:                         b.GetString("Server"),
		User:                         b.GetString("Jid"),
		Password:                     b.GetString("Password"),
		NoTLS:                        true,
		StartTLS:                     !b.GetBool("NoTLS"),
		TLSConfig:                    tc,
		Debug:                        b.GetBool("debug"),
		Session:                      true,
		Status:                       "",
		StatusMessage:                "",
		Resource:                     "",
		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
	}
	var err error
	b.xc, err = options.NewClient()
	return err
}

func (b *Bxmpp) manageConnection() {
	b.setConnected(true)
	initial := true
	bf := &backoff.Backoff{
		Min:    time.Second,
		Max:    5 * time.Minute,
		Jitter: true,
	}

	// Main connection loop. Each iteration corresponds to a successful
	// connection attempt and the subsequent handling of the connection.
	for {
		if initial {
			initial = false
		} else {
			b.Remote <- config.Message{
				Username: "system",
				Text:     "rejoin",
				Channel:  "",
				Account:  b.Account,
				Event:    config.EventRejoinChannels,
			}
		}

		if err := b.handleXMPP(); err != nil {
			b.Log.WithError(err).Error("Disconnected.")
			b.setConnected(false)
		}

		// Reconnection loop using an exponential back-off strategy. We
		// only break out of the loop if we have successfully reconnected.
		for {
			d := bf.Duration()
			b.Log.Infof("Reconnecting in %s.", d)
			time.Sleep(d)

			b.Log.Infof("Reconnecting now.")
			if err := b.createXMPP(); err == nil {
				b.setConnected(true)
				bf.Reset()
				break
			}
			b.Log.Warn("Failed to reconnect.")
		}
	}
}

func (b *Bxmpp) xmppKeepAlive() chan bool {
	done := make(chan bool)
	go func() {
		ticker := time.NewTicker(90 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				b.Log.Debugf("PING")
				if err := b.xc.PingC2S("", ""); err != nil {
					b.Log.Debugf("PING failed %#v", err)
				}
			case <-done:
				return
			}
		}
	}()
	return done
}

func (b *Bxmpp) handleXMPP() error {
	b.startTime = time.Now()

	done := b.xmppKeepAlive()
	defer close(done)

	for {
		m, err := b.xc.Recv()
		if err != nil {
			return err
		}

		switch v := m.(type) {
		case xmpp.Chat:
			if v.Type == "groupchat" {
				b.Log.Debugf("== Receiving %#v", v)

				// Skip invalid messages.
				if b.skipMessage(v) {
					continue
				}

				var event string
				if strings.Contains(v.Text, "has set the subject to:") {
					event = config.EventTopicChange
				}

				available, sok := b.avatarAvailability[v.Remote]
				avatar := ""
				if !sok {
					b.Log.Debugf("Requesting avatar data")
					b.avatarAvailability[v.Remote] = false
					b.xc.AvatarRequestData(v.Remote)
				} else if available {
					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
				}

				msgID := v.ID
				if v.ReplaceID != "" {
					msgID = v.ReplaceID
				}
				rmsg := config.Message{
					Username: b.parseNick(v.Remote),
					Text:     v.Text,
					Channel:  b.parseChannel(v.Remote),
					Account:  b.Account,
					Avatar:   avatar,
					UserID:   v.Remote,
					ID:       msgID,
					Event:    event,
				}

				// Check if we have an action event.
				var ok bool
				rmsg.Text, ok = b.replaceAction(rmsg.Text)
				if ok {
					rmsg.Event = config.EventUserAction
				}

				b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
				b.Log.Debugf("<= Message is %#v", rmsg)
				b.Remote <- rmsg
			}
		case xmpp.AvatarData:
			b.handleDownloadAvatar(v)
			b.avatarAvailability[v.From] = true
			b.Log.Debugf("Avatar for %s is now available", v.From)
		case xmpp.Presence:
			// Do nothing.
		}
	}
}

func (b *Bxmpp) replaceAction(text string) (string, bool) {
	if strings.HasPrefix(text, "/me ") {
		return strings.Replace(text, "/me ", "", -1), true
	}
	return text, false
}

// handleUploadFile handles native upload of files
func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
	var urlDesc string

	for _, file := range msg.Extra["file"] {
		fileInfo := file.(config.FileInfo)
		if fileInfo.Comment != "" {
			msg.Text += fileInfo.Comment + ": "
		}
		if fileInfo.URL != "" {
			msg.Text = fileInfo.URL
			if fileInfo.Comment != "" {
				msg.Text = fileInfo.Comment + ": " + fileInfo.URL
				urlDesc = fileInfo.Comment
			}
		}
		if _, err := b.xc.Send(xmpp.Chat{
			Type:   "groupchat",
			Remote: msg.Channel + "@" + b.GetString("Muc"),
			Text:   msg.Username + msg.Text,
		}); err != nil {
			return err
		}

		if fileInfo.URL != "" {
			if _, err := b.xc.SendOOB(xmpp.Chat{
				Type:    "groupchat",
				Remote:  msg.Channel + "@" + b.GetString("Muc"),
				Ooburl:  fileInfo.URL,
				Oobdesc: urlDesc,
			}); err != nil {
				b.Log.WithError(err).Warn("Failed to send share URL.")
			}
		}
	}
	return nil
}

func (b *Bxmpp) parseNick(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) > 1 {
		s = strings.Split(s[1], "/")
		if len(s) == 2 {
			return s[1] // nick
		}
	}
	return ""
}

func (b *Bxmpp) parseChannel(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) >= 2 {
		return s[0] // channel
	}
	return ""
}

// skipMessage skips messages that need to be skipped
func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
	// skip messages from ourselves
	if b.parseNick(message.Remote) == b.GetString("Nick") {
		return true
	}

	// skip empty messages
	if message.Text == "" {
		return true
	}

	// skip subject messages
	if strings.Contains(message.Text, "</subject>") {
		return true
	}

	// do not show subjects on connect #732
	if strings.Contains(message.Text, "has set the subject to:") && time.Since(b.startTime) < time.Second*5 {
		return true
	}

	// Ignore messages posted by our webhook
	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
		return true
	}

	// skip delayed messages
	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
}

func (b *Bxmpp) setConnected(state bool) {
	b.Lock()
	b.connected = state
	defer b.Unlock()
}

func (b *Bxmpp) Connected() bool {
	b.RLock()
	defer b.RUnlock()
	return b.connected
}
<commit_msg>Do not fail on no avatar data (xmpp) #1529 (#1627)<commit_after>package bxmpp

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/jpillora/backoff"
	"github.com/matterbridge/go-xmpp"
	"github.com/rs/xid"
)

type Bxmpp struct {
	*bridge.Config

	startTime time.Time
	xc        *xmpp.Client
	xmppMap   map[string]string
	connected bool
	sync.RWMutex

	avatarAvailability map[string]bool
	avatarMap          map[string]string
}

func New(cfg *bridge.Config) bridge.Bridger {
	return &Bxmpp{
		Config:             cfg,
		xmppMap:            make(map[string]string),
		avatarAvailability: make(map[string]bool),
		avatarMap:          make(map[string]string),
	}
}

func (b *Bxmpp) Connect() error {
	b.Log.Infof("Connecting %s", b.GetString("Server"))
	if err := b.createXMPP(); err != nil {
		b.Log.Debugf("%#v", err)
		return err
	}

	b.Log.Info("Connection succeeded")
	go b.manageConnection()
	return nil
}

func (b *Bxmpp) Disconnect() error {
	return nil
}

func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
	if channel.Options.Key != "" {
		b.Log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
		b.xc.JoinProtectedMUC(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"), channel.Options.Key, xmpp.NoHistory, 0, nil)
	} else {
		b.xc.JoinMUCNoHistory(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"))
	}
	return nil
}

func (b *Bxmpp) Send(msg config.Message) (string, error) {
	// should be fixed by using a cache instead of dropping
	if !b.Connected() {
		return "", fmt.Errorf("bridge %s not connected, dropping message %#v to bridge", b.Account, msg)
	}
	// ignore delete messages
	if msg.Event == config.EventMsgDelete {
		return "", nil
	}

	b.Log.Debugf("=> Receiving %#v", msg)

	if msg.Event == config.EventAvatarDownload {
		return b.cacheAvatar(&msg), nil
	}

	// Make a action /me of the message, prepend the username with it.
	// https://xmpp.org/extensions/xep-0245.html
	if msg.Event == config.EventUserAction {
		msg.Username = "/me " + msg.Username
	}

	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
	var err error
	if msg.Extra != nil {
		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
			if b.GetString("WebhookURL") != "" {
				err = b.postSlackCompatibleWebhook(msg)
			} else {
				_, err = b.xc.Send(xmpp.Chat{
					Type:   "groupchat",
					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
					Text:   rmsg.Username + rmsg.Text,
				})
			}

			if err != nil {
				b.Log.WithError(err).Error("Unable to send message with share URL.")
			}
		}
		if len(msg.Extra["file"]) > 0 {
			return "", b.handleUploadFile(&msg)
		}
	}

	if b.GetString("WebhookURL") != "" {
		b.Log.Debugf("Sending message using Webhook")
		err := b.postSlackCompatibleWebhook(msg)
		if err != nil {
			b.Log.Errorf("Failed to send message using webhook: %s", err)
			return "", err
		}

		return "", nil
	}

	// Post normal message.
	var msgReplaceID string
	msgID := xid.New().String()
	if msg.ID != "" {
		msgReplaceID = msg.ID
	}
	b.Log.Debugf("=> Sending message %#v", msg)
	if _, err := b.xc.Send(xmpp.Chat{
		Type:      "groupchat",
		Remote:    msg.Channel + "@" + b.GetString("Muc"),
		Text:      msg.Username + msg.Text,
		ID:        msgID,
		ReplaceID: msgReplaceID,
	}); err != nil {
		return "", err
	}
	return msgID, nil
}

func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
	type XMPPWebhook struct {
		Username string `json:"username"`
		Text     string `json:"text"`
	}
	webhookBody, err := json.Marshal(XMPPWebhook{
		Username: msg.Username,
		Text:     msg.Text,
	})
	if err != nil {
		b.Log.Errorf("Failed to marshal webhook: %s", err)
		return err
	}

	resp, err := http.Post(b.GetString("WebhookURL")+"/"+url.QueryEscape(msg.Channel), "application/json", bytes.NewReader(webhookBody))
	if err != nil {
		b.Log.Errorf("Failed to POST webhook: %s", err)
		return err
	}

	resp.Body.Close()
	return nil
}

func (b *Bxmpp) createXMPP() error {
	var serverName string
	switch {
	case !b.GetBool("Anonymous"):
		if !strings.Contains(b.GetString("Jid"), "@") {
			return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
		}
		serverName = strings.Split(b.GetString("Jid"), "@")[1]
	case !strings.Contains(b.GetString("Server"), ":"):
		serverName = strings.Split(b.GetString("Server"), ":")[0]
	default:
		serverName = b.GetString("Server")
	}

	tc := &tls.Config{
		ServerName:         serverName,
		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
	}

	xmpp.DebugWriter = b.Log.Writer()

	options := xmpp.Options{
		Host:                         b.GetString("Server"),
		User:                         b.GetString("Jid"),
		Password:                     b.GetString("Password"),
		NoTLS:                        true,
		StartTLS:                     !b.GetBool("NoTLS"),
		TLSConfig:                    tc,
		Debug:                        b.GetBool("debug"),
		Session:                      true,
		Status:                       "",
		StatusMessage:                "",
		Resource:                     "",
		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
	}
	var err error
	b.xc, err = options.NewClient()
	return err
}

func (b *Bxmpp) manageConnection() {
	b.setConnected(true)
	initial := true
	bf := &backoff.Backoff{
		Min:    time.Second,
		Max:    5 * time.Minute,
		Jitter: true,
	}

	// Main connection loop. Each iteration corresponds to a successful
	// connection attempt and the subsequent handling of the connection.
	for {
		if initial {
			initial = false
		} else {
			b.Remote <- config.Message{
				Username: "system",
				Text:     "rejoin",
				Channel:  "",
				Account:  b.Account,
				Event:    config.EventRejoinChannels,
			}
		}

		if err := b.handleXMPP(); err != nil {
			b.Log.WithError(err).Error("Disconnected.")
			b.setConnected(false)
		}

		// Reconnection loop using an exponential back-off strategy. We
		// only break out of the loop if we have successfully reconnected.
		for {
			d := bf.Duration()
			b.Log.Infof("Reconnecting in %s.", d)
			time.Sleep(d)

			b.Log.Infof("Reconnecting now.")
			if err := b.createXMPP(); err == nil {
				b.setConnected(true)
				bf.Reset()
				break
			}
			b.Log.Warn("Failed to reconnect.")
		}
	}
}

func (b *Bxmpp) xmppKeepAlive() chan bool {
	done := make(chan bool)
	go func() {
		ticker := time.NewTicker(90 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				b.Log.Debugf("PING")
				if err := b.xc.PingC2S("", ""); err != nil {
					b.Log.Debugf("PING failed %#v", err)
				}
			case <-done:
				return
			}
		}
	}()
	return done
}

func (b *Bxmpp) handleXMPP() error {
	b.startTime = time.Now()

	done := b.xmppKeepAlive()
	defer close(done)

	for {
		m, err := b.xc.Recv()
		if err != nil {
			// An error together with AvatarData is non-fatal
			switch m.(type) {
			case xmpp.AvatarData:
				continue
			default:
				return err
			}
		}

		switch v := m.(type) {
		case xmpp.Chat:
			if v.Type == "groupchat" {
				b.Log.Debugf("== Receiving %#v", v)

				// Skip invalid messages.
				if b.skipMessage(v) {
					continue
				}

				var event string
				if strings.Contains(v.Text, "has set the subject to:") {
					event = config.EventTopicChange
				}

				available, sok := b.avatarAvailability[v.Remote]
				avatar := ""
				if !sok {
					b.Log.Debugf("Requesting avatar data")
					b.avatarAvailability[v.Remote] = false
					b.xc.AvatarRequestData(v.Remote)
				} else if available {
					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
				}

				msgID := v.ID
				if v.ReplaceID != "" {
					msgID = v.ReplaceID
				}
				rmsg := config.Message{
					Username: b.parseNick(v.Remote),
					Text:     v.Text,
					Channel:  b.parseChannel(v.Remote),
					Account:  b.Account,
					Avatar:   avatar,
					UserID:   v.Remote,
					ID:       msgID,
					Event:    event,
				}

				// Check if we have an action event.
				var ok bool
				rmsg.Text, ok = b.replaceAction(rmsg.Text)
				if ok {
					rmsg.Event = config.EventUserAction
				}

				b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
				b.Log.Debugf("<= Message is %#v", rmsg)
				b.Remote <- rmsg
			}
		case xmpp.AvatarData:
			b.handleDownloadAvatar(v)
			b.avatarAvailability[v.From] = true
			b.Log.Debugf("Avatar for %s is now available", v.From)
		case xmpp.Presence:
			// Do nothing.
		}
	}
}

func (b *Bxmpp) replaceAction(text string) (string, bool) {
	if strings.HasPrefix(text, "/me ") {
		return strings.Replace(text, "/me ", "", -1), true
	}
	return text, false
}

// handleUploadFile handles native upload of files
func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
	var urlDesc string

	for _, file := range msg.Extra["file"] {
		fileInfo := file.(config.FileInfo)
		if fileInfo.Comment != "" {
			msg.Text += fileInfo.Comment + ": "
		}
		if fileInfo.URL != "" {
			msg.Text = fileInfo.URL
			if fileInfo.Comment != "" {
				msg.Text = fileInfo.Comment + ": " + fileInfo.URL
				urlDesc = fileInfo.Comment
			}
		}
		if _, err := b.xc.Send(xmpp.Chat{
			Type:   "groupchat",
			Remote: msg.Channel + "@" + b.GetString("Muc"),
			Text:   msg.Username + msg.Text,
		}); err != nil {
			return err
		}

		if fileInfo.URL != "" {
			if _, err := b.xc.SendOOB(xmpp.Chat{
				Type:    "groupchat",
				Remote:  msg.Channel + "@" + b.GetString("Muc"),
				Ooburl:  fileInfo.URL,
				Oobdesc: urlDesc,
			}); err != nil {
				b.Log.WithError(err).Warn("Failed to send share URL.")
			}
		}
	}
	return nil
}

func (b *Bxmpp) parseNick(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) > 1 {
		s = strings.Split(s[1], "/")
		if len(s) == 2 {
			return s[1] // nick
		}
	}
	return ""
}

func (b *Bxmpp) parseChannel(remote string) string {
	s := strings.Split(remote, "@")
	if len(s) >= 2 {
		return s[0] // channel
	}
	return ""
}

// skipMessage skips messages that need to be skipped
func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
	// skip messages from ourselves
	if b.parseNick(message.Remote) == b.GetString("Nick") {
		return true
	}

	// skip empty messages
	if message.Text == "" {
		return true
	}

	// skip subject messages
	if strings.Contains(message.Text, "</subject>") {
		return true
	}

	// do not show subjects on connect #732
	if strings.Contains(message.Text, "has set the subject to:") && time.Since(b.startTime) < time.Second*5 {
		return true
	}

	// Ignore messages posted by our webhook
	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
		return true
	}

	// skip delayed messages
	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
}

func (b *Bxmpp) setConnected(state bool) {
	b.Lock()
	b.connected = state
	defer b.Unlock()
}

func (b *Bxmpp) Connected() bool {
	b.RLock()
	defer b.RUnlock()
	return b.connected
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"

	dtesting "github.com/fsouza/go-dockerclient/testing"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/provisiontest"
	"github.com/tsuru/tsuru/safe"
	"gopkg.in/check.v1"
)

func (s *S) TestRebalanceContainersManyAppsSegStress(c *check.C) {
	var nodes []cluster.Node
	var nodeHosts []string
	for i := 0; i < 6; i++ {
		newIp := fmt.Sprintf("127.0.0.%d", i+1)
		otherServer, err := dtesting.NewServer(newIp+":0", nil, nil)
		c.Assert(err, check.IsNil)
		defer otherServer.Stop()
		nodes = append(nodes, cluster.Node{Address: otherServer.URL(), Metadata: map[string]string{"pool": "pool1"}})
		nodeHosts = append(nodeHosts, net.URLToHost(otherServer.URL()))
	}
	var err error
	p := &dockerProvisioner{}
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.storage, err = buildClusterStorage()
	c.Assert(err, check.IsNil)
	p.scheduler = &segregatedScheduler{provisioner: p}
	p.cluster, err = cluster.New(p.scheduler, p.storage, nodes...)
	c.Assert(err, check.IsNil)
	opts := provision.AddPoolOptions{Name: "pool1"}
	err = provision.AddPool(opts)
	c.Assert(err, check.IsNil)
	err = provision.AddTeamsToPool("pool1", []string{"team1"})
	c.Assert(err, check.IsNil)
	variation := []int{10, 20, 30, 40, 50, 100}
	maxContainers := 40
	for i := 0; i < maxContainers; i++ {
		appName := fmt.Sprintf("myapp-%d", i)
		err = s.newFakeImage(p, "tsuru/app-"+appName, nil)
		c.Assert(err, check.IsNil)
		appInstance := provisiontest.NewFakeApp(appName, "python", 0)
		defer p.Destroy(appInstance)
		p.Provision(appInstance)
		imageId, err := appCurrentImageName(appInstance.GetName())
		c.Assert(err, check.IsNil)
		var chosenNode string
		for j := range variation {
			if i < (maxContainers*variation[j])/100 {
				chosenNode = nodeHosts[j]
				break
			}
		}
		args := changeUnitsPipelineArgs{
			app:         appInstance,
			toAdd:       map[string]*containersToAdd{"web": {Quantity: 6}},
			imageId:     imageId,
			provisioner: p,
			toHost:      chosenNode,
		}
		pipeline := action.NewPipeline(
			&provisionAddUnitsToHost,
			&bindAndHealthcheck,
			&addNewRoutes,
			&updateAppImage,
		)
		err = pipeline.Execute(args)
		c.Assert(err, check.IsNil)
		appStruct := &app.App{
			Name:      appInstance.GetName(),
			TeamOwner: "team1",
			Pool:      "pool1",
		}
		err = s.storage.Apps().Insert(appStruct)
		c.Assert(err, check.IsNil)
	}
	buf := safe.NewBuffer(nil)
	cloneProv, err := p.rebalanceContainersByFilter(buf, []string{}, map[string]string{"pool": "pool1"}, false)
	c.Assert(err, check.IsNil)
	c.Assert(cloneProv.cluster.Healer, check.Equals, p.cluster.Healer)
	for i := range nodeHosts {
		conts, err := p.listContainersByHost(nodeHosts[i])
		c.Assert(err, check.IsNil)
		c.Logf("containers in %q: %d", nodeHosts[i], len(conts))
		c.Check(len(conts) >= 39 || len(conts) <= 41, check.Equals, true)
	}
}
<commit_msg>fix shadows declaration<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"

	dtesting "github.com/fsouza/go-dockerclient/testing"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/provisiontest"
	"github.com/tsuru/tsuru/safe"
	"gopkg.in/check.v1"
)

func (s *S) TestRebalanceContainersManyAppsSegStress(c *check.C) {
	var nodes []cluster.Node
	var nodeHosts []string
	for i := 0; i < 6; i++ {
		newIp := fmt.Sprintf("127.0.0.%d", i+1)
		otherServer, err := dtesting.NewServer(newIp+":0", nil, nil)
		c.Assert(err, check.IsNil)
		defer otherServer.Stop()
		nodes = append(nodes, cluster.Node{Address: otherServer.URL(), Metadata: map[string]string{"pool": "pool1"}})
		nodeHosts = append(nodeHosts, net.URLToHost(otherServer.URL()))
	}
	var err error
	p := &dockerProvisioner{}
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.storage, err = buildClusterStorage()
	c.Assert(err, check.IsNil)
	p.scheduler = &segregatedScheduler{provisioner: p}
	p.cluster, err = cluster.New(p.scheduler, p.storage, nodes...)
	c.Assert(err, check.IsNil)
	opts := provision.AddPoolOptions{Name: "pool1"}
	err = provision.AddPool(opts)
	c.Assert(err, check.IsNil)
	err = provision.AddTeamsToPool("pool1", []string{"team1"})
	c.Assert(err, check.IsNil)
	variation := []int{10, 20, 30, 40, 50, 100}
	maxContainers := 40
	for i := 0; i < maxContainers; i++ {
		appName := fmt.Sprintf("myapp-%d", i)
		err = s.newFakeImage(p, "tsuru/app-"+appName, nil)
		c.Assert(err, check.IsNil)
		appInstance := provisiontest.NewFakeApp(appName, "python", 0)
		defer p.Destroy(appInstance)
		p.Provision(appInstance)
		imageId, aErr := appCurrentImageName(appInstance.GetName())
		c.Assert(aErr, check.IsNil)
		var chosenNode string
		for j := range variation {
			if i < (maxContainers*variation[j])/100 {
				chosenNode = nodeHosts[j]
				break
			}
		}
		args := changeUnitsPipelineArgs{
			app:         appInstance,
			toAdd:       map[string]*containersToAdd{"web": {Quantity: 6}},
			imageId:     imageId,
			provisioner: p,
			toHost:      chosenNode,
		}
		pipeline := action.NewPipeline(
			&provisionAddUnitsToHost,
			&bindAndHealthcheck,
			&addNewRoutes,
			&updateAppImage,
		)
		err = pipeline.Execute(args)
		c.Assert(err, check.IsNil)
		appStruct := &app.App{
			Name:      appInstance.GetName(),
			TeamOwner: "team1",
			Pool:      "pool1",
		}
		err = s.storage.Apps().Insert(appStruct)
		c.Assert(err, check.IsNil)
	}
	buf := safe.NewBuffer(nil)
	cloneProv, err := p.rebalanceContainersByFilter(buf, []string{}, map[string]string{"pool": "pool1"}, false)
	c.Assert(err, check.IsNil)
	c.Assert(cloneProv.cluster.Healer, check.Equals, p.cluster.Healer)
	for i := range nodeHosts {
		conts, err := p.listContainersByHost(nodeHosts[i])
		c.Assert(err, check.IsNil)
		c.Logf("containers in %q: %d", nodeHosts[i], len(conts))
		c.Check(len(conts) >= 39 || len(conts) <= 41, check.Equals, true)
	}
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"net"
	"net/http"

	"github.com/elazarl/go-bindata-assetfs"
	router "github.com/gorilla/mux"
	jsonrpc "github.com/gorilla/rpc/v2"
	"github.com/gorilla/rpc/v2/json2"
	"github.com/minio/minio-go"
	"github.com/minio/minio/pkg/fs"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/s3/signature4"
	"github.com/minio/miniobrowser"
)

// storageAPI container for S3 compatible API.
type storageAPI struct {
	// Once true log all incoming requests.
	AccessLog bool
	// Filesystem instance.
	Filesystem fs.Filesystem
	// Signature instance.
	Signature *signature4.Sign
	// Region instance.
	Region string
}

// webAPI container for Web API.
type webAPI struct {
	// FSPath filesystem path.
	FSPath string
	// Once true log all incoming request.
	AccessLog bool
	// Minio client instance.
	Client minio.CloudStorageClient

	// private params.
	apiAddress string // api destination address.
	// accessKeys kept to be used internally.
	accessKeyID     string
	secretAccessKey string
}

// indexHandler - Handler to serve index.html
type indexHandler struct {
	handler http.Handler
}

func (h indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	r.URL.Path = privateBucket + "/"
	h.handler.ServeHTTP(w, r)
}

const assetPrefix = "production"

func assetFS() *assetfs.AssetFS {
	return &assetfs.AssetFS{
		Asset:     miniobrowser.Asset,
		AssetDir:  miniobrowser.AssetDir,
		AssetInfo: miniobrowser.AssetInfo,
		Prefix:    assetPrefix,
	}
}

// registerAPIHandlers - register all the handlers to their respective paths
func registerAPIHandlers(mux *router.Router, a storageAPI, w *webAPI) {
	// Minio rpc router
	minio := mux.NewRoute().PathPrefix(privateBucket).Subrouter()

	// Initialize json rpc handlers.
	rpc := jsonrpc.NewServer()
	codec := json2.NewCodec()
	rpc.RegisterCodec(codec, "application/json")
	rpc.RegisterCodec(codec, "application/json; charset=UTF-8")
	rpc.RegisterService(w, "Web")

	// RPC handler at URI - /minio/rpc
	minio.Path("/rpc").Handler(rpc)
	// Serve javascript files and favicon.ico from assets
	minio.Path("/{assets:[^/]+.js|favicon.ico}").Handler(http.StripPrefix(privateBucket, http.FileServer(assetFS())))
	// Serve index.html for rest of the requests
	minio.Path("/{index:.*}").Handler(indexHandler{http.StripPrefix(privateBucket, http.FileServer(assetFS()))})

	// API Router
	api := mux.NewRoute().PathPrefix("/").Subrouter()

	// Bucket router
	bucket := api.PathPrefix("/{bucket}").Subrouter()

	/// Object operations

	// HeadObject
	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(a.HeadObjectHandler)
	// PutObjectPart
	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(a.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
	// ListObjectPxarts
	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(a.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
	// CompleteMultipartUpload
	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(a.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
	// NewMultipartUpload
	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(a.NewMultipartUploadHandler).Queries("uploads", "")
	// AbortMultipartUpload
	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(a.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
	// GetObject
	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(a.GetObjectHandler)
	// CopyObject
	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/).*?").HandlerFunc(a.CopyObjectHandler)
	// PutObject
	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(a.PutObjectHandler)
	// DeleteObject
	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(a.DeleteObjectHandler)

	/// Bucket operations

	// GetBucketLocation
	bucket.Methods("GET").HandlerFunc(a.GetBucketLocationHandler).Queries("location", "")
	// GetBucketACL
	bucket.Methods("GET").HandlerFunc(a.GetBucketACLHandler).Queries("acl", "")
	// ListMultipartUploads
	bucket.Methods("GET").HandlerFunc(a.ListMultipartUploadsHandler).Queries("uploads", "")
	// ListObjects
	bucket.Methods("GET").HandlerFunc(a.ListObjectsHandler)
	// PutBucketACL
	bucket.Methods("PUT").HandlerFunc(a.PutBucketACLHandler).Queries("acl", "")
	// PutBucket
	bucket.Methods("PUT").HandlerFunc(a.PutBucketHandler)
	// HeadBucket
	bucket.Methods("HEAD").HandlerFunc(a.HeadBucketHandler)
	// PostPolicy
	bucket.Methods("POST").HandlerFunc(a.PostPolicyBucketHandler)
	// DeleteBucket
	bucket.Methods("DELETE").HandlerFunc(a.DeleteBucketHandler)

	/// Root operation

	// ListBuckets
	api.Methods("GET").HandlerFunc(a.ListBucketsHandler)
}

// initWeb instantiate a new Web.
func initWeb(conf cloudServerConfig) *webAPI {
	// Split host port.
	host, port, e := net.SplitHostPort(conf.Address)
	fatalIf(probe.NewError(e), "Unable to parse web addess.", nil)

	// Default host is 'localhost', if no host present.
	if host == "" {
		host = "localhost"
	}

	// Initialize minio client for AWS Signature Version '4'
	inSecure := !conf.TLS // Insecure true when TLS is false.
	client, e := minio.NewV4(net.JoinHostPort(host, port), conf.AccessKeyID, conf.SecretAccessKey, inSecure)
	fatalIf(probe.NewError(e), "Unable to initialize minio client", nil)

	w := &webAPI{
		FSPath:          conf.Path,
		AccessLog:       conf.AccessLog,
		Client:          client,
		apiAddress:      conf.Address,
		accessKeyID:     conf.AccessKeyID,
		secretAccessKey: conf.SecretAccessKey,
	}
	return w
}

// initAPI instantiate a new StorageAPI.
func initAPI(conf cloudServerConfig) storageAPI {
	fs, err := fs.New(conf.Path, conf.MinFreeDisk)
	fatalIf(err.Trace(), "Initializing filesystem failed.", nil)

	sign, err := signature4.New(conf.AccessKeyID, conf.SecretAccessKey, conf.Region)
	fatalIf(err.Trace(conf.AccessKeyID, conf.SecretAccessKey, conf.Region), "Initializing signature version '4' failed.", nil)

	return storageAPI{
		AccessLog:  conf.AccessLog,
		Filesystem: fs,
		Signature:  sign,
		Region:     conf.Region,
	}
}

// server handler returns final handler before initializing server.
func serverHandler(conf cloudServerConfig) http.Handler {
	// Initialize API.
	api := initAPI(conf)

	// Initialize Web.
	web := initWeb(conf)

	var handlerFns = []HandlerFunc{
		// Redirect some pre-defined browser request paths to a static
		// location prefix.
		setBrowserRedirectHandler,
		// Validates if incoming request is for restricted buckets.
		setPrivateBucketHandler,
		// Adds cache control for all browser requests.
		setBrowserCacheControlHandler,
		// Validates all incoming requests to have a valid date header.
		setTimeValidityHandler,
		// CORS setting for all browser API requests.
		setCorsHandler,
		// Validates all incoming URL resources, for invalid/unsupported
		// resources client receives a HTTP error.
		setIgnoreResourcesHandler,
		// Auth handler verifies incoming authorization headers and
		// routes them accordingly. Client receives a HTTP error for
		// invalid/unsupported signatures.
		setAuthHandler,
	}

	// Initialize router.
	mux := router.NewRouter()

	// Register all API handlers.
	registerAPIHandlers(mux, api, web)

	// Register rest of the handlers.
	return registerHandlers(mux, handlerFns...)
}
<commit_msg>routers: Support special asset files.<commit_after>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"net"
	"net/http"

	"github.com/elazarl/go-bindata-assetfs"
	router "github.com/gorilla/mux"
	jsonrpc "github.com/gorilla/rpc/v2"
	"github.com/gorilla/rpc/v2/json2"
	"github.com/minio/minio-go"
	"github.com/minio/minio/pkg/fs"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/s3/signature4"
	"github.com/minio/miniobrowser"
)

// storageAPI container for S3 compatible API.
type storageAPI struct {
	// Once true log all incoming requests.
	AccessLog bool
	// Filesystem instance.
	Filesystem fs.Filesystem
	// Signature instance.
	Signature *signature4.Sign
	// Region instance.
	Region string
}

// webAPI container for Web API.
type webAPI struct {
	// FSPath filesystem path.
	FSPath string
	// Once true log all incoming request.
	AccessLog bool
	// Minio client instance.
	Client minio.CloudStorageClient

	// private params.
	apiAddress string // api destination address.
	// accessKeys kept to be used internally.
	accessKeyID     string
	secretAccessKey string
}

// indexHandler - Handler to serve index.html
type indexHandler struct {
	handler http.Handler
}

func (h indexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	r.URL.Path = privateBucket + "/"
	h.handler.ServeHTTP(w, r)
}

const assetPrefix = "production"

func assetFS() *assetfs.AssetFS {
	return &assetfs.AssetFS{
		Asset:     miniobrowser.Asset,
		AssetDir:  miniobrowser.AssetDir,
		AssetInfo: miniobrowser.AssetInfo,
		Prefix:    assetPrefix,
	}
}

// specialAssets are files which are unique files not embedded inside index_bundle.js.
const specialAssets = "loader.css|logo.svg|firefox.png|safari.png|chrome.png|favicon.ico"

// registerAPIHandlers - register all the handlers to their respective paths
func registerAPIHandlers(mux *router.Router, a storageAPI, w *webAPI) {
	// Minio rpc router
	minio := mux.NewRoute().PathPrefix(privateBucket).Subrouter()

	// Initialize json rpc handlers.
	rpc := jsonrpc.NewServer()
	codec := json2.NewCodec()
	rpc.RegisterCodec(codec, "application/json")
	rpc.RegisterCodec(codec, "application/json; charset=UTF-8")
	rpc.RegisterService(w, "Web")

	// RPC handler at URI - /minio/rpc
	minio.Path("/rpc").Handler(rpc)
	// Serve all assets.
	minio.Path(fmt.Sprintf("/{assets:[^/]+.js|%s}", specialAssets)).Handler(http.StripPrefix(privateBucket, http.FileServer(assetFS())))
	// Serve index.html for rest of the requests
	minio.Path("/{index:.*}").Handler(indexHandler{http.StripPrefix(privateBucket, http.FileServer(assetFS()))})

	// API Router
	api := mux.NewRoute().PathPrefix("/").Subrouter()

	// Bucket router
	bucket := api.PathPrefix("/{bucket}").Subrouter()

	/// Object operations

	// HeadObject
	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(a.HeadObjectHandler)
	// PutObjectPart
	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(a.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
	// ListObjectPxarts
	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(a.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
	// CompleteMultipartUpload
	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(a.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
	// NewMultipartUpload
	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(a.NewMultipartUploadHandler).Queries("uploads", "")
	// AbortMultipartUpload
	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(a.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
	// GetObject
	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(a.GetObjectHandler)
	// CopyObject
	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/).*?").HandlerFunc(a.CopyObjectHandler)
	// PutObject
	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(a.PutObjectHandler)
	// DeleteObject
	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(a.DeleteObjectHandler)

	/// Bucket operations

	// GetBucketLocation
	bucket.Methods("GET").HandlerFunc(a.GetBucketLocationHandler).Queries("location", "")
	// GetBucketACL
	bucket.Methods("GET").HandlerFunc(a.GetBucketACLHandler).Queries("acl", "")
	// ListMultipartUploads
	bucket.Methods("GET").HandlerFunc(a.ListMultipartUploadsHandler).Queries("uploads", "")
	// ListObjects
	bucket.Methods("GET").HandlerFunc(a.ListObjectsHandler)
	// PutBucketACL
	bucket.Methods("PUT").HandlerFunc(a.PutBucketACLHandler).Queries("acl", "")
	// PutBucket
	bucket.Methods("PUT").HandlerFunc(a.PutBucketHandler)
	// HeadBucket
	bucket.Methods("HEAD").HandlerFunc(a.HeadBucketHandler)
	// PostPolicy
	bucket.Methods("POST").HandlerFunc(a.PostPolicyBucketHandler)
	// DeleteBucket
	bucket.Methods("DELETE").HandlerFunc(a.DeleteBucketHandler)

	/// Root operation

	// ListBuckets
	api.Methods("GET").HandlerFunc(a.ListBucketsHandler)
}

// initWeb instantiate a new Web.
func initWeb(conf cloudServerConfig) *webAPI {
	// Split host port.
	host, port, e := net.SplitHostPort(conf.Address)
	fatalIf(probe.NewError(e), "Unable to parse web addess.", nil)

	// Default host is 'localhost', if no host present.
	if host == "" {
		host = "localhost"
	}

	// Initialize minio client for AWS Signature Version '4'
	inSecure := !conf.TLS // Insecure true when TLS is false.
	client, e := minio.NewV4(net.JoinHostPort(host, port), conf.AccessKeyID, conf.SecretAccessKey, inSecure)
	fatalIf(probe.NewError(e), "Unable to initialize minio client", nil)

	w := &webAPI{
		FSPath:          conf.Path,
		AccessLog:       conf.AccessLog,
		Client:          client,
		apiAddress:      conf.Address,
		accessKeyID:     conf.AccessKeyID,
		secretAccessKey: conf.SecretAccessKey,
	}
	return w
}

// initAPI instantiate a new StorageAPI.
func initAPI(conf cloudServerConfig) storageAPI {
	fs, err := fs.New(conf.Path, conf.MinFreeDisk)
	fatalIf(err.Trace(), "Initializing filesystem failed.", nil)

	sign, err := signature4.New(conf.AccessKeyID, conf.SecretAccessKey, conf.Region)
	fatalIf(err.Trace(conf.AccessKeyID, conf.SecretAccessKey, conf.Region), "Initializing signature version '4' failed.", nil)

	return storageAPI{
		AccessLog:  conf.AccessLog,
		Filesystem: fs,
		Signature:  sign,
		Region:     conf.Region,
	}
}

// server handler returns final handler before initializing server.
func serverHandler(conf cloudServerConfig) http.Handler {
	// Initialize API.
	api := initAPI(conf)

	// Initialize Web.
	web := initWeb(conf)

	var handlerFns = []HandlerFunc{
		// Redirect some pre-defined browser request paths to a static
		// location prefix.
		setBrowserRedirectHandler,
		// Validates if incoming request is for restricted buckets.
		setPrivateBucketHandler,
		// Adds cache control for all browser requests.
		setBrowserCacheControlHandler,
		// Validates all incoming requests to have a valid date header.
		setTimeValidityHandler,
		// CORS setting for all browser API requests.
		setCorsHandler,
		// Validates all incoming URL resources, for invalid/unsupported
		// resources client receives a HTTP error.
		setIgnoreResourcesHandler,
		// Auth handler verifies incoming authorization headers and
		// routes them accordingly. Client receives a HTTP error for
		// invalid/unsupported signatures.
		setAuthHandler,
	}

	// Initialize router.
	mux := router.NewRouter()

	// Register all API handlers.
	registerAPIHandlers(mux, api, web)

	// Register rest of the handlers.
	return registerHandlers(mux, handlerFns...)
}
<|endoftext|>


<commit_before>package dht

import (
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	peer "github.com/jbenet/go-ipfs/peer"
	"github.com/jbenet/go-ipfs/routing"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	kb "github.com/jbenet/go-ipfs/routing/kbucket"
	u "github.com/jbenet/go-ipfs/util"
)

// This file implements the Routing interface for the IpfsDHT struct.

// Basic Put/Get

// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error {
	log.Debugf("PutValue %s", key)
	err := dht.putLocal(key, value)
	if err != nil {
		return err
	}

	rec, err := dht.makePutRecord(key, value)
	if err != nil {
		log.Error("Creation of record failed!")
		return err
	}

	var peers []peer.Peer
	for _, route := range dht.routingTables {
		npeers := route.NearestPeers(kb.ConvertKey(key), KValue)
		peers = append(peers, npeers...)
	}

	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {
		log.Debugf("%s PutValue qry part %v", dht.self, p)
		err := dht.putValueToNetwork(ctx, p, string(key), rec)
		if err != nil {
			return nil, err
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err = query.Run(ctx, peers)
	return err
}

// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
	log.Debugf("Get Value [%s]", key)

	// If we have it local, dont bother doing an RPC!
	// NOTE: this might not be what we want to do...
	val, err := dht.getLocal(key)
	if err == nil {
		log.Debug("Got value locally!")
		return val, nil
	}

	// get closest peers in the routing tables
	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertKey(key), PoolSize)
	if closest == nil || len(closest) == 0 {
		log.Warning("Got no peers back from routing table!")
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		val, peers, err := dht.getValueOrPeers(ctx, p, key, routeLevel)
		if err != nil {
			return nil, err
		}

		res := &dhtQueryResult{value: val, closerPeers: peers}
		if val != nil {
			res.success = true
		}

		return res, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("GetValue %v %v", key, result.value)
	if result.value == nil {
		return nil, routing.ErrNotFound
	}

	return result.value, nil
}

// Value provider layer of indirection.
// This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

// Provide makes this node announce that it can provide a value for the given key
func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error {

	dht.providers.AddProvider(key, dht.self)
	peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize)
	if len(peers) == 0 {
		return nil
	}

	//TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers.
	// `peers` are the closest peers we have, not the ones that should get the value.
	for _, p := range peers {
		err := dht.putProvider(ctx, p, string(key))
		if err != nil {
			return err
		}
	}
	return nil
}

func (dht *IpfsDHT) FindProvidersAsync(ctx context.Context, key u.Key, count int) <-chan peer.Peer {
	log.Event(ctx, "findProviders", &key)
	peerOut := make(chan peer.Peer, count)
	go func() {
		defer close(peerOut)

		ps := newPeerSet()
		// TODO may want to make this function async to hide latency
		provs := dht.providers.GetProviders(ctx, key)
		for _, p := range provs {
			count--
			// NOTE: assuming that this list of peers is unique
			ps.Add(p)
			select {
			case peerOut <- p:
			case <-ctx.Done():
				return
			}
			if count <= 0 {
				return
			}
		}

		var wg sync.WaitGroup
		for _, pp := range dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue) {
			wg.Add(1)
			go func(p peer.Peer) {
				defer wg.Done()
				pmes, err := dht.findProvidersSingle(ctx, p, key, 0)
				if err != nil {
					log.Error(err)
					return
				}
				dht.addPeerListAsync(ctx, key, pmes.GetProviderPeers(), ps, count, peerOut)
			}(pp)
		}
		wg.Wait()
	}()
	return peerOut
}

func (dht *IpfsDHT) addPeerListAsync(ctx context.Context, k u.Key, peers []*pb.Message_Peer, ps *peerSet, count int, out chan peer.Peer) {
	var wg sync.WaitGroup
	for _, pbp := range peers {
		wg.Add(1)
		go func(mp *pb.Message_Peer) {
			defer wg.Done()
			// construct new peer
			p, err := dht.ensureConnectedToPeer(ctx, mp)
			if err != nil {
				log.Errorf("%s", err)
				return
			}
			if p == nil {
				log.Error("Got nil peer from ensureConnectedToPeer")
				return
			}

			dht.providers.AddProvider(k, p)
			if ps.AddIfSmallerThan(p, count) {
				select {
				case out <- p:
				case <-ctx.Done():
					return
				}
			} else if ps.Size() >= count {
				return
			}
		}(pbp)
	}
	wg.Wait()
}

// Find specific Peer
// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.FindLocal(id)
	if p != nil {
		return p, nil
	}

	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// Sanity...
	for _, p := range closest {
		if p.ID().Equal(id) {
			log.Error("Found target peer in list of closest peers...")
			return p, nil
		}
	}

	// setup the Query
	query := newQuery(u.Key(id), dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			return nil, err
		}

		closer := pmes.GetCloserPeers()
		var clpeers []peer.Peer
		for _, pbp := range closer {
			np, err := dht.getPeer(peer.ID(pbp.GetId()))
			if err != nil {
				log.Warningf("Received invalid peer from query: %v", err)
				continue
			}
			ma, err := pbp.Address()
			if err != nil {
				log.Warning("Received peer with bad or missing address.")
				continue
			}
			np.AddAddress(ma)
			if pbp.GetId() == string(id) {
				return &dhtQueryResult{
					peer:    np,
					success: true,
				}, nil
			}
			clpeers = append(clpeers, np)
		}

		return &dhtQueryResult{closerPeers: clpeers}, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("FindPeer %v %v", id, result.success)
	if result.peer == nil {
		return nil, routing.ErrNotFound
	}

	return result.peer, nil
}

// Ping a peer, log the time it took
func (dht *IpfsDHT) Ping(ctx context.Context, p peer.Peer) error {
	// Thoughts: maybe this should accept an ID and do a peer lookup?
	log.Infof("ping %s start", p)

	pmes := pb.NewMessage(pb.Message_PING, "", 0)
	_, err := dht.sendRequest(ctx, p, pmes)
	log.Infof("ping %s end (err = %s)", p, err)
	return err
}
<commit_msg>style: readability<commit_after>package dht

import (
	"sync"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	peer "github.com/jbenet/go-ipfs/peer"
	"github.com/jbenet/go-ipfs/routing"
	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
	kb "github.com/jbenet/go-ipfs/routing/kbucket"
	u "github.com/jbenet/go-ipfs/util"
)

// This file implements the Routing interface for the IpfsDHT struct.

// Basic Put/Get

// PutValue adds value corresponding to given Key.
// This is the top level "Store" operation of the DHT
func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error {
	log.Debugf("PutValue %s", key)
	err := dht.putLocal(key, value)
	if err != nil {
		return err
	}

	rec, err := dht.makePutRecord(key, value)
	if err != nil {
		log.Error("Creation of record failed!")
		return err
	}

	var peers []peer.Peer
	for _, route := range dht.routingTables {
		npeers := route.NearestPeers(kb.ConvertKey(key), KValue)
		peers = append(peers, npeers...)
	}

	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {
		log.Debugf("%s PutValue qry part %v", dht.self, p)
		err := dht.putValueToNetwork(ctx, p, string(key), rec)
		if err != nil {
			return nil, err
		}
		return &dhtQueryResult{success: true}, nil
	})

	_, err = query.Run(ctx, peers)
	return err
}

// GetValue searches for the value corresponding to given Key.
// If the search does not succeed, a multiaddr string of a closer peer is
// returned along with util.ErrSearchIncomplete
func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
	log.Debugf("Get Value [%s]", key)

	// If we have it local, dont bother doing an RPC!
	// NOTE: this might not be what we want to do...
	val, err := dht.getLocal(key)
	if err == nil {
		log.Debug("Got value locally!")
		return val, nil
	}

	// get closest peers in the routing tables
	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertKey(key), PoolSize)
	if closest == nil || len(closest) == 0 {
		log.Warning("Got no peers back from routing table!")
		return nil, kb.ErrLookupFailure
	}

	// setup the Query
	query := newQuery(key, dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		val, peers, err := dht.getValueOrPeers(ctx, p, key, routeLevel)
		if err != nil {
			return nil, err
		}

		res := &dhtQueryResult{value: val, closerPeers: peers}
		if val != nil {
			res.success = true
		}

		return res, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("GetValue %v %v", key, result.value)
	if result.value == nil {
		return nil, routing.ErrNotFound
	}

	return result.value, nil
}

// Value provider layer of indirection.
// This is what DSHTs (Coral and MainlineDHT) do to store large values in a DHT.

// Provide makes this node announce that it can provide a value for the given key
func (dht *IpfsDHT) Provide(ctx context.Context, key u.Key) error {

	dht.providers.AddProvider(key, dht.self)
	peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), PoolSize)
	if len(peers) == 0 {
		return nil
	}

	//TODO FIX: this doesn't work! it needs to be sent to the actual nearest peers.
	// `peers` are the closest peers we have, not the ones that should get the value.
	for _, p := range peers {
		err := dht.putProvider(ctx, p, string(key))
		if err != nil {
			return err
		}
	}
	return nil
}

func (dht *IpfsDHT) FindProvidersAsync(ctx context.Context, key u.Key, count int) <-chan peer.Peer {
	log.Event(ctx, "findProviders", &key)
	peerOut := make(chan peer.Peer, count)
	go func() {
		defer close(peerOut)

		ps := newPeerSet()
		// TODO may want to make this function async to hide latency
		provs := dht.providers.GetProviders(ctx, key)
		for _, p := range provs {
			count--
			// NOTE: assuming that this list of peers is unique
			ps.Add(p)
			select {
			case peerOut <- p:
			case <-ctx.Done():
				return
			}
			if count <= 0 {
				return
			}
		}

		var wg sync.WaitGroup
		peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(key), AlphaValue)
		for _, pp := range peers {
			wg.Add(1)
			go func(p peer.Peer) {
				defer wg.Done()
				pmes, err := dht.findProvidersSingle(ctx, p, key, 0)
				if err != nil {
					log.Error(err)
					return
				}
				dht.addPeerListAsync(ctx, key, pmes.GetProviderPeers(), ps, count, peerOut)
			}(pp)
		}
		wg.Wait()
	}()
	return peerOut
}

func (dht *IpfsDHT) addPeerListAsync(ctx context.Context, k u.Key, peers []*pb.Message_Peer, ps *peerSet, count int, out chan peer.Peer) {
	var wg sync.WaitGroup
	for _, pbp := range peers {
		wg.Add(1)
		go func(mp *pb.Message_Peer) {
			defer wg.Done()
			// construct new peer
			p, err := dht.ensureConnectedToPeer(ctx, mp)
			if err != nil {
				log.Errorf("%s", err)
				return
			}
			if p == nil {
				log.Error("Got nil peer from ensureConnectedToPeer")
				return
			}

			dht.providers.AddProvider(k, p)
			if ps.AddIfSmallerThan(p, count) {
				select {
				case out <- p:
				case <-ctx.Done():
					return
				}
			} else if ps.Size() >= count {
				return
			}
		}(pbp)
	}
	wg.Wait()
}

// Find specific Peer
// FindPeer searches for a peer with given ID.
func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.Peer, error) {

	// Check if were already connected to them
	p, _ := dht.FindLocal(id)
	if p != nil {
		return p, nil
	}

	routeLevel := 0
	closest := dht.routingTables[routeLevel].NearestPeers(kb.ConvertPeerID(id), AlphaValue)
	if closest == nil || len(closest) == 0 {
		return nil, kb.ErrLookupFailure
	}

	// Sanity...
	for _, p := range closest {
		if p.ID().Equal(id) {
			log.Error("Found target peer in list of closest peers...")
			return p, nil
		}
	}

	// setup the Query
	query := newQuery(u.Key(id), dht.dialer, func(ctx context.Context, p peer.Peer) (*dhtQueryResult, error) {

		pmes, err := dht.findPeerSingle(ctx, p, id, routeLevel)
		if err != nil {
			return nil, err
		}

		closer := pmes.GetCloserPeers()
		var clpeers []peer.Peer
		for _, pbp := range closer {
			np, err := dht.getPeer(peer.ID(pbp.GetId()))
			if err != nil {
				log.Warningf("Received invalid peer from query: %v", err)
				continue
			}
			ma, err := pbp.Address()
			if err != nil {
				log.Warning("Received peer with bad or missing address.")
				continue
			}
			np.AddAddress(ma)
			if pbp.GetId() == string(id) {
				return &dhtQueryResult{
					peer:    np,
					success: true,
				}, nil
			}
			clpeers = append(clpeers, np)
		}

		return &dhtQueryResult{closerPeers: clpeers}, nil
	})

	// run it!
	result, err := query.Run(ctx, closest)
	if err != nil {
		return nil, err
	}

	log.Debugf("FindPeer %v %v", id, result.success)
	if result.peer == nil {
		return nil, routing.ErrNotFound
	}

	return result.peer, nil
}

// Ping a peer, log the time it took
func (dht *IpfsDHT) Ping(ctx context.Context, p peer.Peer) error {
	// Thoughts: maybe this should accept an ID and do a peer lookup?
	log.Infof("ping %s start", p)

	pmes := pb.NewMessage(pb.Message_PING, "", 0)
	_, err := dht.sendRequest(ctx, p, pmes)
	log.Infof("ping %s end (err = %s)", p, err)
	return err
}
<|endoftext|>


<commit_before>package rss

import (
	"encoding/xml"
	"time"
)

type RSS struct {
	XMLName        xml.Name `xml:"rss"`
	XMLXmlnsAtom   string   `xml:"xmlns:atom,attr"`
	XMLXmlnsItunes string   `xml:"xmlns:itunes,attr"`
	XMLVersion     string   `xml:"version,attr"`
	Channel        *Channel `xml:"channel,omitempty"`
}

type Channel struct {
	Title          string        `xml:"title,omitempty"`
	Description    string        `xml:"description,omitempty"`
	Link           string        `xml:"link,omitempty"`
	Language       string        `xml:"language,omitempty"`
	Copyright      string        `xml:"copyright,omitempty"`
	ChannelImage   *ChannelImage `xml:"image,omitempty"`
	LastBuildDate  string        `xml:"lastBuildDate,omitempty"`
	AtomLink       *AtomLink     `xml:"atom:link,omitempty"`
	ItunesSubtitle string        `xml:"itunes:subtitle,omitempty"`
	ItunesAuthor   string        `xml:"itunes:author,omitempty"`
	ItunesSummary  string        `xml:"itunes:summary,omitempty"`
	ItunesKeywords string        `xml:"itunes:keywords,omitempty"`
	ItunesExplicit string        `xml:"itunes:explicit,omitempty"`
	ItunesOwner    *ItunesOwner  `xml:"itunes:owner,omitempty"`
	ItunesImage    *ItunesImage
	Item           []*Item `xml:"item,omitempty"`
}

type ItunesOwner struct {
	ItunesName  string `xml:"itunes:name,omitempty"`
	ItunesEmail string `xml:"itunes:mail,omitempty"`
}

type ItunesImage struct {
	XMLName xml.Name `xml:"itunes:image,omitempty"`
	Href    string   `xml:"href,attr"`
}

type ChannelImage struct {
	URL   string `xml:"url,omitempty"`
	Title string `xml:"title,omitempty"`
	Link  string `xml:"link,omitempty"`
}

type AtomLink struct {
	Href string `xml:"href,attr,omitempty"`
	Rel  string `xml:"rel,attr,omitempty"`
	Type string `xml:"type,attr,omitempty"`
}

type Item struct {
	Title                   string     `xml:"title,omitempty"`
	Description             string     `xml:"description,omitempty"`
	Guid                    string     `xml:"guid,omitempty"`
	PubDate                 string     `xml:"pubDate,omitempty"`
	Enclosure               *Enclosure `xml:"enclosure,omitempty"`
	ItunesAuthor            string     `xml:"itunes:author,omitempty"`
	ItunesSubtitle          string     `xml:"itunes:subtitle,omitempty"`
	ItunesSummary           string     `xml:"itunes:summary,omitempty"`
	ItunesDuration          string     `xml:"itunes:duration,omitempty"`
	ItunesExplicit          string     `xml:"itunes:explicit,omitempty"`
	ItunesOrder             string     `xml:"itunes:order,omitempty"`
	ItunesisClosedCaptioned string     `xml:"itunes:isClosedCaptioned,omitempty"`
}

type Enclosure struct {
	URL    string `xml:"url,attr,omitempty"`
	Type   string `xml:"type,attr,omitempty"`
	Length int64  `xml:"length,attr,omitempty"`
}

// Sort Interface

type ByPubDate []*Item

func (p ByPubDate) Len() int      { return len(p) }
func (p ByPubDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByPubDate) Less(i, j int) bool {
	iDate, _ := time.Parse(time.RFC1123, p[i].PubDate)
	jDate, _ := time.Parse(time.RFC1123, p[j].PubDate)
	return iDate.Unix() < jDate.Unix()
}
<commit_msg>Add `<itunes:block>` tag<commit_after>package rss

import (
	"encoding/xml"
	"time"
)

type RSS struct {
	XMLName        xml.Name `xml:"rss"`
	XMLXmlnsAtom   string   `xml:"xmlns:atom,attr"`
	XMLXmlnsItunes string   `xml:"xmlns:itunes,attr"`
	XMLVersion     string   `xml:"version,attr"`
	Channel        *Channel `xml:"channel,omitempty"`
}

type Channel struct {
	Title          string        `xml:"title,omitempty"`
	Description    string        `xml:"description,omitempty"`
	Link           string        `xml:"link,omitempty"`
	Language       string        `xml:"language,omitempty"`
	Copyright      string        `xml:"copyright,omitempty"`
	ChannelImage   *ChannelImage `xml:"image,omitempty"`
	LastBuildDate  string        `xml:"lastBuildDate,omitempty"`
	AtomLink       *AtomLink     `xml:"atom:link,omitempty"`
	ItunesSubtitle string        `xml:"itunes:subtitle,omitempty"`
	ItunesAuthor   string        `xml:"itunes:author,omitempty"`
	ItunesSummary  string        `xml:"itunes:summary,omitempty"`
	ItunesKeywords string        `xml:"itunes:keywords,omitempty"`
	ItunesExplicit string        `xml:"itunes:explicit,omitempty"`
	ItunesOwner    *ItunesOwner  `xml:"itunes:owner,omitempty"`
	ItunesImage    *ItunesImage
	Item           []*Item `xml:"item,omitempty"`
}

type ItunesOwner struct {
	ItunesName  string `xml:"itunes:name,omitempty"`
	ItunesEmail string `xml:"itunes:mail,omitempty"`
}

type ItunesImage struct {
	XMLName xml.Name `xml:"itunes:image,omitempty"`
	Href    string   `xml:"href,attr"`
}

type ChannelImage struct {
	URL   string `xml:"url,omitempty"`
	Title string `xml:"title,omitempty"`
	Link  string `xml:"link,omitempty"`
}

type AtomLink struct {
	Href string `xml:"href,attr,omitempty"`
	Rel  string `xml:"rel,attr,omitempty"`
	Type string `xml:"type,attr,omitempty"`
}

type Item struct {
	Title                   string     `xml:"title,omitempty"`
	Description             string     `xml:"description,omitempty"`
	Guid                    string     `xml:"guid,omitempty"`
	PubDate                 string     `xml:"pubDate,omitempty"`
	Enclosure               *Enclosure `xml:"enclosure,omitempty"`
	ItunesAuthor            string     `xml:"itunes:author,omitempty"`
	ItunesSubtitle          string     `xml:"itunes:subtitle,omitempty"`
	ItunesSummary           string     `xml:"itunes:summary,omitempty"`
	ItunesDuration          string     `xml:"itunes:duration,omitempty"`
	ItunesExplicit          string     `xml:"itunes:explicit,omitempty"`
	ItunesOrder             string     `xml:"itunes:order,omitempty"`
	ItunesisClosedCaptioned string     `xml:"itunes:isClosedCaptioned,omitempty"`
	ItunesBlock             string     `xml:"itunes:block,allowempty"`
}

type Enclosure struct {
	URL    string `xml:"url,attr,omitempty"`
	Type   string `xml:"type,attr,omitempty"`
	Length int64  `xml:"length,attr,omitempty"`
}

// Sort Interface

type ByPubDate []*Item

func (p ByPubDate) Len() int      { return len(p) }
func (p ByPubDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByPubDate) Less(i, j int) bool {
	iDate, _ := time.Parse(time.RFC1123, p[i].PubDate)
	jDate, _ := time.Parse(time.RFC1123, p[j].PubDate)
	return iDate.Unix() < jDate.Unix()
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// +build integration

package integration

import (
	"bytes"
	"log"
	"os"
	"path/filepath"
	"testing"
)

func TestReset(t *testing.T) {
	// Clean and start a syncthing instance

	log.Println("Cleaning...")
	err := removeAll("s1", "h1/index*")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir("s1", 0755); err != nil {
		t.Fatal(err)
	}

	log.Println("Creating files...")
	size := createFiles(t)

	p := startInstance(t, 1)
	defer p.Stop() // Not checkedStop, because Syncthing will exit on it's own

	m, err := p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected := size
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
	}

	// Clear all files but restore the folder marker
	log.Println("Cleaning...")
	err = removeAll("s1")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir("s1", 0755); err != nil {
		t.Fatal(err)
	}
	if fd, err := os.Create("s1/.stfolder"); err != nil {
		t.Fatal(err)
	} else {
		fd.Close()
	}

	// Reset indexes of an invalid folder
	log.Println("Reset invalid folder")
	_, err = p.Post("/rest/system/reset?folder=invalid", nil)
	if err == nil {
		t.Fatalf("Cannot reset indexes of an invalid folder")
	}

	// Reset indexes of the default folder
	log.Println("Reset indexes of default folder")
	bs, err := p.Post("/rest/system/reset?folder=default", nil)
	if err != nil {
		t.Fatalf("Failed to reset indexes (default): %v (%s)", err, bytes.TrimSpace(bs))
	}

	// ---- Syncthing exits here ----

	p = startInstance(t, 1)
	defer p.Stop() // Not checkedStop, because Syncthing will exit on it's own

	m, err = p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected = 0
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
	}

	// Recreate the files and scan
	log.Println("Creating files...")
	size = createFiles(t)

	if err := p.Rescan("default"); err != nil {
		t.Fatal(err)
	}

	// Verify that we see them
	m, err = p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected = size
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after second creation phase, %d != %d", m.LocalFiles, expected)
	}

	// Reset all indexes
	log.Println("Reset DB...")
	bs, err = p.Post("/rest/system/reset", nil)
	if err != nil {
		t.Fatalf("Failed to reset indexes (all): %v (%s)", err, bytes.TrimSpace(bs))
	}

	// ---- Syncthing exits here ----

	p = startInstance(t, 1)
	defer checkedStop(t, p)

	m, err = p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected = size
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
	}
}

func createFiles(t *testing.T) int {
	// Create eight empty files and directories
	files := []string{"f1", "f2", "f3", "f4", "f11", "f12", "f13", "f14"}
	dirs := []string{"d1", "d2", "d3", "d4", "d11", "d12", "d13", "d14"}
	all := append(files, dirs...)

	for _, file := range files {
		fd, err := os.Create(filepath.Join("s1", file))
		if err != nil {
			t.Fatal(err)
		}
		fd.Close()
	}

	for _, dir := range dirs {
		err := os.Mkdir(filepath.Join("s1", dir), 0755)
		if err != nil {
			t.Fatal(err)
		}
	}

	return len(all)
}
<commit_msg>Fix TestReset when Syncthing shuts down too fast<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// +build integration

package integration

import (
	"bytes"
	"io"
	"log"
	"os"
	"path/filepath"
	"testing"
)

func TestReset(t *testing.T) {
	// Clean and start a syncthing instance

	log.Println("Cleaning...")
	err := removeAll("s1", "h1/index*")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir("s1", 0755); err != nil {
		t.Fatal(err)
	}

	log.Println("Creating files...")
	size := createFiles(t)

	p := startInstance(t, 1)
	defer p.Stop() // Not checkedStop, because Syncthing will exit on it's own

	m, err := p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected := size
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
	}

	// Clear all files but restore the folder marker
	log.Println("Cleaning...")
	err = removeAll("s1")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir("s1", 0755); err != nil {
		t.Fatal(err)
	}
	if fd, err := os.Create("s1/.stfolder"); err != nil {
		t.Fatal(err)
	} else {
		fd.Close()
	}

	// Reset indexes of an invalid folder
	log.Println("Reset invalid folder")
	_, err = p.Post("/rest/system/reset?folder=invalid", nil)
	if err == nil {
		t.Fatalf("Cannot reset indexes of an invalid folder")
	}

	// Reset indexes of the default folder
	log.Println("Reset indexes of default folder")
	bs, err := p.Post("/rest/system/reset?folder=default", nil)
	if err != nil && err != io.ErrUnexpectedEOF {
		t.Fatalf("Failed to reset indexes (default): %v (%s)", err, bytes.TrimSpace(bs))
	}

	// ---- Syncthing exits here ----

	p = startInstance(t, 1)
	defer p.Stop() // Not checkedStop, because Syncthing will exit on it's own

	m, err = p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected = 0
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
	}

	// Recreate the files and scan
	log.Println("Creating files...")
	size = createFiles(t)

	if err := p.Rescan("default"); err != nil {
		t.Fatal(err)
	}

	// Verify that we see them
	m, err = p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected = size
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after second creation phase, %d != %d", m.LocalFiles, expected)
	}

	// Reset all indexes
	log.Println("Reset DB...")
	bs, err = p.Post("/rest/system/reset", nil)
	if err != nil {
		t.Fatalf("Failed to reset indexes (all): %v (%s)", err, bytes.TrimSpace(bs))
	}

	// ---- Syncthing exits here ----

	p = startInstance(t, 1)
	defer checkedStop(t, p)

	m, err = p.Model("default")
	if err != nil {
		t.Fatal(err)
	}
	expected = size
	if m.LocalFiles != expected {
		t.Fatalf("Incorrect number of files after initial scan, %d != %d", m.LocalFiles, expected)
	}
}

func createFiles(t *testing.T) int {
	// Create eight empty files and directories
	files := []string{"f1", "f2", "f3", "f4", "f11", "f12", "f13", "f14"}
	dirs := []string{"d1", "d2", "d3", "d4", "d11", "d12", "d13", "d14"}
	all := append(files, dirs...)

	for _, file := range files {
		fd, err := os.Create(filepath.Join("s1", file))
		if err != nil {
			t.Fatal(err)
		}
		fd.Close()
	}

	for _, dir := range dirs {
		err := os.Mkdir(filepath.Join("s1", dir), 0755)
		if err != nil {
			t.Fatal(err)
		}
	}

	return len(all)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests that involve both reading and writing.

package zip

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
	"time"
)

func TestOver65kFiles(t *testing.T) {
	buf := new(bytes.Buffer)
	w := NewWriter(buf)
	const nFiles = (1 << 16) + 42
	for i := 0; i < nFiles; i++ {
		_, err := w.CreateHeader(&FileHeader{
			Name:   fmt.Sprintf("%d.dat", i),
			Method: Store, // avoid Issue 6136 and Issue 6138
		})
		if err != nil {
			t.Fatalf("creating file %d: %v", i, err)
		}
	}
	if err := w.Close(); err != nil {
		t.Fatalf("Writer.Close: %v", err)
	}
	s := buf.String()
	zr, err := NewReader(strings.NewReader(s), int64(len(s)))
	if err != nil {
		t.Fatalf("NewReader: %v", err)
	}
	if got := len(zr.File); got != nFiles {
		t.Fatalf("File contains %d files, want %d", got, nFiles)
	}
	for i := 0; i < nFiles; i++ {
		want := fmt.Sprintf("%d.dat", i)
		if zr.File[i].Name != want {
			t.Fatalf("File(%d) = %q, want %q", i, zr.File[i].Name, want)
		}
	}
}

func TestModTime(t *testing.T) {
	var testTime = time.Date(2009, time.November, 10, 23, 45, 58, 0, time.UTC)
	fh := new(FileHeader)
	fh.SetModTime(testTime)
	outTime := fh.ModTime()
	if !outTime.Equal(testTime) {
		t.Errorf("times don't match: got %s, want %s", outTime, testTime)
	}
}

func testHeaderRoundTrip(fh *FileHeader, wantUncompressedSize uint32, wantUncompressedSize64 uint64, t *testing.T) {
	fi := fh.FileInfo()
	fh2, err := FileInfoHeader(fi)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := fh2.Name, fh.Name; got != want {
		t.Errorf("Name: got %s, want %s\n", got, want)
	}
	if got, want := fh2.UncompressedSize, wantUncompressedSize; got != want {
		t.Errorf("UncompressedSize: got %d, want %d\n", got, want)
	}
	if got, want := fh2.UncompressedSize64, wantUncompressedSize64; got != want {
		t.Errorf("UncompressedSize64: got %d, want %d\n", got, want)
	}
	if got, want := fh2.ModifiedTime, fh.ModifiedTime; got != want {
		t.Errorf("ModifiedTime: got %d, want %d\n", got, want)
	}
	if got, want := fh2.ModifiedDate, fh.ModifiedDate; got != want {
		t.Errorf("ModifiedDate: got %d, want %d\n", got, want)
	}

	if sysfh, ok := fi.Sys().(*FileHeader); !ok && sysfh != fh {
		t.Errorf("Sys didn't return original *FileHeader")
	}
}

func TestFileHeaderRoundTrip(t *testing.T) {
	fh := &FileHeader{
		Name:             "foo.txt",
		UncompressedSize: 987654321,
		ModifiedTime:     1234,
		ModifiedDate:     5678,
	}
	testHeaderRoundTrip(fh, fh.UncompressedSize, uint64(fh.UncompressedSize), t)
}

func TestFileHeaderRoundTrip64(t *testing.T) {
	fh := &FileHeader{
		Name:               "foo.txt",
		UncompressedSize64: 9876543210,
		ModifiedTime:       1234,
		ModifiedDate:       5678,
	}
	testHeaderRoundTrip(fh, uint32max, fh.UncompressedSize64, t)
}

func TestZip64(t *testing.T) {
	if testing.Short() {
		t.Skip("slow test; skipping")
	}
	// write 2^32 bytes plus "END\n" to a zip file
	buf := new(bytes.Buffer)
	w := NewWriter(buf)
	f, err := w.Create("huge.txt")
	if err != nil {
		t.Fatal(err)
	}
	chunk := make([]byte, 1024)
	for i := range chunk {
		chunk[i] = '.'
	}
	chunk[len(chunk)-1] = '\n'
	end := []byte("END\n")
	for i := 0; i < (1<<32)/1024; i++ {
		_, err := f.Write(chunk)
		if err != nil {
			t.Fatal("write chunk:", err)
		}
	}
	_, err = f.Write(end)
	if err != nil {
		t.Fatal("write end:", err)
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}

	// read back zip file and check that we get to the end of it
	r, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
	if err != nil {
		t.Fatal("reader:", err)
	}
	f0 := r.File[0]
	rc, err := f0.Open()
	if err != nil {
		t.Fatal("opening:", err)
	}
	for i := 0; i < (1<<32)/1024; i++ {
		_, err := io.ReadFull(rc, chunk)
		if err != nil {
			t.Fatal("read:", err)
		}
	}
	gotEnd, err := ioutil.ReadAll(rc)
	if err != nil {
		t.Fatal("read end:", err)
	}
	if !bytes.Equal(gotEnd, end) {
		t.Errorf("End of zip64 archive %q, want %q", gotEnd, end)
	}
	err = rc.Close()
	if err != nil {
		t.Fatal("closing:", err)
	}
	if got, want := f0.UncompressedSize, uint32(uint32max); got != want {
		t.Errorf("UncompressedSize %d, want %d", got, want)
	}

	if got, want := f0.UncompressedSize64, (1<<32)+uint64(len(end)); got != want {
		t.Errorf("UncompressedSize64 %d, want %d", got, want)
	}
}

func testInvalidHeader(h *FileHeader, t *testing.T) {
	var buf bytes.Buffer
	z := NewWriter(&buf)

	f, err := z.CreateHeader(h)
	if err != nil {
		t.Fatalf("error creating header: %v", err)
	}
	if _, err := f.Write([]byte("hi")); err != nil {
		t.Fatalf("error writing content: %v", err)
	}
	if err := z.Close(); err != nil {
		t.Fatalf("error closing zip writer: %v", err)
	}

	b := buf.Bytes()
	if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != ErrFormat {
		t.Fatalf("got %v, expected ErrFormat", err)
	}
}

func testValidHeader(h *FileHeader, t *testing.T) {
	var buf bytes.Buffer
	z := NewWriter(&buf)

	f, err := z.CreateHeader(h)
	if err != nil {
		t.Fatalf("error creating header: %v", err)
	}
	if _, err := f.Write([]byte("hi")); err != nil {
		t.Fatalf("error writing content: %v", err)
	}
	if err := z.Close(); err != nil {
		t.Fatalf("error closing zip writer: %v", err)
	}

	b := buf.Bytes()
	if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != nil {
		t.Fatalf("got %v, expected nil", err)
	}
}

// Issue 4302.
func TestHeaderInvalidTagAndSize(t *testing.T) {
	const timeFormat = "20060102T150405.000.txt"

	ts := time.Now()
	filename := ts.Format(timeFormat)

	h := FileHeader{
		Name:   filename,
		Method: Deflate,
		Extra:  []byte(ts.Format(time.RFC3339Nano)), // missing tag and len
	}
	h.SetModTime(ts)

	testInvalidHeader(&h, t)
}

func TestHeaderTooShort(t *testing.T) {
	h := FileHeader{
		Name:   "foo.txt",
		Method: Deflate,
		Extra:  []byte{zip64ExtraId}, // missing size
	}
	testInvalidHeader(&h, t)
}

// Issue 4393. It is valid to have an extra data header
// which contains no body.
func TestZeroLengthHeader(t *testing.T) {
	h := FileHeader{
		Name:   "extadata.txt",
		Method: Deflate,
		Extra: []byte{
			85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5
			85, 120, 0, 0, // tag 30805 size 0
		},
	}
	testValidHeader(&h, t)
}
<commit_msg>archive/zip: speed up Zip64 test<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests that involve both reading and writing.

package zip

import (
	"bytes"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"sort"
	"strings"
	"testing"
	"time"
)

func TestOver65kFiles(t *testing.T) {
	buf := new(bytes.Buffer)
	w := NewWriter(buf)
	const nFiles = (1 << 16) + 42
	for i := 0; i < nFiles; i++ {
		_, err := w.CreateHeader(&FileHeader{
			Name:   fmt.Sprintf("%d.dat", i),
			Method: Store, // avoid Issue 6136 and Issue 6138
		})
		if err != nil {
			t.Fatalf("creating file %d: %v", i, err)
		}
	}
	if err := w.Close(); err != nil {
		t.Fatalf("Writer.Close: %v", err)
	}
	s := buf.String()
	zr, err := NewReader(strings.NewReader(s), int64(len(s)))
	if err != nil {
		t.Fatalf("NewReader: %v", err)
	}
	if got := len(zr.File); got != nFiles {
		t.Fatalf("File contains %d files, want %d", got, nFiles)
	}
	for i := 0; i < nFiles; i++ {
		want := fmt.Sprintf("%d.dat", i)
		if zr.File[i].Name != want {
			t.Fatalf("File(%d) = %q, want %q", i, zr.File[i].Name, want)
		}
	}
}

func TestModTime(t *testing.T) {
	var testTime = time.Date(2009, time.November, 10, 23, 45, 58, 0, time.UTC)
	fh := new(FileHeader)
	fh.SetModTime(testTime)
	outTime := fh.ModTime()
	if !outTime.Equal(testTime) {
		t.Errorf("times don't match: got %s, want %s", outTime, testTime)
	}
}

func testHeaderRoundTrip(fh *FileHeader, wantUncompressedSize uint32, wantUncompressedSize64 uint64, t *testing.T) {
	fi := fh.FileInfo()
	fh2, err := FileInfoHeader(fi)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := fh2.Name, fh.Name; got != want {
		t.Errorf("Name: got %s, want %s\n", got, want)
	}
	if got, want := fh2.UncompressedSize, wantUncompressedSize; got != want {
		t.Errorf("UncompressedSize: got %d, want %d\n", got, want)
	}
	if got, want := fh2.UncompressedSize64, wantUncompressedSize64; got != want {
		t.Errorf("UncompressedSize64: got %d, want %d\n", got, want)
	}
	if got, want := fh2.ModifiedTime, fh.ModifiedTime; got != want {
		t.Errorf("ModifiedTime: got %d, want %d\n", got, want)
	}
	if got, want := fh2.ModifiedDate, fh.ModifiedDate; got != want {
		t.Errorf("ModifiedDate: got %d, want %d\n", got, want)
	}

	if sysfh, ok := fi.Sys().(*FileHeader); !ok && sysfh != fh {
		t.Errorf("Sys didn't return original *FileHeader")
	}
}

func TestFileHeaderRoundTrip(t *testing.T) {
	fh := &FileHeader{
		Name:             "foo.txt",
		UncompressedSize: 987654321,
		ModifiedTime:     1234,
		ModifiedDate:     5678,
	}
	testHeaderRoundTrip(fh, fh.UncompressedSize, uint64(fh.UncompressedSize), t)
}

func TestFileHeaderRoundTrip64(t *testing.T) {
	fh := &FileHeader{
		Name:               "foo.txt",
		UncompressedSize64: 9876543210,
		ModifiedTime:       1234,
		ModifiedDate:       5678,
	}
	testHeaderRoundTrip(fh, uint32max, fh.UncompressedSize64, t)
}

type repeatedByte struct {
	off int64
	b   byte
	n   int64
}

// rleBuffer is a run-length-encoded byte buffer.
// It's an io.Writer (like a bytes.Buffer) and also an io.ReaderAt,
// allowing random-access reads.
type rleBuffer struct {
	buf []repeatedByte
}

func (r *rleBuffer) Size() int64 {
	if len(r.buf) == 0 {
		return 0
	}
	last := &r.buf[len(r.buf)-1]
	return last.off + last.n
}

func (r *rleBuffer) Write(p []byte) (n int, err error) {
	var rp *repeatedByte
	if len(r.buf) > 0 {
		rp = &r.buf[len(r.buf)-1]
		// Fast path, if p is entirely the same byte repeated.
		if lastByte := rp.b; len(p) > 0 && p[0] == lastByte {
			all := true
			for _, b := range p {
				if b != lastByte {
					all = false
					break
				}
			}
			if all {
				rp.n += int64(len(p))
				return len(p), nil
			}
		}
	}

	for _, b := range p {
		if rp == nil || rp.b != b {
			r.buf = append(r.buf, repeatedByte{r.Size(), b, 1})
			rp = &r.buf[len(r.buf)-1]
		} else {
			rp.n++
		}
	}
	return len(p), nil
}

func (r *rleBuffer) ReadAt(p []byte, off int64) (n int, err error) {
	if len(p) == 0 {
		return
	}
	skipParts := sort.Search(len(r.buf), func(i int) bool {
		part := &r.buf[i]
		return part.off+part.n > off
	})
	parts := r.buf[skipParts:]
	if len(parts) > 0 {
		skipBytes := off - parts[0].off
		for len(parts) > 0 {
			part := parts[0]
			for i := skipBytes; i < part.n; i++ {
				if n == len(p) {
					return
				}
				p[n] = part.b
				n++
			}
			parts = parts[1:]
			skipBytes = 0
		}
	}
	if n != len(p) {
		err = io.ErrUnexpectedEOF
	}
	return
}

// Just testing the rleBuffer used in the Zip64 test above. Not used by the zip code.
func TestRLEBuffer(t *testing.T) {
	b := new(rleBuffer)
	var all []byte
	writes := []string{"abcdeee", "eeeeeee", "eeeefghaaiii"}
	for _, w := range writes {
		b.Write([]byte(w))
		all = append(all, w...)
	}
	if len(b.buf) != 10 {
		t.Fatalf("len(b.buf) = %d; want 10", len(b.buf))
	}

	for i := 0; i < len(all); i++ {
		for j := 0; j < len(all)-i; j++ {
			buf := make([]byte, j)
			n, err := b.ReadAt(buf, int64(i))
			if err != nil || n != len(buf) {
				t.Errorf("ReadAt(%d, %d) = %d, %v; want %d, nil", i, j, n, err, len(buf))
			}
			if !bytes.Equal(buf, all[i:i+j]) {
				t.Errorf("ReadAt(%d, %d) = %q; want %q", i, j, buf, all[i:i+j])
			}
		}
	}
}

// fakeHash32 is a dummy Hash32 that always returns 0.
type fakeHash32 struct {
	hash.Hash32
}

func (fakeHash32) Write(p []byte) (int, error) { return len(p), nil }
func (fakeHash32) Sum32() uint32               { return 0 }

func TestZip64(t *testing.T) {
	if testing.Short() {
		t.Skip("slow test; skipping")
	}
	const size = 1 << 32 // before the "END\n" part
	testZip64(t, size)
}

func testZip64(t testing.TB, size int64) {
	const chunkSize = 1024
	chunks := int(size / chunkSize)
	// write 2^32 bytes plus "END\n" to a zip file
	buf := new(rleBuffer)
	w := NewWriter(buf)
	f, err := w.CreateHeader(&FileHeader{
		Name:   "huge.txt",
		Method: Store,
	})
	if err != nil {
		t.Fatal(err)
	}
	f.(*fileWriter).crc32 = fakeHash32{}
	chunk := make([]byte, chunkSize)
	for i := range chunk {
		chunk[i] = '.'
	}
	for i := 0; i < chunks; i++ {
		_, err := f.Write(chunk)
		if err != nil {
			t.Fatal("write chunk:", err)
		}
	}
	end := []byte("END\n")
	_, err = f.Write(end)
	if err != nil {
		t.Fatal("write end:", err)
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}

	// read back zip file and check that we get to the end of it
	r, err := NewReader(buf, int64(buf.Size()))
	if err != nil {
		t.Fatal("reader:", err)
	}
	f0 := r.File[0]
	rc, err := f0.Open()
	if err != nil {
		t.Fatal("opening:", err)
	}
	rc.(*checksumReader).hash = fakeHash32{}
	for i := 0; i < chunks; i++ {
		_, err := io.ReadFull(rc, chunk)
		if err != nil {
			t.Fatal("read:", err)
		}
	}
	gotEnd, err := ioutil.ReadAll(rc)
	if err != nil {
		t.Fatal("read end:", err)
	}
	if !bytes.Equal(gotEnd, end) {
		t.Errorf("End of zip64 archive %q, want %q", gotEnd, end)
	}
	err = rc.Close()
	if err != nil {
		t.Fatal("closing:", err)
	}
	if size == 1<<32 {
		if got, want := f0.UncompressedSize, uint32(uint32max); got != want {
			t.Errorf("UncompressedSize %d, want %d", got, want)
		}
	}

	if got, want := f0.UncompressedSize64, uint64(size)+uint64(len(end)); got != want {
		t.Errorf("UncompressedSize64 %d, want %d", got, want)
	}
}

func testInvalidHeader(h *FileHeader, t *testing.T) {
	var buf bytes.Buffer
	z := NewWriter(&buf)

	f, err := z.CreateHeader(h)
	if err != nil {
		t.Fatalf("error creating header: %v", err)
	}
	if _, err := f.Write([]byte("hi")); err != nil {
		t.Fatalf("error writing content: %v", err)
	}
	if err := z.Close(); err != nil {
		t.Fatalf("error closing zip writer: %v", err)
	}

	b := buf.Bytes()
	if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != ErrFormat {
		t.Fatalf("got %v, expected ErrFormat", err)
	}
}

func testValidHeader(h *FileHeader, t *testing.T) {
	var buf bytes.Buffer
	z := NewWriter(&buf)

	f, err := z.CreateHeader(h)
	if err != nil {
		t.Fatalf("error creating header: %v", err)
	}
	if _, err := f.Write([]byte("hi")); err != nil {
		t.Fatalf("error writing content: %v", err)
	}
	if err := z.Close(); err != nil {
		t.Fatalf("error closing zip writer: %v", err)
	}

	b := buf.Bytes()
	if _, err = NewReader(bytes.NewReader(b), int64(len(b))); err != nil {
		t.Fatalf("got %v, expected nil", err)
	}
}

// Issue 4302.
func TestHeaderInvalidTagAndSize(t *testing.T) {
	const timeFormat = "20060102T150405.000.txt"

	ts := time.Now()
	filename := ts.Format(timeFormat)

	h := FileHeader{
		Name:   filename,
		Method: Deflate,
		Extra:  []byte(ts.Format(time.RFC3339Nano)), // missing tag and len
	}
	h.SetModTime(ts)

	testInvalidHeader(&h, t)
}

func TestHeaderTooShort(t *testing.T) {
	h := FileHeader{
		Name:   "foo.txt",
		Method: Deflate,
		Extra:  []byte{zip64ExtraId}, // missing size
	}
	testInvalidHeader(&h, t)
}

// Issue 4393. It is valid to have an extra data header
// which contains no body.
func TestZeroLengthHeader(t *testing.T) {
	h := FileHeader{
		Name:   "extadata.txt",
		Method: Deflate,
		Extra: []byte{
			85, 84, 5, 0, 3, 154, 144, 195, 77, // tag 21589 size 5
			85, 120, 0, 0, // tag 30805 size 0
		},
	}
	testValidHeader(&h, t)
}

// Just benchmarking how fast the Zip64 test above is. Not related to
// our zip performance, since the test above disabled CRC32 and flate.
func BenchmarkZip64Test(b *testing.B) {
	for i := 0; i < b.N; i++ {
		testZip64(b, 1<<26)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd linux openbsd netbsd

package tls

import (
	"crypto/x509"
	"io/ioutil"
)

// Possible certificate files; stop after finding one.
var certFiles = []string{
	"/etc/ssl/certs/ca-certificates.crt", // Linux etc
	"/etc/pki/tls/certs/ca-bundle.crt",   // Fedora/RHEL
	"/etc/ssl/ca-bundle.pem",             // OpenSUSE
	"/etc/ssl/cert.pem",                  // OpenBSD
}

func initDefaultRoots() {
	roots := x509.NewCertPool()
	for _, file := range certFiles {
		data, err := ioutil.ReadFile(file)
		if err == nil {
			roots.AppendCertsFromPEM(data)
			break
		}
	}
	varDefaultRoots = roots
}
<commit_msg>crypto/tls: add FreeBSD root certificate location     Fixes issue 2721.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build freebsd linux openbsd netbsd

package tls

import (
	"crypto/x509"
	"io/ioutil"
)

// Possible certificate files; stop after finding one.
var certFiles = []string{
	"/etc/ssl/certs/ca-certificates.crt",     // Linux etc
	"/etc/pki/tls/certs/ca-bundle.crt",       // Fedora/RHEL
	"/etc/ssl/ca-bundle.pem",                 // OpenSUSE
	"/etc/ssl/cert.pem",                      // OpenBSD
	"/usr/local/share/certs/ca-root-nss.crt", // FreeBSD
}

func initDefaultRoots() {
	roots := x509.NewCertPool()
	for _, file := range certFiles {
		data, err := ioutil.ReadFile(file)
		if err == nil {
			roots.AppendCertsFromPEM(data)
			break
		}
	}
	varDefaultRoots = roots
}
<|endoftext|>