<commit_before>// Package swarm implements a connection muxer with a pair of channels
// to synchronize all network communication.
package swarm

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"time"

	logging "github.com/ipfs/go-log"
	"github.com/jbenet/goprocess"
	goprocessctx "github.com/jbenet/goprocess/context"
	addrutil "github.com/libp2p/go-addr-util"
	conn "github.com/libp2p/go-libp2p-conn"
	ci "github.com/libp2p/go-libp2p-crypto"
	ipnet "github.com/libp2p/go-libp2p-interface-pnet"
	metrics "github.com/libp2p/go-libp2p-metrics"
	mconn "github.com/libp2p/go-libp2p-metrics/conn"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	transport "github.com/libp2p/go-libp2p-transport"
	filter "github.com/libp2p/go-maddr-filter"
	ps "github.com/libp2p/go-peerstream"
	pst "github.com/libp2p/go-stream-muxer"
	tcpt "github.com/libp2p/go-tcp-transport"
	ws "github.com/libp2p/go-ws-transport"
	ma "github.com/multiformats/go-multiaddr"
	psmss "github.com/whyrusleeping/go-smux-multistream"
	spdy "github.com/whyrusleeping/go-smux-spdystream"
	yamux "github.com/whyrusleeping/go-smux-yamux"
	mafilter "github.com/whyrusleeping/multiaddr-filter"
)

var log = logging.Logger("swarm2")

// PSTransport is the default peerstream transport that will be used by
// any libp2p swarms.
var PSTransport pst.Transport

func init() {
	msstpt := psmss.NewBlankTransport()

	ymxtpt := &yamux.Transport{
		AcceptBacklog:          8192,
		ConnectionWriteTimeout: time.Second * 10,
		KeepAliveInterval:      time.Second * 30,
		EnableKeepAlive:        true,
		MaxStreamWindowSize:    uint32(1024 * 512),
		LogOutput:              ioutil.Discard,
	}

	msstpt.AddTransport("/yamux/1.0.0", ymxtpt)
	msstpt.AddTransport("/spdy/3.1.0", spdy.Transport)

	// allow overriding of muxer preferences
	if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" {
		msstpt.OrderPreference = strings.Fields(prefs)
	}

	PSTransport = msstpt
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
//
// Uses peerstream.Swarm
type Swarm struct {
	swarm *ps.Swarm
	local peer.ID
	peers pstore.Peerstore
	connh ConnHandler

	dsync *DialSync
	backf dialbackoff
	dialT time.Duration // mainly for tests

	dialer *conn.Dialer

	notifmu sync.RWMutex
	notifs  map[inet.Notifiee]ps.Notifiee

	transports []transport.Transport

	// filters for addresses that shouldnt be dialed
	Filters *filter.Filters

	// file descriptor rate limited
	fdRateLimit chan struct{}

	proc goprocess.Process
	ctx  context.Context
	bwc  metrics.Reporter

	limiter *dialLimiter

	protec ipnet.Protector
}

func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID,
	peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) {
	return NewSwarmWithProtector(ctx, listenAddrs, local, peers, nil, PSTransport, bwc)
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID,
	peers pstore.Peerstore, protec ipnet.Protector, tpt pst.Transport, bwc metrics.Reporter) (*Swarm, error) {

	listenAddrs, err := filterAddrs(listenAddrs)
	if err != nil {
		return nil, err
	}

	var wrap func(c transport.Conn) transport.Conn
	if bwc != nil {
		wrap = func(c transport.Conn) transport.Conn {
			return mconn.WrapConn(bwc, c)
		}
	}

	s := &Swarm{
		swarm:  ps.NewSwarm(tpt),
		local:  local,
		peers:  peers,
		ctx:    ctx,
		dialT:  DialTimeout,
		notifs: make(map[inet.Notifiee]ps.Notifiee),
		transports: []transport.Transport{
			tcpt.NewTCPTransport(),
			new(ws.WebsocketTransport),
		},
		bwc:         bwc,
		fdRateLimit: make(chan struct{}, concurrentFdDials),
		Filters:     filter.NewFilters(),
		dialer:      conn.NewDialer(local, peers.PrivKey(local), wrap),
		protec:      protec,
	}
	s.dialer.Protector = protec

	s.dsync = NewDialSync(s.doDial)
	s.limiter = newDialLimiter(s.dialAddr)

	// configure Swarm
	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
	s.SetConnHandler(nil) // make sure to setup our own conn handler.

	err = s.setupInterfaces(listenAddrs)
	if err != nil {
		return nil, err
	}

	return s, nil
}

func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt pst.Transport) *Swarm {
	s := &Swarm{
		swarm:       ps.NewSwarm(pstpt),
		local:       id,
		peers:       pstore.NewPeerstore(),
		ctx:         ctx,
		dialT:       DialTimeout,
		notifs:      make(map[inet.Notifiee]ps.Notifiee),
		fdRateLimit: make(chan struct{}, concurrentFdDials),
		Filters:     filter.NewFilters(),
		dialer:      conn.NewDialer(id, privkey, nil),
	}

	// configure Swarm
	s.limiter = newDialLimiter(s.dialAddr)
	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
	s.SetConnHandler(nil) // make sure to setup our own conn handler.

	return s
}

func (s *Swarm) AddTransport(t transport.Transport) {
	s.transports = append(s.transports, t)
}

func (s *Swarm) teardown() error {
	return s.swarm.Close()
}

// AddAddrFilter adds a multiaddr filter to the set of filters the swarm will
// use to determine which addresses not to dial to.
func (s *Swarm) AddAddrFilter(f string) error {
	m, err := mafilter.NewMask(f)
	if err != nil {
		return err
	}

	s.Filters.AddDialFilter(m)
	return nil
}

func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) {
	if len(listenAddrs) > 0 {
		filtered := addrutil.FilterUsableAddrs(listenAddrs)
		if len(filtered) < 1 {
			return nil, fmt.Errorf("swarm cannot use any addr in: %s", listenAddrs)
		}
		listenAddrs = filtered
	}

	return listenAddrs, nil
}

// Listen sets up listeners for all of the given addresses
func (s *Swarm) Listen(addrs ...ma.Multiaddr) error {
	addrs, err := filterAddrs(addrs)
	if err != nil {
		return err
	}

	return s.setupInterfaces(addrs)
}

// Process returns the Process of the swarm
func (s *Swarm) Process() goprocess.Process {
	return s.proc
}

// Context returns the context of the swarm
func (s *Swarm) Context() context.Context {
	return s.ctx
}

// Close stops the Swarm.
func (s *Swarm) Close() error {
	return s.proc.Close()
}

// StreamSwarm returns the underlying peerstream.Swarm
func (s *Swarm) StreamSwarm() *ps.Swarm {
	return s.swarm
}

// SetConnHandler assigns the handler for new connections.
// See peerstream. You will rarely use this. See SetStreamHandler
func (s *Swarm) SetConnHandler(handler ConnHandler) {

	// handler is nil if user wants to clear the old handler.
	if handler == nil {
		s.swarm.SetConnHandler(func(psconn *ps.Conn) {
			s.connHandler(psconn)
		})
		return
	}

	s.swarm.SetConnHandler(func(psconn *ps.Conn) {
		// sc is nil if closed in our handler.
		if sc := s.connHandler(psconn); sc != nil {
			// call the user's handler. in a goroutine for sync safety.
			go handler(sc)
		}
	})
}

// SetStreamHandler assigns the handler for new streams.
// See peerstream.
func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) {
	s.swarm.SetStreamHandler(func(s *ps.Stream) {
		handler((*Stream)(s))
	})
}

// NewStreamWithPeer creates a new stream on any available connection to p
func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) {
	// if we have no connections, try connecting.
	if len(s.ConnectionsToPeer(p)) == 0 {
		log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...")
		if _, err := s.Dial(ctx, p); err != nil {
			return nil, err
		}
	}
	log.Debug("Swarm: NewStreamWithPeer...")

	// TODO: think about passing a context down to NewStreamWithGroup
	st, err := s.swarm.NewStreamWithGroup(p)
	return (*Stream)(st), err
}

// ConnectionsToPeer returns all the live connections to p
func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn {
	return wrapConns(ps.ConnsWithGroup(p, s.swarm.Conns()))
}

func (s *Swarm) HaveConnsToPeer(p peer.ID) bool {
	for _, c := range s.swarm.Conns() {
		if c.InGroup(p) {
			return true
		}
	}
	return false
}

// Connections returns a slice of all connections.
func (s *Swarm) Connections() []*Conn {
	return wrapConns(s.swarm.Conns())
}

// CloseConnection removes a given peer from swarm + closes the connection
func (s *Swarm) CloseConnection(p peer.ID) error {
	conns := s.swarm.ConnsWithGroup(p) // boom.
	for _, c := range conns {
		c.Close()
	}
	return nil
}

// Peers returns a copy of the set of peers swarm is connected to.
func (s *Swarm) Peers() []peer.ID {
	conns := s.Connections()

	seen := make(map[peer.ID]struct{})
	peers := make([]peer.ID, 0, len(conns))
	for _, c := range conns {
		p := c.RemotePeer()
		if _, found := seen[p]; found {
			continue
		}

		seen[p] = struct{}{}
		peers = append(peers, p)
	}
	return peers
}

// LocalPeer returns the local peer swarm is associated to.
func (s *Swarm) LocalPeer() peer.ID {
	return s.local
}

// Backoff returns the dialbackoff object for this swarm.
func (s *Swarm) Backoff() *dialbackoff {
	return &s.backf
}

// notifyAll sends a signal to all Notifiees
func (s *Swarm) notifyAll(notify func(inet.Notifiee)) {
	s.notifmu.RLock()
	for f := range s.notifs {
		go notify(f)
	}
	s.notifmu.RUnlock()
}

// Notify signs up Notifiee to receive signals when events happen
func (s *Swarm) Notify(f inet.Notifiee) {
	// wrap with our notifiee, to translate function calls
	n := &ps2netNotifee{net: (*Network)(s), not: f}

	s.notifmu.Lock()
	s.notifs[f] = n
	s.notifmu.Unlock()

	// register for notifications in the peer swarm.
	s.swarm.Notify(n)
}

// StopNotify unregisters Notifiee fromr receiving signals
func (s *Swarm) StopNotify(f inet.Notifiee) {
	s.notifmu.Lock()
	n, found := s.notifs[f]
	if found {
		delete(s.notifs, f)
	}
	s.notifmu.Unlock()

	if found {
		s.swarm.StopNotify(n)
	}
}

type ps2netNotifee struct {
	net *Network
	not inet.Notifiee
}

func (n *ps2netNotifee) Connected(c *ps.Conn) {
	n.not.Connected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) Disconnected(c *ps.Conn) {
	n.not.Disconnected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) OpenedStream(s *ps.Stream) {
	n.not.OpenedStream(n.net, (*Stream)(s))
}

func (n *ps2netNotifee) ClosedStream(s *ps.Stream) {
	n.not.ClosedStream(n.net, (*Stream)(s))
}
<commit_msg>use optimized 'HaveConnToPeer' checks<commit_after>// Package swarm implements a connection muxer with a pair of channels
// to synchronize all network communication.
package swarm

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"
	"time"

	logging "github.com/ipfs/go-log"
	"github.com/jbenet/goprocess"
	goprocessctx "github.com/jbenet/goprocess/context"
	addrutil "github.com/libp2p/go-addr-util"
	conn "github.com/libp2p/go-libp2p-conn"
	ci "github.com/libp2p/go-libp2p-crypto"
	ipnet "github.com/libp2p/go-libp2p-interface-pnet"
	metrics "github.com/libp2p/go-libp2p-metrics"
	mconn "github.com/libp2p/go-libp2p-metrics/conn"
	inet "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	transport "github.com/libp2p/go-libp2p-transport"
	filter "github.com/libp2p/go-maddr-filter"
	ps "github.com/libp2p/go-peerstream"
	pst "github.com/libp2p/go-stream-muxer"
	tcpt "github.com/libp2p/go-tcp-transport"
	ws "github.com/libp2p/go-ws-transport"
	ma "github.com/multiformats/go-multiaddr"
	psmss "github.com/whyrusleeping/go-smux-multistream"
	spdy "github.com/whyrusleeping/go-smux-spdystream"
	yamux "github.com/whyrusleeping/go-smux-yamux"
	mafilter "github.com/whyrusleeping/multiaddr-filter"
)

var log = logging.Logger("swarm2")

// PSTransport is the default peerstream transport that will be used by
// any libp2p swarms.
var PSTransport pst.Transport

func init() {
	msstpt := psmss.NewBlankTransport()

	ymxtpt := &yamux.Transport{
		AcceptBacklog:          8192,
		ConnectionWriteTimeout: time.Second * 10,
		KeepAliveInterval:      time.Second * 30,
		EnableKeepAlive:        true,
		MaxStreamWindowSize:    uint32(1024 * 512),
		LogOutput:              ioutil.Discard,
	}

	msstpt.AddTransport("/yamux/1.0.0", ymxtpt)
	msstpt.AddTransport("/spdy/3.1.0", spdy.Transport)

	// allow overriding of muxer preferences
	if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" {
		msstpt.OrderPreference = strings.Fields(prefs)
	}

	PSTransport = msstpt
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
//
// Uses peerstream.Swarm
type Swarm struct {
	swarm *ps.Swarm
	local peer.ID
	peers pstore.Peerstore
	connh ConnHandler

	dsync *DialSync
	backf dialbackoff
	dialT time.Duration // mainly for tests

	dialer *conn.Dialer

	notifmu sync.RWMutex
	notifs  map[inet.Notifiee]ps.Notifiee

	transports []transport.Transport

	// filters for addresses that shouldnt be dialed
	Filters *filter.Filters

	// file descriptor rate limited
	fdRateLimit chan struct{}

	proc goprocess.Process
	ctx  context.Context
	bwc  metrics.Reporter

	limiter *dialLimiter

	protec ipnet.Protector
}

func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID,
	peers pstore.Peerstore, bwc metrics.Reporter) (*Swarm, error) {
	return NewSwarmWithProtector(ctx, listenAddrs, local, peers, nil, PSTransport, bwc)
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarmWithProtector(ctx context.Context, listenAddrs []ma.Multiaddr, local peer.ID,
	peers pstore.Peerstore, protec ipnet.Protector, tpt pst.Transport, bwc metrics.Reporter) (*Swarm, error) {

	listenAddrs, err := filterAddrs(listenAddrs)
	if err != nil {
		return nil, err
	}

	var wrap func(c transport.Conn) transport.Conn
	if bwc != nil {
		wrap = func(c transport.Conn) transport.Conn {
			return mconn.WrapConn(bwc, c)
		}
	}

	s := &Swarm{
		swarm:  ps.NewSwarm(tpt),
		local:  local,
		peers:  peers,
		ctx:    ctx,
		dialT:  DialTimeout,
		notifs: make(map[inet.Notifiee]ps.Notifiee),
		transports: []transport.Transport{
			tcpt.NewTCPTransport(),
			new(ws.WebsocketTransport),
		},
		bwc:         bwc,
		fdRateLimit: make(chan struct{}, concurrentFdDials),
		Filters:     filter.NewFilters(),
		dialer:      conn.NewDialer(local, peers.PrivKey(local), wrap),
		protec:      protec,
	}
	s.dialer.Protector = protec

	s.dsync = NewDialSync(s.doDial)
	s.limiter = newDialLimiter(s.dialAddr)

	// configure Swarm
	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
	s.SetConnHandler(nil) // make sure to setup our own conn handler.

	err = s.setupInterfaces(listenAddrs)
	if err != nil {
		return nil, err
	}

	return s, nil
}

func NewBlankSwarm(ctx context.Context, id peer.ID, privkey ci.PrivKey, pstpt pst.Transport) *Swarm {
	s := &Swarm{
		swarm:       ps.NewSwarm(pstpt),
		local:       id,
		peers:       pstore.NewPeerstore(),
		ctx:         ctx,
		dialT:       DialTimeout,
		notifs:      make(map[inet.Notifiee]ps.Notifiee),
		fdRateLimit: make(chan struct{}, concurrentFdDials),
		Filters:     filter.NewFilters(),
		dialer:      conn.NewDialer(id, privkey, nil),
	}

	// configure Swarm
	s.limiter = newDialLimiter(s.dialAddr)
	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
	s.SetConnHandler(nil) // make sure to setup our own conn handler.

	return s
}

func (s *Swarm) AddTransport(t transport.Transport) {
	s.transports = append(s.transports, t)
}

func (s *Swarm) teardown() error {
	return s.swarm.Close()
}

// AddAddrFilter adds a multiaddr filter to the set of filters the swarm will
// use to determine which addresses not to dial to.
func (s *Swarm) AddAddrFilter(f string) error {
	m, err := mafilter.NewMask(f)
	if err != nil {
		return err
	}

	s.Filters.AddDialFilter(m)
	return nil
}

func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) {
	if len(listenAddrs) > 0 {
		filtered := addrutil.FilterUsableAddrs(listenAddrs)
		if len(filtered) < 1 {
			return nil, fmt.Errorf("swarm cannot use any addr in: %s", listenAddrs)
		}
		listenAddrs = filtered
	}

	return listenAddrs, nil
}

// Listen sets up listeners for all of the given addresses
func (s *Swarm) Listen(addrs ...ma.Multiaddr) error {
	addrs, err := filterAddrs(addrs)
	if err != nil {
		return err
	}

	return s.setupInterfaces(addrs)
}

// Process returns the Process of the swarm
func (s *Swarm) Process() goprocess.Process {
	return s.proc
}

// Context returns the context of the swarm
func (s *Swarm) Context() context.Context {
	return s.ctx
}

// Close stops the Swarm.
func (s *Swarm) Close() error {
	return s.proc.Close()
}

// StreamSwarm returns the underlying peerstream.Swarm
func (s *Swarm) StreamSwarm() *ps.Swarm {
	return s.swarm
}

// SetConnHandler assigns the handler for new connections.
// See peerstream. You will rarely use this. See SetStreamHandler
func (s *Swarm) SetConnHandler(handler ConnHandler) {

	// handler is nil if user wants to clear the old handler.
	if handler == nil {
		s.swarm.SetConnHandler(func(psconn *ps.Conn) {
			s.connHandler(psconn)
		})
		return
	}

	s.swarm.SetConnHandler(func(psconn *ps.Conn) {
		// sc is nil if closed in our handler.
		if sc := s.connHandler(psconn); sc != nil {
			// call the user's handler. in a goroutine for sync safety.
			go handler(sc)
		}
	})
}

// SetStreamHandler assigns the handler for new streams.
// See peerstream.
func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) {
	s.swarm.SetStreamHandler(func(s *ps.Stream) {
		handler((*Stream)(s))
	})
}

// NewStreamWithPeer creates a new stream on any available connection to p
func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) {
	// if we have no connections, try connecting.
	if !s.HaveConnsToPeer(p) {
		log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...")
		if _, err := s.Dial(ctx, p); err != nil {
			return nil, err
		}
	}
	log.Debug("Swarm: NewStreamWithPeer...")

	// TODO: think about passing a context down to NewStreamWithGroup
	st, err := s.swarm.NewStreamWithGroup(p)
	return (*Stream)(st), err
}

// ConnectionsToPeer returns all the live connections to p
func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn {
	return wrapConns(s.swarm.ConnsWithGroup(p))
}

func (s *Swarm) HaveConnsToPeer(p peer.ID) bool {
	return len(s.swarm.ConnsWithGroup(p)) > 0
}

// Connections returns a slice of all connections.
func (s *Swarm) Connections() []*Conn {
	return wrapConns(s.swarm.Conns())
}

// CloseConnection removes a given peer from swarm + closes the connection
func (s *Swarm) CloseConnection(p peer.ID) error {
	conns := s.swarm.ConnsWithGroup(p) // boom.
	for _, c := range conns {
		c.Close()
	}
	return nil
}

// Peers returns a copy of the set of peers swarm is connected to.
func (s *Swarm) Peers() []peer.ID {
	conns := s.Connections()

	seen := make(map[peer.ID]struct{})
	peers := make([]peer.ID, 0, len(conns))
	for _, c := range conns {
		p := c.RemotePeer()
		if _, found := seen[p]; found {
			continue
		}

		seen[p] = struct{}{}
		peers = append(peers, p)
	}
	return peers
}

// LocalPeer returns the local peer swarm is associated to.
func (s *Swarm) LocalPeer() peer.ID {
	return s.local
}

// Backoff returns the dialbackoff object for this swarm.
func (s *Swarm) Backoff() *dialbackoff {
	return &s.backf
}

// notifyAll sends a signal to all Notifiees
func (s *Swarm) notifyAll(notify func(inet.Notifiee)) {
	s.notifmu.RLock()
	for f := range s.notifs {
		go notify(f)
	}
	s.notifmu.RUnlock()
}

// Notify signs up Notifiee to receive signals when events happen
func (s *Swarm) Notify(f inet.Notifiee) {
	// wrap with our notifiee, to translate function calls
	n := &ps2netNotifee{net: (*Network)(s), not: f}

	s.notifmu.Lock()
	s.notifs[f] = n
	s.notifmu.Unlock()

	// register for notifications in the peer swarm.
	s.swarm.Notify(n)
}

// StopNotify unregisters Notifiee fromr receiving signals
func (s *Swarm) StopNotify(f inet.Notifiee) {
	s.notifmu.Lock()
	n, found := s.notifs[f]
	if found {
		delete(s.notifs, f)
	}
	s.notifmu.Unlock()

	if found {
		s.swarm.StopNotify(n)
	}
}

type ps2netNotifee struct {
	net *Network
	not inet.Notifiee
}

func (n *ps2netNotifee) Connected(c *ps.Conn) {
	n.not.Connected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) Disconnected(c *ps.Conn) {
	n.not.Disconnected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) OpenedStream(s *ps.Stream) {
	n.not.OpenedStream(n.net, (*Stream)(s))
}

func (n *ps2netNotifee) ClosedStream(s *ps.Stream) {
	n.not.ClosedStream(n.net, (*Stream)(s))
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bttest

import (
	"fmt"
	"math/rand"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/context"
	btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
	btpb "google.golang.org/genproto/googleapis/bigtable/v2"
	"google.golang.org/grpc"
	"strconv"
)

func TestConcurrentMutationsReadModifyAndGC(t *testing.T) {
	s := &server{
		tables: make(map[string]*table),
	}
	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	defer cancel()
	if _, err := s.CreateTable(
		ctx,
		&btapb.CreateTableRequest{Parent: "cluster", TableId: "t"}); err != nil {
		t.Fatal(err)
	}
	const name = `cluster/tables/t`
	tbl := s.tables[name]
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: name,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  "cf",
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
		}},
	}
	_, err := s.ModifyColumnFamilies(ctx, req)
	if err != nil {
		t.Fatal(err)
	}
	req = &btapb.ModifyColumnFamiliesRequest{
		Name: name,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id: "cf",
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Update{&btapb.ColumnFamily{
				GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}},
			}},
		}},
	}
	if _, err := s.ModifyColumnFamilies(ctx, req); err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	var ts int64
	ms := func() []*btpb.Mutation {
		return []*btpb.Mutation{{
			Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
				FamilyName:      "cf",
				ColumnQualifier: []byte(`col`),
				TimestampMicros: atomic.AddInt64(&ts, 1000),
			}},
		}}
	}

	rmw := func() *btpb.ReadModifyWriteRowRequest {
		return &btpb.ReadModifyWriteRowRequest{
			TableName: name,
			RowKey:    []byte(fmt.Sprint(rand.Intn(100))),
			Rules: []*btpb.ReadModifyWriteRule{{
				FamilyName:      "cf",
				ColumnQualifier: []byte("col"),
				Rule:            &btpb.ReadModifyWriteRule_IncrementAmount{1},
			}},
		}
	}
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for ctx.Err() == nil {
				req := &btpb.MutateRowRequest{
					TableName: name,
					RowKey:    []byte(fmt.Sprint(rand.Intn(100))),
					Mutations: ms(),
				}
				s.MutateRow(ctx, req)
			}
		}()
		wg.Add(1)
		go func() {
			defer wg.Done()
			for ctx.Err() == nil {
				_, _ = s.ReadModifyWriteRow(ctx, rmw())
			}
		}()

		wg.Add(1)
		go func() {
			defer wg.Done()
			tbl.gc()
		}()
	}
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-time.After(100 * time.Millisecond):
		t.Error("Concurrent mutations and GCs haven't completed after 100ms")
	}
}

func TestCreateTableWithFamily(t *testing.T) {
	// The Go client currently doesn't support creating a table with column families
	// in one operation but it is allowed by the API. This must still be supported by the
	// fake server so this test lives here instead of in the main bigtable
	// integration test.
	s := &server{
		tables: make(map[string]*table),
	}
	ctx := context.Background()
	newTbl := btapb.Table{
		ColumnFamilies: map[string]*btapb.ColumnFamily{
			"cf1": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{123}}},
			"cf2": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{456}}},
		},
	}
	cTbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
	if err != nil {
		t.Fatalf("Creating table: %v", err)
	}
	tbl, err := s.GetTable(ctx, &btapb.GetTableRequest{Name: cTbl.Name})
	if err != nil {
		t.Fatalf("Getting table: %v", err)
	}
	cf := tbl.ColumnFamilies["cf1"]
	if cf == nil {
		t.Fatalf("Missing col family cf1")
	}
	if got, want := cf.GcRule.GetMaxNumVersions(), int32(123); got != want {
		t.Errorf("Invalid MaxNumVersions: wanted:%d, got:%d", want, got)
	}
	cf = tbl.ColumnFamilies["cf2"]
	if cf == nil {
		t.Fatalf("Missing col family cf2")
	}
	if got, want := cf.GcRule.GetMaxNumVersions(), int32(456); got != want {
		t.Errorf("Invalid MaxNumVersions: wanted:%d, got:%d", want, got)
	}
}

type MockSampleRowKeysServer struct {
	responses []*btpb.SampleRowKeysResponse
	grpc.ServerStream
}

func (s *MockSampleRowKeysServer) Send(resp *btpb.SampleRowKeysResponse) error {
	s.responses = append(s.responses, resp)
	return nil
}

func TestSampleRowKeys(t *testing.T) {
	s := &server{
		tables: make(map[string]*table),
	}
	ctx := context.Background()
	newTbl := btapb.Table{
		ColumnFamilies: map[string]*btapb.ColumnFamily{
			"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}}},
		},
	}
	tbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
	if err != nil {
		t.Fatalf("Creating table: %v", err)
	}

	// Populate the table
	val := []byte("value")
	rowCount := 1000
	for i := 0; i < rowCount; i++ {
		req := &btpb.MutateRowRequest{
			TableName: tbl.Name,
			RowKey:    []byte("row-" + strconv.Itoa(i)),
			Mutations: []*btpb.Mutation{{
				Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
					FamilyName:      "cf",
					ColumnQualifier: []byte("col"),
					TimestampMicros: 0,
					Value:           val,
				}},
			}},
		}
		if _, err := s.MutateRow(ctx, req); err != nil {
			t.Fatalf("Populating table: %v", err)
		}
	}

	mock := &MockSampleRowKeysServer{}
	if err := s.SampleRowKeys(&btpb.SampleRowKeysRequest{TableName: tbl.Name}, mock); err != nil {
		t.Errorf("SampleRowKeys error: %v", err)
	}
	if len(mock.responses) == 0 {
		t.Fatal("Response count: got 0, want > 0")
	}
	// Make sure the offset of the final response is the offset of the final row
	got := mock.responses[len(mock.responses)-1].OffsetBytes
	want := int64((rowCount - 1) * len(val))
	if got != want {
		t.Errorf("Invalid offset: got %d, want %d", got, want)
	}
}

func TestDropRowRange(t *testing.T) {
	s := &server{
		tables: make(map[string]*table),
	}
	ctx := context.Background()
	newTbl := btapb.Table{
		ColumnFamilies: map[string]*btapb.ColumnFamily{
			"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}}},
		},
	}
	tblInfo, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
	if err != nil {
		t.Fatalf("Creating table: %v", err)
	}

	tbl := s.tables[tblInfo.Name]

	// Populate the table
	prefixes := []string{"AAA", "BBB", "CCC", "DDD"}
	count := 3

	for _, prefix := range prefixes {
		for i := 0; i < count; i++ {
			req := &btpb.MutateRowRequest{
				TableName: tblInfo.Name,
				RowKey:    []byte(prefix + strconv.Itoa(i)),
				Mutations: []*btpb.Mutation{{
					Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
						FamilyName:      "cf",
						ColumnQualifier: []byte("col"),
						TimestampMicros: 0,
						Value:           []byte{},
					}},
				}},
			}
			if _, err := s.MutateRow(ctx, req); err != nil {
				t.Fatalf("Populating table: %v", err)
			}
		}
	}

	tblSize := len(tbl.rows)
	req := &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte("AAA")},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping first range: %v", err)
	}
	got, want := len(tbl.rows), tblSize - count
	if got != want {
		t.Errorf("Row count after first drop: got %d (%v), want %d", got, tbl.rows, want)
	}

	req = &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte("DDD")},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping second range: %v", err)
	}
	got, want = len(tbl.rows), tblSize - (2 * count)
	if got != want  {
		t.Errorf("Row count after second drop: got %d (%v), want %d", got, tbl.rows, want)
	}

	req = &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte("XXX")},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping invalid range: %v", err)
	}
	got, want = len(tbl.rows), tblSize - (2 * count)
	if got != want  {
		t.Errorf("Row count after invalid drop: got %d (%v), want %d", got, tbl.rows, want)
	}

	req = &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_DeleteAllDataFromTable{true},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping all data: %v", err)
	}
	got, want = len(tbl.rows), 0
	if got != want  {
		t.Errorf("Row count after drop all: got %d, want %d", got, want)
	}
}<commit_msg>bigtable/bttest: Allow more time for concurrent mod test.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bttest

import (
	"fmt"
	"math/rand"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/context"
	btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
	btpb "google.golang.org/genproto/googleapis/bigtable/v2"
	"google.golang.org/grpc"
	"strconv"
)

func TestConcurrentMutationsReadModifyAndGC(t *testing.T) {
	s := &server{
		tables: make(map[string]*table),
	}
	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	defer cancel()
	if _, err := s.CreateTable(
		ctx,
		&btapb.CreateTableRequest{Parent: "cluster", TableId: "t"}); err != nil {
		t.Fatal(err)
	}
	const name = `cluster/tables/t`
	tbl := s.tables[name]
	req := &btapb.ModifyColumnFamiliesRequest{
		Name: name,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id:  "cf",
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{&btapb.ColumnFamily{}},
		}},
	}
	_, err := s.ModifyColumnFamilies(ctx, req)
	if err != nil {
		t.Fatal(err)
	}
	req = &btapb.ModifyColumnFamiliesRequest{
		Name: name,
		Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
			Id: "cf",
			Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Update{&btapb.ColumnFamily{
				GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}},
			}},
		}},
	}
	if _, err := s.ModifyColumnFamilies(ctx, req); err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	var ts int64
	ms := func() []*btpb.Mutation {
		return []*btpb.Mutation{{
			Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
				FamilyName:      "cf",
				ColumnQualifier: []byte(`col`),
				TimestampMicros: atomic.AddInt64(&ts, 1000),
			}},
		}}
	}

	rmw := func() *btpb.ReadModifyWriteRowRequest {
		return &btpb.ReadModifyWriteRowRequest{
			TableName: name,
			RowKey:    []byte(fmt.Sprint(rand.Intn(100))),
			Rules: []*btpb.ReadModifyWriteRule{{
				FamilyName:      "cf",
				ColumnQualifier: []byte("col"),
				Rule:            &btpb.ReadModifyWriteRule_IncrementAmount{1},
			}},
		}
	}
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for ctx.Err() == nil {
				req := &btpb.MutateRowRequest{
					TableName: name,
					RowKey:    []byte(fmt.Sprint(rand.Intn(100))),
					Mutations: ms(),
				}
				s.MutateRow(ctx, req)
			}
		}()
		wg.Add(1)
		go func() {
			defer wg.Done()
			for ctx.Err() == nil {
				_, _ = s.ReadModifyWriteRow(ctx, rmw())
			}
		}()

		wg.Add(1)
		go func() {
			defer wg.Done()
			tbl.gc()
		}()
	}
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-time.After(1 * time.Second):
		t.Error("Concurrent mutations and GCs haven't completed after 1s")
	}
}

func TestCreateTableWithFamily(t *testing.T) {
	// The Go client currently doesn't support creating a table with column families
	// in one operation but it is allowed by the API. This must still be supported by the
	// fake server so this test lives here instead of in the main bigtable
	// integration test.
	s := &server{
		tables: make(map[string]*table),
	}
	ctx := context.Background()
	newTbl := btapb.Table{
		ColumnFamilies: map[string]*btapb.ColumnFamily{
			"cf1": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{123}}},
			"cf2": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{456}}},
		},
	}
	cTbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
	if err != nil {
		t.Fatalf("Creating table: %v", err)
	}
	tbl, err := s.GetTable(ctx, &btapb.GetTableRequest{Name: cTbl.Name})
	if err != nil {
		t.Fatalf("Getting table: %v", err)
	}
	cf := tbl.ColumnFamilies["cf1"]
	if cf == nil {
		t.Fatalf("Missing col family cf1")
	}
	if got, want := cf.GcRule.GetMaxNumVersions(), int32(123); got != want {
		t.Errorf("Invalid MaxNumVersions: wanted:%d, got:%d", want, got)
	}
	cf = tbl.ColumnFamilies["cf2"]
	if cf == nil {
		t.Fatalf("Missing col family cf2")
	}
	if got, want := cf.GcRule.GetMaxNumVersions(), int32(456); got != want {
		t.Errorf("Invalid MaxNumVersions: wanted:%d, got:%d", want, got)
	}
}

type MockSampleRowKeysServer struct {
	responses []*btpb.SampleRowKeysResponse
	grpc.ServerStream
}

func (s *MockSampleRowKeysServer) Send(resp *btpb.SampleRowKeysResponse) error {
	s.responses = append(s.responses, resp)
	return nil
}

func TestSampleRowKeys(t *testing.T) {
	s := &server{
		tables: make(map[string]*table),
	}
	ctx := context.Background()
	newTbl := btapb.Table{
		ColumnFamilies: map[string]*btapb.ColumnFamily{
			"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}}},
		},
	}
	tbl, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
	if err != nil {
		t.Fatalf("Creating table: %v", err)
	}

	// Populate the table
	val := []byte("value")
	rowCount := 1000
	for i := 0; i < rowCount; i++ {
		req := &btpb.MutateRowRequest{
			TableName: tbl.Name,
			RowKey:    []byte("row-" + strconv.Itoa(i)),
			Mutations: []*btpb.Mutation{{
				Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
					FamilyName:      "cf",
					ColumnQualifier: []byte("col"),
					TimestampMicros: 0,
					Value:           val,
				}},
			}},
		}
		if _, err := s.MutateRow(ctx, req); err != nil {
			t.Fatalf("Populating table: %v", err)
		}
	}

	mock := &MockSampleRowKeysServer{}
	if err := s.SampleRowKeys(&btpb.SampleRowKeysRequest{TableName: tbl.Name}, mock); err != nil {
		t.Errorf("SampleRowKeys error: %v", err)
	}
	if len(mock.responses) == 0 {
		t.Fatal("Response count: got 0, want > 0")
	}
	// Make sure the offset of the final response is the offset of the final row
	got := mock.responses[len(mock.responses)-1].OffsetBytes
	want := int64((rowCount - 1) * len(val))
	if got != want {
		t.Errorf("Invalid offset: got %d, want %d", got, want)
	}
}

func TestDropRowRange(t *testing.T) {
	s := &server{
		tables: make(map[string]*table),
	}
	ctx := context.Background()
	newTbl := btapb.Table{
		ColumnFamilies: map[string]*btapb.ColumnFamily{
			"cf": {GcRule: &btapb.GcRule{Rule: &btapb.GcRule_MaxNumVersions{1}}},
		},
	}
	tblInfo, err := s.CreateTable(ctx, &btapb.CreateTableRequest{Parent: "cluster", TableId: "t", Table: &newTbl})
	if err != nil {
		t.Fatalf("Creating table: %v", err)
	}

	tbl := s.tables[tblInfo.Name]

	// Populate the table
	prefixes := []string{"AAA", "BBB", "CCC", "DDD"}
	count := 3

	for _, prefix := range prefixes {
		for i := 0; i < count; i++ {
			req := &btpb.MutateRowRequest{
				TableName: tblInfo.Name,
				RowKey:    []byte(prefix + strconv.Itoa(i)),
				Mutations: []*btpb.Mutation{{
					Mutation: &btpb.Mutation_SetCell_{&btpb.Mutation_SetCell{
						FamilyName:      "cf",
						ColumnQualifier: []byte("col"),
						TimestampMicros: 0,
						Value:           []byte{},
					}},
				}},
			}
			if _, err := s.MutateRow(ctx, req); err != nil {
				t.Fatalf("Populating table: %v", err)
			}
		}
	}

	tblSize := len(tbl.rows)
	req := &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte("AAA")},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping first range: %v", err)
	}
	got, want := len(tbl.rows), tblSize - count
	if got != want {
		t.Errorf("Row count after first drop: got %d (%v), want %d", got, tbl.rows, want)
	}

	req = &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte("DDD")},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping second range: %v", err)
	}
	got, want = len(tbl.rows), tblSize - (2 * count)
	if got != want  {
		t.Errorf("Row count after second drop: got %d (%v), want %d", got, tbl.rows, want)
	}

	req = &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_RowKeyPrefix{[]byte("XXX")},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping invalid range: %v", err)
	}
	got, want = len(tbl.rows), tblSize - (2 * count)
	if got != want  {
		t.Errorf("Row count after invalid drop: got %d (%v), want %d", got, tbl.rows, want)
	}

	req = &btapb.DropRowRangeRequest{
		Name:tblInfo.Name,
		Target: &btapb.DropRowRangeRequest_DeleteAllDataFromTable{true},
	}
	if _, err = s.DropRowRange(ctx, req); err != nil {
		t.Fatalf("Dropping all data: %v", err)
	}
	got, want = len(tbl.rows), 0
	if got != want  {
		t.Errorf("Row count after drop all: got %d, want %d", got, want)
	}
}<|endoftext|>


<commit_before>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package uniqush

import (
	"crypto/tls"
	"net"
	"json"
	"os"
    "io"
	"bytes"
	"encoding/hex"
	"encoding/binary"
    "strconv"
)

type APNSPushService struct {
}

func init() {
    GetPushServiceManager().RegisterPushServiceType(NewAPNSPushService())
}

func NewAPNSPushService() *APNSPushService {
	return new(APNSPushService)
}

func (p *APNSPushService) Name() string {
    return "apns"
}

func (p *APNSPushService) BuildPushServiceProviderFromMap(kv map[string]string) (*PushServiceProvider, os.Error) {
    psp := NewEmptyPushServiceProvider()
    if service, ok := kv["service"]; ok {
        psp.FixedData["service"] = service
    } else {
        return nil, os.NewError("NoService")
    }

    if cert, ok := kv["cert"]; ok {
        psp.FixedData["cert"] = cert
    } else {
        return nil, os.NewError("NoCertificate")
    }

    if key, ok := kv["key"]; ok {
        psp.FixedData["key"] = key
    } else {
        return nil, os.NewError("NoPrivateKey")
    }

    if sandbox, ok := kv["sandbox"]; ok {
        if sandbox == "true" {
            psp.VolatileData["addr"] = "gateway.sandbox.push.apple.com:2195"
            return psp, nil
        }
    }
    psp.VolatileData["addr"] = "gateway.push.apple.com:2195"
    return psp, nil
}

func (p *APNSPushService) BuildDeliveryPointFromMap(kv map[string]string) (*DeliveryPoint, os.Error) {
    dp := NewEmptyDeliveryPoint()

    if service, ok := kv["service"]; ok {
        dp.FixedData["service"] = service
    } else {
        return nil, os.NewError("NoService")
    }
    if sub, ok := kv["subscriber"]; ok {
        dp.FixedData["subscriber"] = sub
    } else {
        return nil, os.NewError("NoSubscriber")
    }
    if devtoken, ok := kv["devtoken"]; ok {
        dp.FixedData["devtoken"] = devtoken
    } else {
        return nil, os.NewError("NoDevToken")
    }
    return dp, nil
}

/*
type APNSService struct {
    psp *PushServiceProvider
}

func (s *APNSService) Certificate() string {
    return s.psp.auth_token
}

func (s *APNSService) PrivateKey() string {
    return s.psp.sender_id
}

func (s *APNSService) Address() string {
    if s.psp.real_auth_token == "0" {
        return "gateway.sandbox.push.apple.com:2195"
    }
    return "gateway.push.apple.com:2195"
}

func NewAPNSService(psp *PushServiceProvider) *APNSService {
    ret := new(APNSService)
    ret.psp = psp
    return ret
}
*/

func toAPNSPayload(n *Notification) []byte {
    payload := make(map[string]interface{})
    aps := make(map[string]interface{})
    alert := make(map[string]interface{})
    for k, v := range n.Data {
        switch (k) {
        case "msg":
            alert["body"] = v
        case "badge":
            b, err := strconv.Atoi(v)
            if err != nil {
                continue
            } else {
                aps["badge"] = b
            }
        case "sound":
            aps["sound"] = v
        case "img":
            alert["launch-image"] = v
        default:
            payload[k] = v
        }
    }
    aps["alert"] = alert
    payload["aps"] = aps
    j, err := json.Marshal(payload)
    if err != nil {
        return nil
    }
    return j
}

func writen (w io.Writer, buf []byte) os.Error {
    n := len(buf)
    for n >= 0 {
        l, err := w.Write(buf)
        if err != nil {
            return err
        }
        if l >= n {
            return nil
        }
        n -= l
        buf = buf[l:]
    }
    return nil
}

func (p *APNSPushService) Push(sp *PushServiceProvider,
                        s *DeliveryPoint,
                        n *Notification) (string, os.Error) {
    cert, err := tls.LoadX509KeyPair(sp.FixedData["cert"], sp.FixedData["key"])
    if err != nil {
		return "", NewInvalidPushServiceProviderError(sp)
    }
    conf := &tls.Config {
              Certificates: []tls.Certificate{cert},
          }
    conn, err := net.Dial("tcp", sp.VolatileData["addr"])
    if err != nil {
		return "", NewInvalidPushServiceProviderError(sp)
    }

	tlsconn := tls.Client(conn, conf)
	err = tlsconn.Handshake()
    if err != nil {
		return "", NewInvalidPushServiceProviderError(sp)
    }
    devtoken := s.FixedData["devtoken"]
	btoken, err := hex.DecodeString(devtoken)
    if err != nil {
        return "", NewInvalidDeliveryPointError(sp, s)
    }

    bpayload := toAPNSPayload(n)
    if bpayload == nil {
        /* FIXME new error type */
        return "", NewInvalidDeliveryPointError(sp, s)
    }
	buffer := bytes.NewBuffer([]byte{})
	// command
	binary.Write(buffer, binary.BigEndian, uint8(0))

	// push device token
	binary.Write(buffer, binary.BigEndian, uint16(len(btoken)))
	binary.Write(buffer, binary.BigEndian, btoken)

	// push payload
	binary.Write(buffer, binary.BigEndian, uint16(len(bpayload)))
	binary.Write(buffer, binary.BigEndian, bpayload)
	pdu := buffer.Bytes()

	// write pdu
	err = writen(tlsconn, pdu)
    if err != nil {
		return "", NewInvalidPushServiceProviderError(sp)
    }
	tlsconn.SetReadTimeout(5E8)
	readb := [6]byte{}
	nr, err := tlsconn.Read(readb[:])
    /* TODO error handling */
    if nr > 0 {
        switch(readb[1]) {
        case 2:
            return "", NewInvalidDeliveryPointError(sp, s)
        default:
            return "", NewInvalidPushServiceProviderError(sp)
        }
    }
    return "", nil
}

<commit_msg>APNs: support enhanced format<commit_after>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package uniqush

import (
    "crypto/tls"
    "net"
    "json"
    "os"
    "io"
    "bytes"
    "encoding/hex"
    "encoding/binary"
    "strconv"
    "sync/atomic"
    "time"
    "fmt"
)

type APNSPushService struct {
    nextid uint32
}

func init() {
    GetPushServiceManager().RegisterPushServiceType(NewAPNSPushService())
}

func NewAPNSPushService() *APNSPushService {
    ret := new(APNSPushService)
    return ret
}

func (p *APNSPushService) Name() string {
    return "apns"
}

func (p *APNSPushService) BuildPushServiceProviderFromMap(kv map[string]string) (*PushServiceProvider, os.Error) {
    psp := NewEmptyPushServiceProvider()
    if service, ok := kv["service"]; ok {
        psp.FixedData["service"] = service
    } else {
        return nil, os.NewError("NoService")
    }

    if cert, ok := kv["cert"]; ok {
        psp.FixedData["cert"] = cert
    } else {
        return nil, os.NewError("NoCertificate")
    }

    if key, ok := kv["key"]; ok {
        psp.FixedData["key"] = key
    } else {
        return nil, os.NewError("NoPrivateKey")
    }

    if sandbox, ok := kv["sandbox"]; ok {
        if sandbox == "true" {
            psp.VolatileData["addr"] = "gateway.sandbox.push.apple.com:2195"
            return psp, nil
        }
    }
    psp.VolatileData["addr"] = "gateway.push.apple.com:2195"
    return psp, nil
}

func (p *APNSPushService) BuildDeliveryPointFromMap(kv map[string]string) (*DeliveryPoint, os.Error) {
    dp := NewEmptyDeliveryPoint()

    if service, ok := kv["service"]; ok {
        dp.FixedData["service"] = service
    } else {
        return nil, os.NewError("NoService")
    }
    if sub, ok := kv["subscriber"]; ok {
        dp.FixedData["subscriber"] = sub
    } else {
        return nil, os.NewError("NoSubscriber")
    }
    if devtoken, ok := kv["devtoken"]; ok {
        dp.FixedData["devtoken"] = devtoken
    } else {
        return nil, os.NewError("NoDevToken")
    }
    return dp, nil
}

func toAPNSPayload(n *Notification) []byte {
    payload := make(map[string]interface{})
    aps := make(map[string]interface{})
    alert := make(map[string]interface{})
    for k, v := range n.Data {
        switch (k) {
        case "msg":
            alert["body"] = v
        case "badge":
            b, err := strconv.Atoi(v)
            if err != nil {
                continue
            } else {
                aps["badge"] = b
            }
        case "sound":
            aps["sound"] = v
        case "img":
            alert["launch-image"] = v
        case "id":
            continue
        case "expiry":
            continue
        default:
            payload[k] = v
        }
    }
    aps["alert"] = alert
    payload["aps"] = aps
    j, err := json.Marshal(payload)
    if err != nil {
        return nil
    }
    return j
}

func writen (w io.Writer, buf []byte) os.Error {
    n := len(buf)
    for n >= 0 {
        l, err := w.Write(buf)
        if err != nil {
            return err
        }
        if l >= n {
            return nil
        }
        n -= l
        buf = buf[l:]
    }
    return nil
}

func (p *APNSPushService) Push(sp *PushServiceProvider,
                        s *DeliveryPoint,
                        n *Notification) (string, os.Error) {
    cert, err := tls.LoadX509KeyPair(sp.FixedData["cert"], sp.FixedData["key"])
    if err != nil {
        return "", NewInvalidPushServiceProviderError(sp)
    }
    conf := &tls.Config {
        Certificates: []tls.Certificate{cert},
    }
    conn, err := net.Dial("tcp", sp.VolatileData["addr"])
    if err != nil {
        return "", NewInvalidPushServiceProviderError(sp)
    }

    tlsconn := tls.Client(conn, conf)
    err = tlsconn.Handshake()
    if err != nil {
        return "", NewInvalidPushServiceProviderError(sp)
    }
    devtoken := s.FixedData["devtoken"]
    btoken, err := hex.DecodeString(devtoken)
    if err != nil {
        return "", NewInvalidDeliveryPointError(sp, s)
    }

    bpayload := toAPNSPayload(n)
    if bpayload == nil {
        /* FIXME new error type */
        return "", NewInvalidDeliveryPointError(sp, s)
    }
    buffer := bytes.NewBuffer([]byte{})
    // command
    binary.Write(buffer, binary.BigEndian, uint8(1))

    // transaction id
    mid := atomic.AddUint32(&(p.nextid), 1)
    if smid, ok := n.Data["id"]; ok {
        imid, err := strconv.Atoui(smid)
        if err == nil {
            mid = uint32(imid)
        }
    }
    binary.Write(buffer, binary.BigEndian, mid)

    // Expiry
    expiry := uint32(time.Seconds() + 60*60)

    if sexpiry, ok := n.Data["expiry"]; ok {
        uiexp, err := strconv.Atoui(sexpiry)
        if err == nil {
            expiry = uint32(uiexp)
        }
    }
    binary.Write(buffer, binary.BigEndian, expiry)

    // device token
    binary.Write(buffer, binary.BigEndian, uint16(len(btoken)))
    binary.Write(buffer, binary.BigEndian, btoken)

    // payload
    binary.Write(buffer, binary.BigEndian, uint16(len(bpayload)))
    binary.Write(buffer, binary.BigEndian, bpayload)
    pdu := buffer.Bytes()

    err = writen(tlsconn, pdu)
    if err != nil {
    return "", NewInvalidPushServiceProviderError(sp)
    }
    tlsconn.SetReadTimeout(5E8)
    readb := [6]byte{}
    nr, err := tlsconn.Read(readb[:])
    /* TODO error handling */
    if nr > 0 {
        switch(readb[1]) {
        case 2:
            return "", NewInvalidDeliveryPointError(sp, s)
        default:
            return "", NewInvalidPushServiceProviderError(sp)
        }
    }
    return fmt.Sprintf("%d", mid), nil
}

<|endoftext|>


<commit_before>package gocsv

import (
	"encoding/csv"
	"fmt"
	"reflect"
)

// Unmarshaller is a CSV to struct unmarshaller.
type Unmarshaller struct {
	reader                 *csv.Reader
	Headers                []string
	fieldInfoMap           []*fieldInfo
	MismatchedHeaders      []string
	MismatchedStructFields []string
	outType                reflect.Type
	out                    interface{}
}

// NewUnmarshaller creates an unmarshaller from a csv.Reader and a struct.
func NewUnmarshaller(reader *csv.Reader, out interface{}) (*Unmarshaller, error) {
	headers, err := reader.Read()
	if err != nil {
		return nil, err
	}
	headers = normalizeHeaders(headers)

	um := &Unmarshaller{reader: reader, outType: reflect.TypeOf(out)}
	err = validate(um, out, headers)
	if err != nil {
		return nil, err
	}
	return um, nil
}

// Read returns an interface{} whose runtime type is the same as the struct that
// was used to create the Unmarshaller.
func (um *Unmarshaller) Read() (interface{}, error) {
	row, err := um.reader.Read()
	if err != nil {
		return nil, err
	}
	return um.unmarshalRow(row, nil)
}

// ReadUnmatched is same as Read(), but returns a map of the columns that didn't match a field in the struct
func (um *Unmarshaller) ReadUnmatched() (interface{}, map[string]string, error) {
	row, err := um.reader.Read()
	if err != nil {
		return nil, nil, err
	}
	unmatched := make(map[string]string)
	value, err := um.unmarshalRow(row, unmatched)
	return value, unmatched, err
}

// validate ensures that a struct was used to create the Unmarshaller, and validates
// CSV headers against the CSV tags in the struct.
func validate(um *Unmarshaller, s interface{}, headers []string) error {
	concreteType := reflect.TypeOf(s)
	if concreteType.Kind() == reflect.Ptr {
		concreteType = concreteType.Elem()
	}
	if err := ensureOutInnerType(concreteType); err != nil {
		return err
	}
	structInfo := getStructInfo(concreteType) // Get struct info to get CSV annotations.
	if len(structInfo.Fields) == 0 {
		return ErrNoStructTags
	}
	csvHeadersLabels := make([]*fieldInfo, len(headers)) // Used to store the corresponding header <-> position in CSV
	headerCount := map[string]int{}
	for i, csvColumnHeader := range headers {
		curHeaderCount := headerCount[csvColumnHeader]
		if fieldInfo := getCSVFieldPosition(csvColumnHeader, structInfo, curHeaderCount); fieldInfo != nil {
			csvHeadersLabels[i] = fieldInfo
			if ShouldAlignDuplicateHeadersWithStructFieldOrder {
				curHeaderCount++
				headerCount[csvColumnHeader] = curHeaderCount
			}
		}
	}

	if FailIfDoubleHeaderNames {
		if err := maybeDoubleHeaderNames(headers); err != nil {
			return err
		}
	}

	um.Headers = headers
	um.fieldInfoMap = csvHeadersLabels
	um.MismatchedHeaders = mismatchHeaderFields(structInfo.Fields, headers)
	um.MismatchedStructFields = mismatchStructFields(structInfo.Fields, headers)
	um.out = s
	return nil
}

// unmarshalRow converts a CSV row to a struct, based on CSV struct tags.
// If unmatched is non nil, it is populated with any columns that don't map to a struct field
func (um *Unmarshaller) unmarshalRow(row []string, unmatched map[string]string) (interface{}, error) {
	isPointer := false
	concreteOutType := um.outType
	if um.outType.Kind() == reflect.Ptr {
		isPointer = true
		concreteOutType = concreteOutType.Elem()
	}
	outValue := createNewOutInner(isPointer, concreteOutType)
	for j, csvColumnContent := range row {
		if j < len(um.fieldInfoMap) && um.fieldInfoMap[j] != nil {
			fieldInfo := um.fieldInfoMap[j]
			if err := setInnerField(&outValue, isPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct
				return nil, fmt.Errorf("cannot assign field at %v to %s through index chain %v: %v", j, outValue.Type(), fieldInfo.IndexChain, err)
			}
		} else if unmatched != nil {
			unmatched[um.Headers[j]] = csvColumnContent
		}
	}
	return outValue.Interface(), nil
}

func (um *Unmarshaller) RenormalizeHeaders(headerNormalizer func([]string) []string) error {
	headers := um.Headers
	if headerNormalizer != nil {
		headers = headerNormalizer(headers)
	}
	err := validate(um, um.out, headers)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>docs: add go doc<commit_after>package gocsv

import (
	"encoding/csv"
	"fmt"
	"reflect"
)

// Unmarshaller is a CSV to struct unmarshaller.
type Unmarshaller struct {
	reader                 *csv.Reader
	Headers                []string
	fieldInfoMap           []*fieldInfo
	MismatchedHeaders      []string
	MismatchedStructFields []string
	outType                reflect.Type
	out                    interface{}
}

// NewUnmarshaller creates an unmarshaller from a csv.Reader and a struct.
func NewUnmarshaller(reader *csv.Reader, out interface{}) (*Unmarshaller, error) {
	headers, err := reader.Read()
	if err != nil {
		return nil, err
	}
	headers = normalizeHeaders(headers)

	um := &Unmarshaller{reader: reader, outType: reflect.TypeOf(out)}
	err = validate(um, out, headers)
	if err != nil {
		return nil, err
	}
	return um, nil
}

// Read returns an interface{} whose runtime type is the same as the struct that
// was used to create the Unmarshaller.
func (um *Unmarshaller) Read() (interface{}, error) {
	row, err := um.reader.Read()
	if err != nil {
		return nil, err
	}
	return um.unmarshalRow(row, nil)
}

// ReadUnmatched is same as Read(), but returns a map of the columns that didn't match a field in the struct
func (um *Unmarshaller) ReadUnmatched() (interface{}, map[string]string, error) {
	row, err := um.reader.Read()
	if err != nil {
		return nil, nil, err
	}
	unmatched := make(map[string]string)
	value, err := um.unmarshalRow(row, unmatched)
	return value, unmatched, err
}

// validate ensures that a struct was used to create the Unmarshaller, and validates
// CSV headers against the CSV tags in the struct.
func validate(um *Unmarshaller, s interface{}, headers []string) error {
	concreteType := reflect.TypeOf(s)
	if concreteType.Kind() == reflect.Ptr {
		concreteType = concreteType.Elem()
	}
	if err := ensureOutInnerType(concreteType); err != nil {
		return err
	}
	structInfo := getStructInfo(concreteType) // Get struct info to get CSV annotations.
	if len(structInfo.Fields) == 0 {
		return ErrNoStructTags
	}
	csvHeadersLabels := make([]*fieldInfo, len(headers)) // Used to store the corresponding header <-> position in CSV
	headerCount := map[string]int{}
	for i, csvColumnHeader := range headers {
		curHeaderCount := headerCount[csvColumnHeader]
		if fieldInfo := getCSVFieldPosition(csvColumnHeader, structInfo, curHeaderCount); fieldInfo != nil {
			csvHeadersLabels[i] = fieldInfo
			if ShouldAlignDuplicateHeadersWithStructFieldOrder {
				curHeaderCount++
				headerCount[csvColumnHeader] = curHeaderCount
			}
		}
	}

	if FailIfDoubleHeaderNames {
		if err := maybeDoubleHeaderNames(headers); err != nil {
			return err
		}
	}

	um.Headers = headers
	um.fieldInfoMap = csvHeadersLabels
	um.MismatchedHeaders = mismatchHeaderFields(structInfo.Fields, headers)
	um.MismatchedStructFields = mismatchStructFields(structInfo.Fields, headers)
	um.out = s
	return nil
}

// unmarshalRow converts a CSV row to a struct, based on CSV struct tags.
// If unmatched is non nil, it is populated with any columns that don't map to a struct field
func (um *Unmarshaller) unmarshalRow(row []string, unmatched map[string]string) (interface{}, error) {
	isPointer := false
	concreteOutType := um.outType
	if um.outType.Kind() == reflect.Ptr {
		isPointer = true
		concreteOutType = concreteOutType.Elem()
	}
	outValue := createNewOutInner(isPointer, concreteOutType)
	for j, csvColumnContent := range row {
		if j < len(um.fieldInfoMap) && um.fieldInfoMap[j] != nil {
			fieldInfo := um.fieldInfoMap[j]
			if err := setInnerField(&outValue, isPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct
				return nil, fmt.Errorf("cannot assign field at %v to %s through index chain %v: %v", j, outValue.Type(), fieldInfo.IndexChain, err)
			}
		} else if unmatched != nil {
			unmatched[um.Headers[j]] = csvColumnContent
		}
	}
	return outValue.Interface(), nil
}

// RenormalizeHeaders will remap the header names based on the headerNormalizer.
// This can be used to map a CSV to a struct where the CSV header names do not match in the file but a mapping is known
func (um *Unmarshaller) RenormalizeHeaders(headerNormalizer func([]string) []string) error {
	headers := um.Headers
	if headerNormalizer != nil {
		headers = headerNormalizer(headers)
	}
	err := validate(um, um.out, headers)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package field

import (
	"github.com/satori/go.uuid"
)

// Hash field which can be embedded into high level models.
// It represents some random unique value.
type Hash struct {
	Value string
}

// NewHash creates a new Hash from the string.
func NewHash(value string) *Hash {
	return &Hash{value}
}

// NewHash creates a new Hash. A length of Hash.Value depends on the iterations number.
// If iteration is equal to 1 resulting Hash.Value will be 36. If it is 2, resulting Hash.Value will be 72, end so on.
func NewRandomHash(iterations int) *Hash {
	var value string

	for i := 0; i < iterations; i++ {
		value += uuid.NewV4().String()
	}

	return NewHash(value)
}
<commit_msg>Add DefaultHashIterations constant<commit_after>package field

import (
	"github.com/satori/go.uuid"
)

const DefaultHashIterations = 1

// Hash field which can be embedded into high level models.
// It represents some random unique value.
type Hash struct {
	Value string
}

// NewHash creates a new Hash from the string.
func NewHash(value string) *Hash {
	return &Hash{value}
}

// NewHash creates a new Hash. A length of Hash.Value depends on the iterations number.
// If iteration is equal to 1 resulting Hash.Value will be 36. If it is 2, resulting Hash.Value will be 72, end so on.
func NewRandomHash(iterations int) *Hash {
	var value string

	for i := 0; i < iterations; i++ {
		value += uuid.NewV4().String()
	}

	return NewHash(value)
}
<|endoftext|>


<commit_before>package purell

import (
	"log"
	"testing"
)

// This tests normalization to a unicode representation
// precent escapes for unreserved values are unescaped to their unicode value
// tests normalization to idna domains
// test ip word handling, ipv6 address handling, and trailing domain periods
// in general, this matches google chromes unescaping for things in the address bar.
// spaces are converted to '+' (perhaphs controversial)
// http://code.google.com/p/google-url/ probably is another good reference for this approach
func TestUrlnorm(t *testing.T) {
	// from https://github.com/jehiah/urlnorm/blob/master/test_urlnorm.py
	testcases := map[string]string{
		"http://1113982867/":                       "http://66.102.7.147/",                    //ip dword encoding
		"http://www.thedraymin.co.uk:/main/?p=308": "http://www.thedraymin.co.uk/main/?p=308", //empty port
		"http://www.foo.com:80/foo":                "http://www.foo.com/foo",
		"http://www.foo.com:8000/foo":              "http://www.foo.com:8000/foo",
		"http://www.foo.com./foo/bar.html":         "http://www.foo.com/foo/bar.html",
		"http://www.foo.com.:81/foo":               "http://www.foo.com:81/foo",
		"http://www.foo.com/%7ebar":                "http://www.foo.com/~bar",
		"http://www.foo.com/%7Ebar":                "http://www.foo.com/~bar",
		"ftp://user:pass@ftp.foo.net/foo/bar":      "ftp://user:pass@ftp.foo.net/foo/bar",
		"http://USER:pass@www.Example.COM/foo/bar": "http://USER:pass@www.example.com/foo/bar",
		"http://www.example.com./":                 "http://www.example.com/",
		"http://test.example/?a=%26&b=1":           "http://test.example/?a=%26&b=1",         //should not un-encode the & that is part of a parameter value
		"http://test.example/?a=%e3%82%82%26":      "http://test.example/?a=\xe3\x82\x82%26", //should return a unicode character

		"http://s.xn--q-bga.de/": "http://s.q\xc3\xa9.de/", //should be in idna format
		"http://test.example/?":  "http://test.example/",   //no trailing ?
		"http://test.example?":   "http://test.example/",   //with trailing /
		"http://a.COM/path/?b&a": "http://a.com/path/?b&a",
		//test utf8 and unicode
		"http://XBLA\u306eXbox.com": "http://xbla\xe3\x81\xaexbox.com/",
		//test idna + utf8 domain
		"http://xn--q-bga.XBLA\u306eXbox.com":                                                                                "http://q\xc3\xa9.xbla\xe3\x81\xaexbox.com",
		"http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3": "http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3",
		"http://test.example/\xe3\x82\xad":                                                                                "http://test.example/\xe3\x82\xad",

		//check that %23 (#) is not escaped where it shouldn"t be
		"http://test.example/?p=%23val#test-%23-val%25": "http://test.example/?p=%23val#test-%23-val%25",
		//check that %20 or %25 is not unescaped to " " or %
		"http://test.example/%25/?p=%20val%20%25":                                       "http://test.example/%25/?p=%20val%20%25",
		"http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n": "http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n",
		//check that spaces are collated to "+"
		"http://test.example/path/with a%20space+/":     "http://test.example/path/with%20a%20space+/",
		"http://[2001:db8:1f70::999:de8:7648:6e8]/test": "http://[2001:db8:1f70::999:de8:7648:6e8]/test", // ipv6 address
		"http://[::ffff:192.168.1.1]/test":              "http://[::ffff:192.168.1.1]/test",              //ipv4 address in ipv6 notation
		"http://[::ffff:192.168.1.1]:80/test":           "http://[::ffff:192.168.1.1]/test",              //ipv4 address in ipv6 notation
		"htTps://[::fFff:192.168.1.1]:443/test":         "https://[::ffff:192.168.1.1]/test",             //ipv4 address in ipv6 notation
	}

	for bad, good := range testcases {
		s, e := NormalizeUrlString(bad, FlagsSafe | FlagRemoveDotSegments);
		if e != nil {
			log.Printf("%s normalizing %v to %v", e.Error(), bad, good)
			t.Fail()
		} else {
			if s != good {
				log.Printf("expected: %v got: %v", good, s)
				t.Fail()
			}
		}
	}
}

func TestSlashes(t *testing.T) {
	// some taken from RFC1808, rfc3986
	testcases := map[string]string{
		"http://test.example/foo/bar/.":               "http://test.example/foo/bar/",
		"http://test.example/foo/bar/./":              "http://test.example/foo/bar/",
		"http://test.example/foo/bar/..":              "http://test.example/foo/",
		"http://test.example/foo/bar/../":             "http://test.example/foo/",
		"http://test.example/foo/bar/../baz":          "http://test.example/foo/baz",
		"http://test.example/foo/bar/../..":           "http://test.example/",
		"http://test.example/foo/bar/../../":          "http://test.example/",
		"http://test.example/foo/bar/../../baz":       "http://test.example/baz",
		"http://test.example/foo/bar/../../../baz":    "http://test.example/baz",
		"http://test.example/foo/bar/../../../../baz": "http://test.example/baz",
		"http://test.example/./foo":                   "http://test.example/foo",
		"http://test.example/../foo":                  "http://test.example/foo",
		"http://test.example/foo.":                    "http://test.example/foo.",
		"http://test.example/.foo":                    "http://test.example/.foo",
		"http://test.example/foo..":                   "http://test.example/foo..",
		"http://test.example/..foo":                   "http://test.example/..foo",
		"http://test.example/./../foo":                "http://test.example/foo",
		"http://test.example/./foo/.":                 "http://test.example/foo/",
		"http://test.example/foo/./bar":               "http://test.example/foo/bar",
		"http://test.example/foo/../bar":              "http://test.example/bar",
		"http://test.example/foo//":                   "http://test.example/foo/",
		"http://test.example/foo///bar//":             "http://test.example/foo/bar/",
	}

	for bad, good := range testcases {
		s, e := NormalizeUrlString(bad, FlagsSafe | FlagRemoveDotSegments);
		if e != nil {
			log.Printf("%s normalizing %v to %v", e.Error(), bad, good)
			t.Fail()
		} else {
			if s != good {
				log.Printf("expected: %v got: %v", good, s)
				t.Fail()
			}
		}
	}
}
<commit_msg>adjust merged tests to new naming<commit_after>package purell

import (
	"log"
	"testing"
)

// This tests normalization to a unicode representation
// precent escapes for unreserved values are unescaped to their unicode value
// tests normalization to idna domains
// test ip word handling, ipv6 address handling, and trailing domain periods
// in general, this matches google chromes unescaping for things in the address bar.
// spaces are converted to '+' (perhaphs controversial)
// http://code.google.com/p/google-url/ probably is another good reference for this approach
func TestUrlnorm(t *testing.T) {
	// from https://github.com/jehiah/urlnorm/blob/master/test_urlnorm.py
	testcases := map[string]string{
		"http://1113982867/":                       "http://66.102.7.147/",                    //ip dword encoding
		"http://www.thedraymin.co.uk:/main/?p=308": "http://www.thedraymin.co.uk/main/?p=308", //empty port
		"http://www.foo.com:80/foo":                "http://www.foo.com/foo",
		"http://www.foo.com:8000/foo":              "http://www.foo.com:8000/foo",
		"http://www.foo.com./foo/bar.html":         "http://www.foo.com/foo/bar.html",
		"http://www.foo.com.:81/foo":               "http://www.foo.com:81/foo",
		"http://www.foo.com/%7ebar":                "http://www.foo.com/~bar",
		"http://www.foo.com/%7Ebar":                "http://www.foo.com/~bar",
		"ftp://user:pass@ftp.foo.net/foo/bar":      "ftp://user:pass@ftp.foo.net/foo/bar",
		"http://USER:pass@www.Example.COM/foo/bar": "http://USER:pass@www.example.com/foo/bar",
		"http://www.example.com./":                 "http://www.example.com/",
		"http://test.example/?a=%26&b=1":           "http://test.example/?a=%26&b=1",         //should not un-encode the & that is part of a parameter value
		"http://test.example/?a=%e3%82%82%26":      "http://test.example/?a=\xe3\x82\x82%26", //should return a unicode character

		"http://s.xn--q-bga.de/": "http://s.q\xc3\xa9.de/", //should be in idna format
		"http://test.example/?":  "http://test.example/",   //no trailing ?
		"http://test.example?":   "http://test.example/",   //with trailing /
		"http://a.COM/path/?b&a": "http://a.com/path/?b&a",
		//test utf8 and unicode
		"http://XBLA\u306eXbox.com": "http://xbla\xe3\x81\xaexbox.com/",
		//test idna + utf8 domain
		"http://xn--q-bga.XBLA\u306eXbox.com":                                                                                "http://q\xc3\xa9.xbla\xe3\x81\xaexbox.com",
		"http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3": "http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3",
		"http://test.example/\xe3\x82\xad":                                                                                "http://test.example/\xe3\x82\xad",

		//check that %23 (#) is not escaped where it shouldn"t be
		"http://test.example/?p=%23val#test-%23-val%25": "http://test.example/?p=%23val#test-%23-val%25",
		//check that %20 or %25 is not unescaped to " " or %
		"http://test.example/%25/?p=%20val%20%25":                                       "http://test.example/%25/?p=%20val%20%25",
		"http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n": "http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n",
		//check that spaces are collated to "+"
		"http://test.example/path/with a%20space+/":     "http://test.example/path/with%20a%20space+/",
		"http://[2001:db8:1f70::999:de8:7648:6e8]/test": "http://[2001:db8:1f70::999:de8:7648:6e8]/test", // ipv6 address
		"http://[::ffff:192.168.1.1]/test":              "http://[::ffff:192.168.1.1]/test",              //ipv4 address in ipv6 notation
		"http://[::ffff:192.168.1.1]:80/test":           "http://[::ffff:192.168.1.1]/test",              //ipv4 address in ipv6 notation
		"htTps://[::fFff:192.168.1.1]:443/test":         "https://[::ffff:192.168.1.1]/test",             //ipv4 address in ipv6 notation
	}

	for bad, good := range testcases {
		s, e := NormalizeURLString(bad, FlagsSafe | FlagRemoveDotSegments);
		if e != nil {
			log.Printf("%s normalizing %v to %v", e.Error(), bad, good)
			t.Fail()
		} else {
			if s != good {
				log.Printf("expected: %v got: %v", good, s)
				t.Fail()
			}
		}
	}
}

func TestSlashes(t *testing.T) {
	// some taken from RFC1808, rfc3986
	testcases := map[string]string{
		"http://test.example/foo/bar/.":               "http://test.example/foo/bar/",
		"http://test.example/foo/bar/./":              "http://test.example/foo/bar/",
		"http://test.example/foo/bar/..":              "http://test.example/foo/",
		"http://test.example/foo/bar/../":             "http://test.example/foo/",
		"http://test.example/foo/bar/../baz":          "http://test.example/foo/baz",
		"http://test.example/foo/bar/../..":           "http://test.example/",
		"http://test.example/foo/bar/../../":          "http://test.example/",
		"http://test.example/foo/bar/../../baz":       "http://test.example/baz",
		"http://test.example/foo/bar/../../../baz":    "http://test.example/baz",
		"http://test.example/foo/bar/../../../../baz": "http://test.example/baz",
		"http://test.example/./foo":                   "http://test.example/foo",
		"http://test.example/../foo":                  "http://test.example/foo",
		"http://test.example/foo.":                    "http://test.example/foo.",
		"http://test.example/.foo":                    "http://test.example/.foo",
		"http://test.example/foo..":                   "http://test.example/foo..",
		"http://test.example/..foo":                   "http://test.example/..foo",
		"http://test.example/./../foo":                "http://test.example/foo",
		"http://test.example/./foo/.":                 "http://test.example/foo/",
		"http://test.example/foo/./bar":               "http://test.example/foo/bar",
		"http://test.example/foo/../bar":              "http://test.example/bar",
		"http://test.example/foo//":                   "http://test.example/foo/",
		"http://test.example/foo///bar//":             "http://test.example/foo/bar/",
	}

	for bad, good := range testcases {
		s, e := NormalizeURLString(bad, FlagsSafe | FlagRemoveDotSegments);
		if e != nil {
			log.Printf("%s normalizing %v to %v", e.Error(), bad, good)
			t.Fail()
		} else {
			if s != good {
				log.Printf("expected: %v got: %v", good, s)
				t.Fail()
			}
		}
	}
}
<|endoftext|>


<commit_before>package container

import (
	"fmt"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/go-connections/nat"
)

// Snapshot is a read only view for Containers
type Snapshot struct {
	ID           string `json:"Id"`
	Name         string
	Pid          int
	Managed      bool
	Image        string
	ImageID      string
	Command      string
	Ports        []types.Port
	ExposedPorts nat.PortSet
	PublishPorts nat.PortSet
	Labels       map[string]string
	State        string
	Status       string
	Health       string
	HostConfig   struct {
		NetworkMode string
		Isolation   string
	}
	NetworkSettings types.SummaryNetworkSettings
	Mounts          []types.MountPoint
	Created         time.Time
	StartedAt       time.Time
	Running         bool
	Paused          bool
	ExitCode        int
}

// Snapshot provides a read only view of a Container. Callers must hold a Lock on the container object.
func (container *Container) snapshot() *Snapshot {
	snapshot := &Snapshot{
		ID:           container.ID,
		Name:         container.Name,
		Pid:          container.Pid,
		Managed:      container.Managed,
		ImageID:      container.ImageID.String(),
		Ports:        []types.Port{},
		ExposedPorts: make(nat.PortSet),
		PublishPorts: make(nat.PortSet),
		State:        container.State.StateString(),
		Status:       container.State.String(),
		Health:       container.State.HealthString(),
		Mounts:       container.GetMountPoints(),
		Created:      container.Created,
		StartedAt:    container.StartedAt,
		Running:      container.Running,
		Paused:       container.Paused,
		ExitCode:     container.ExitCode(),
	}

	if container.HostConfig != nil {
		snapshot.HostConfig.Isolation = string(container.HostConfig.Isolation)
		snapshot.HostConfig.NetworkMode = string(container.HostConfig.NetworkMode)
		for publish := range container.HostConfig.PortBindings {
			snapshot.PublishPorts[publish] = struct{}{}
		}
	}

	if container.Config != nil {
		snapshot.Image = container.Config.Image
		snapshot.Labels = container.Config.Labels
		for exposed := range container.Config.ExposedPorts {
			snapshot.ExposedPorts[exposed] = struct{}{}
		}
	}

	if len(container.Args) > 0 {
		args := []string{}
		for _, arg := range container.Args {
			if strings.Contains(arg, " ") {
				args = append(args, fmt.Sprintf("'%s'", arg))
			} else {
				args = append(args, arg)
			}
		}
		argsAsString := strings.Join(args, " ")
		snapshot.Command = fmt.Sprintf("%s %s", container.Path, argsAsString)
	} else {
		snapshot.Command = container.Path
	}

	if container.NetworkSettings != nil {
		networks := make(map[string]*network.EndpointSettings)
		for name, netw := range container.NetworkSettings.Networks {
			if netw == nil || netw.EndpointSettings == nil {
				continue
			}
			networks[name] = &network.EndpointSettings{
				EndpointID:          netw.EndpointID,
				Gateway:             netw.Gateway,
				IPAddress:           netw.IPAddress,
				IPPrefixLen:         netw.IPPrefixLen,
				IPv6Gateway:         netw.IPv6Gateway,
				GlobalIPv6Address:   netw.GlobalIPv6Address,
				GlobalIPv6PrefixLen: netw.GlobalIPv6PrefixLen,
				MacAddress:          netw.MacAddress,
				NetworkID:           netw.NetworkID,
			}
			if netw.IPAMConfig != nil {
				networks[name].IPAMConfig = &network.EndpointIPAMConfig{
					IPv4Address: netw.IPAMConfig.IPv4Address,
					IPv6Address: netw.IPAMConfig.IPv6Address,
				}
			}
		}
		snapshot.NetworkSettings = types.SummaryNetworkSettings{Networks: networks}
		for port, bindings := range container.NetworkSettings.Ports {
			p, err := nat.ParsePort(port.Port())
			if err != nil {
				logrus.Warnf("invalid port map %+v", err)
				continue
			}
			if len(bindings) == 0 {
				snapshot.Ports = append(snapshot.Ports, types.Port{
					PrivatePort: uint16(p),
					Type:        port.Proto(),
				})
				continue
			}
			for _, binding := range bindings {
				h, err := nat.ParsePort(binding.HostPort)
				if err != nil {
					logrus.Warnf("invalid host port map %+v", err)
					continue
				}
				snapshot.Ports = append(snapshot.Ports, types.Port{
					PrivatePort: uint16(p),
					PublicPort:  uint16(h),
					Type:        port.Proto(),
					IP:          binding.HostIP,
				})
			}
		}

	}

	return snapshot
}
<commit_msg>how to maintain the container snapshot struct<commit_after>package container

import (
	"fmt"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/go-connections/nat"
)

// Snapshot is a read only view for Containers. It holds all information necessary to serve container queries in a
// versioned ACID in-memory store. Pointers are avoided here to make sure all values are copied into the store.
type Snapshot struct {
	ID           string `json:"Id"`
	Name         string
	Pid          int
	Managed      bool
	Image        string
	ImageID      string
	Command      string
	Ports        []types.Port
	ExposedPorts nat.PortSet
	PublishPorts nat.PortSet
	Labels       map[string]string
	State        string
	Status       string
	Health       string
	HostConfig   struct {
		NetworkMode string
		Isolation   string
	}
	NetworkSettings types.SummaryNetworkSettings
	Mounts          []types.MountPoint
	Created         time.Time
	StartedAt       time.Time
	Running         bool
	Paused          bool
	ExitCode        int
}

// Snapshot provides a read only view of a Container. Callers must hold a Lock on the container object.
func (container *Container) snapshot() *Snapshot {
	snapshot := &Snapshot{
		ID:           container.ID,
		Name:         container.Name,
		Pid:          container.Pid,
		Managed:      container.Managed,
		ImageID:      container.ImageID.String(),
		Ports:        []types.Port{},
		ExposedPorts: make(nat.PortSet),
		PublishPorts: make(nat.PortSet),
		State:        container.State.StateString(),
		Status:       container.State.String(),
		Health:       container.State.HealthString(),
		Mounts:       container.GetMountPoints(),
		Created:      container.Created,
		StartedAt:    container.StartedAt,
		Running:      container.Running,
		Paused:       container.Paused,
		ExitCode:     container.ExitCode(),
	}

	if container.HostConfig != nil {
		snapshot.HostConfig.Isolation = string(container.HostConfig.Isolation)
		snapshot.HostConfig.NetworkMode = string(container.HostConfig.NetworkMode)
		for publish := range container.HostConfig.PortBindings {
			snapshot.PublishPorts[publish] = struct{}{}
		}
	}

	if container.Config != nil {
		snapshot.Image = container.Config.Image
		snapshot.Labels = container.Config.Labels
		for exposed := range container.Config.ExposedPorts {
			snapshot.ExposedPorts[exposed] = struct{}{}
		}
	}

	if len(container.Args) > 0 {
		args := []string{}
		for _, arg := range container.Args {
			if strings.Contains(arg, " ") {
				args = append(args, fmt.Sprintf("'%s'", arg))
			} else {
				args = append(args, arg)
			}
		}
		argsAsString := strings.Join(args, " ")
		snapshot.Command = fmt.Sprintf("%s %s", container.Path, argsAsString)
	} else {
		snapshot.Command = container.Path
	}

	if container.NetworkSettings != nil {
		networks := make(map[string]*network.EndpointSettings)
		for name, netw := range container.NetworkSettings.Networks {
			if netw == nil || netw.EndpointSettings == nil {
				continue
			}
			networks[name] = &network.EndpointSettings{
				EndpointID:          netw.EndpointID,
				Gateway:             netw.Gateway,
				IPAddress:           netw.IPAddress,
				IPPrefixLen:         netw.IPPrefixLen,
				IPv6Gateway:         netw.IPv6Gateway,
				GlobalIPv6Address:   netw.GlobalIPv6Address,
				GlobalIPv6PrefixLen: netw.GlobalIPv6PrefixLen,
				MacAddress:          netw.MacAddress,
				NetworkID:           netw.NetworkID,
			}
			if netw.IPAMConfig != nil {
				networks[name].IPAMConfig = &network.EndpointIPAMConfig{
					IPv4Address: netw.IPAMConfig.IPv4Address,
					IPv6Address: netw.IPAMConfig.IPv6Address,
				}
			}
		}
		snapshot.NetworkSettings = types.SummaryNetworkSettings{Networks: networks}
		for port, bindings := range container.NetworkSettings.Ports {
			p, err := nat.ParsePort(port.Port())
			if err != nil {
				logrus.Warnf("invalid port map %+v", err)
				continue
			}
			if len(bindings) == 0 {
				snapshot.Ports = append(snapshot.Ports, types.Port{
					PrivatePort: uint16(p),
					Type:        port.Proto(),
				})
				continue
			}
			for _, binding := range bindings {
				h, err := nat.ParsePort(binding.HostPort)
				if err != nil {
					logrus.Warnf("invalid host port map %+v", err)
					continue
				}
				snapshot.Ports = append(snapshot.Ports, types.Port{
					PrivatePort: uint16(p),
					PublicPort:  uint16(h),
					Type:        port.Proto(),
					IP:          binding.HostIP,
				})
			}
		}

	}

	return snapshot
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"strings"
)

type metadataParseHandler func(key, value string)
type bodyParseHandler func(value string)
type fileInfoParseHandler func(fileInfo os.FileInfo)

func loadBlogFile(path string, fileInfoHandler fileInfoParseHandler, metadataHandler metadataParseHandler, bodyHandler bodyParseHandler) error {

	info, err := os.Stat(path)

	if err != nil {
		return err
	}

	fileInfoHandler(info)

	file, err := ioutil.ReadFile(path)

	if err != nil {
		return err
	}

	parseBlogFile(string(file), metadataHandler, bodyHandler)

	return nil
}

func parseBlogFile(text string, metadataHandler metadataParseHandler, bodyHandler bodyParseHandler) {
	text = strings.Replace(text, "\r", "", -1)

	headerSize := parseBlogFileHeader(text, metadataHandler)
	body := text[headerSize:]
	bodyHandler(body)
}

func parseBlogFileHeader(text string, handler metadataParseHandler) int {

	lines := strings.Split(text, "\n")
	headerSize := 0

	for _, line := range lines {
		if strings.Contains(line, ":") {
			components := strings.Split(line, ":")
			key := strings.ToLower(strings.TrimSpace(components[0]))
			separatorIndex := strings.Index(line, ":") + 1
			value := strings.TrimSpace(line[separatorIndex:])

			headerSize += len(line) + 1

			handler(key, value)
		} else {
			break
		}
	}

	return headerSize
}

func convertMarkdownToHtml(markdown *[]byte) string {
	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	output := blackfriday.Markdown(*markdown, renderer, extensions)

	return string(output)
}
<commit_msg>Configured BlackFriday to support strikethrough.<commit_after>package main

import (
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"strings"
)

type metadataParseHandler func(key, value string)
type bodyParseHandler func(value string)
type fileInfoParseHandler func(fileInfo os.FileInfo)

func loadBlogFile(path string, fileInfoHandler fileInfoParseHandler, metadataHandler metadataParseHandler, bodyHandler bodyParseHandler) error {

	info, err := os.Stat(path)

	if err != nil {
		return err
	}

	fileInfoHandler(info)

	file, err := ioutil.ReadFile(path)

	if err != nil {
		return err
	}

	parseBlogFile(string(file), metadataHandler, bodyHandler)

	return nil
}

func parseBlogFile(text string, metadataHandler metadataParseHandler, bodyHandler bodyParseHandler) {
	text = strings.Replace(text, "\r", "", -1)

	headerSize := parseBlogFileHeader(text, metadataHandler)
	body := text[headerSize:]
	bodyHandler(body)
}

func parseBlogFileHeader(text string, handler metadataParseHandler) int {

	lines := strings.Split(text, "\n")
	headerSize := 0

	for _, line := range lines {
		if strings.Contains(line, ":") {
			components := strings.Split(line, ":")
			key := strings.ToLower(strings.TrimSpace(components[0]))
			separatorIndex := strings.Index(line, ":") + 1
			value := strings.TrimSpace(line[separatorIndex:])

			headerSize += len(line) + 1

			handler(key, value)
		} else {
			break
		}
	}

	return headerSize
}

func convertMarkdownToHtml(markdown *[]byte) string {
	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS | blackfriday.EXTENSION_STRIKETHROUGH

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	output := blackfriday.Markdown(*markdown, renderer, extensions)

	return string(output)
}
<|endoftext|>