<commit_before>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"sync"

	"github.com/01org/cc-oci-runtime/proxy/api"
)

// Main struct holding the proxy state
type proxy struct {
	// Protect concurrent accesses from separate client goroutines to this
	// structure fields
	sync.Mutex

	// proxy socket
	listener net.Listener

	// vms are hashed by their containerID
	vms map[string]*vm
}

// Represents a client, either a cc-oci-runtime or cc-shim process having
// opened a socket to the proxy
type client struct {
	proxy *proxy
	vm    *vm

	conn net.Conn
}

// "hello"
func helloHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	hello := api.Hello{}

	if err := json.Unmarshal(data, &hello); err != nil {
		response.SetError(err)
		return
	}

	if hello.ContainerID == "" || hello.CtlSerial == "" || hello.IoSerial == "" {
		response.SetErrorMsg("malformed hello command")
	}

	proxy := client.proxy
	proxy.Lock()
	if _, ok := proxy.vms[hello.ContainerID]; ok {

		proxy.Unlock()
		response.SetErrorf("%s: container already registered",
			hello.ContainerID)
		return
	}
	vm := newVM(hello.ContainerID, hello.CtlSerial, hello.IoSerial)
	proxy.vms[hello.ContainerID] = vm
	proxy.Unlock()

	if err := vm.Connect(); err != nil {
		proxy.Lock()
		delete(proxy.vms, hello.ContainerID)
		proxy.Unlock()
		response.SetError(err)
		return
	}

	client.vm = vm
}

// "attach"
func attachHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	proxy := client.proxy

	attach := api.Attach{}
	if err := json.Unmarshal(data, &attach); err != nil {
		response.SetError(err)
		return
	}

	proxy.Lock()
	vm := proxy.vms[attach.ContainerID]
	proxy.Unlock()

	if vm == nil {
		response.SetErrorf("unknown containerID: %s", attach.ContainerID)
		return
	}

	client.vm = vm
}

// "bye"
func byeHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	proxy := client.proxy

	bye := api.Bye{}
	if err := json.Unmarshal(data, &bye); err != nil {
		response.SetError(err)
		return
	}

	proxy.Lock()
	vm := proxy.vms[bye.ContainerID]
	proxy.Unlock()

	if vm == nil {
		response.SetErrorf("unknown containerID: %s", bye.ContainerID)
		return
	}

	proxy.Lock()
	delete(proxy.vms, vm.containerID)
	proxy.Unlock()

	client.vm = nil
	vm.Close()
}

// "allocateIO"
func allocateIoHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	vm := client.vm

	allocateIo := api.AllocateIo{}
	if err := json.Unmarshal(data, &allocateIo); err != nil {
		response.SetError(err)
		return
	}

	if allocateIo.NStreams < 1 || allocateIo.NStreams > 2 {
		response.SetErrorf("asking for unexpected number of streams (%d)",
			allocateIo.NStreams)
	}

	if vm == nil {
		response.SetErrorMsg("client not attached to a vm")
		return
	}

	// We'll send c0 to the client, keep c1
	c0, c1, err := Socketpair()
	if err != nil {
		response.SetError(err)
		return
	}

	f0, err := c0.File()
	if err != nil {
		response.SetError(err)
		return
	}

	ioBase := vm.AllocateIo(allocateIo.NStreams, c1)

	response.AddResult("ioBase", ioBase)
	response.SetFile(f0)

	// File() dups the underlying fd, so it's safe to close c0 here (will
	// keep the c0 <-> c1 connection alive).
	c0.Close()
}

// "hyper"
func hyperHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	hyper := api.Hyper{}
	vm := client.vm

	if err := json.Unmarshal(data, &hyper); err != nil {
		response.SetError(err)
		return
	}

	if vm == nil {
		response.SetErrorMsg("client not attached to a vm")
		return
	}

	err := vm.SendMessage(hyper.HyperName, hyper.Data)
	response.SetError(err)
}

func newProxy() *proxy {
	return &proxy{
		vms: make(map[string]*vm),
	}
}

// This variable is populated at link time with the value of:
//   ${locatestatedir}/run/cc-oci-runtime/proxy
var socketPath string

func (proxy *proxy) init() error {
	var l net.Listener
	var err error

	fds := listenFds()

	if len(fds) > 1 {
		return fmt.Errorf("too many activated sockets (%d)", len(fds))
	} else if len(fds) == 1 {
		fd := fds[0]
		l, err = net.FileListener(fd)
		if err != nil {
			return fmt.Errorf("couldn't listen on socket: %v", err)
		}
	} else {
		if err = os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("couldn't remove exiting socket: %v\n", err)
		}
		l, err = net.ListenUnix("unix", &net.UnixAddr{Name: socketPath, Net: "unix"})
		if err != nil {
			return fmt.Errorf("couldn't create AF_UNIX socket: %v", err)
		}
		if err = os.Chmod(socketPath, 0666|os.ModeSocket); err != nil {
			return fmt.Errorf("couldn't set mode on socket: %v", err)
		}
	}

	proxy.listener = l

	return nil
}

func (proxy *proxy) serveNewClient(proto *protocol, newConn net.Conn) {
	newClient := &client{
		proxy: proxy,
		conn:  newConn,
	}

	proto.Serve(newConn, newClient)
}

func (proxy *proxy) serve() {

	// Define the client (runtime/shim) <-> proxy protocol
	proto := newProtocol()
	proto.Handle("hello", helloHandler)
	proto.Handle("attach", attachHandler)
	proto.Handle("bye", byeHandler)
	proto.Handle("allocateIO", allocateIoHandler)
	proto.Handle("hyper", hyperHandler)

	for {
		conn, err := proxy.listener.Accept()
		if err != nil {
			fmt.Fprintln(os.Stderr, "couldn't accept connection:", err)
			continue
		}

		go proxy.serveNewClient(proto, conn)
	}
}

func proxyMain() {
	proxy := newProxy()
	if err := proxy.init(); err != nil {
		fmt.Fprintln(os.Stderr, "init:", err.Error())
		os.Exit(1)
	}
	proxy.serve()
}

func main() {
	proxyMain()
}
<commit_msg>Proxy: Fix gofmt newline error.<commit_after>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"sync"

	"github.com/01org/cc-oci-runtime/proxy/api"
)

// Main struct holding the proxy state
type proxy struct {
	// Protect concurrent accesses from separate client goroutines to this
	// structure fields
	sync.Mutex

	// proxy socket
	listener net.Listener

	// vms are hashed by their containerID
	vms map[string]*vm
}

// Represents a client, either a cc-oci-runtime or cc-shim process having
// opened a socket to the proxy
type client struct {
	proxy *proxy
	vm    *vm

	conn net.Conn
}

// "hello"
func helloHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	hello := api.Hello{}

	if err := json.Unmarshal(data, &hello); err != nil {
		response.SetError(err)
		return
	}

	if hello.ContainerID == "" || hello.CtlSerial == "" || hello.IoSerial == "" {
		response.SetErrorMsg("malformed hello command")
	}

	proxy := client.proxy
	proxy.Lock()
	if _, ok := proxy.vms[hello.ContainerID]; ok {

		proxy.Unlock()
		response.SetErrorf("%s: container already registered",
			hello.ContainerID)
		return
	}
	vm := newVM(hello.ContainerID, hello.CtlSerial, hello.IoSerial)
	proxy.vms[hello.ContainerID] = vm
	proxy.Unlock()

	if err := vm.Connect(); err != nil {
		proxy.Lock()
		delete(proxy.vms, hello.ContainerID)
		proxy.Unlock()
		response.SetError(err)
		return
	}

	client.vm = vm
}

// "attach"
func attachHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	proxy := client.proxy

	attach := api.Attach{}
	if err := json.Unmarshal(data, &attach); err != nil {
		response.SetError(err)
		return
	}

	proxy.Lock()
	vm := proxy.vms[attach.ContainerID]
	proxy.Unlock()

	if vm == nil {
		response.SetErrorf("unknown containerID: %s", attach.ContainerID)
		return
	}

	client.vm = vm
}

// "bye"
func byeHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	proxy := client.proxy

	bye := api.Bye{}
	if err := json.Unmarshal(data, &bye); err != nil {
		response.SetError(err)
		return
	}

	proxy.Lock()
	vm := proxy.vms[bye.ContainerID]
	proxy.Unlock()

	if vm == nil {
		response.SetErrorf("unknown containerID: %s", bye.ContainerID)
		return
	}

	proxy.Lock()
	delete(proxy.vms, vm.containerID)
	proxy.Unlock()

	client.vm = nil
	vm.Close()
}

// "allocateIO"
func allocateIoHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	vm := client.vm

	allocateIo := api.AllocateIo{}
	if err := json.Unmarshal(data, &allocateIo); err != nil {
		response.SetError(err)
		return
	}

	if allocateIo.NStreams < 1 || allocateIo.NStreams > 2 {
		response.SetErrorf("asking for unexpected number of streams (%d)",
			allocateIo.NStreams)
	}

	if vm == nil {
		response.SetErrorMsg("client not attached to a vm")
		return
	}

	// We'll send c0 to the client, keep c1
	c0, c1, err := Socketpair()
	if err != nil {
		response.SetError(err)
		return
	}

	f0, err := c0.File()
	if err != nil {
		response.SetError(err)
		return
	}

	ioBase := vm.AllocateIo(allocateIo.NStreams, c1)

	response.AddResult("ioBase", ioBase)
	response.SetFile(f0)

	// File() dups the underlying fd, so it's safe to close c0 here (will
	// keep the c0 <-> c1 connection alive).
	c0.Close()
}

// "hyper"
func hyperHandler(data []byte, userData interface{}, response *handlerResponse) {
	client := userData.(*client)
	hyper := api.Hyper{}
	vm := client.vm

	if err := json.Unmarshal(data, &hyper); err != nil {
		response.SetError(err)
		return
	}

	if vm == nil {
		response.SetErrorMsg("client not attached to a vm")
		return
	}

	err := vm.SendMessage(hyper.HyperName, hyper.Data)
	response.SetError(err)
}

func newProxy() *proxy {
	return &proxy{
		vms: make(map[string]*vm),
	}
}

// This variable is populated at link time with the value of:
//   ${locatestatedir}/run/cc-oci-runtime/proxy
var socketPath string

func (proxy *proxy) init() error {
	var l net.Listener
	var err error

	fds := listenFds()

	if len(fds) > 1 {
		return fmt.Errorf("too many activated sockets (%d)", len(fds))
	} else if len(fds) == 1 {
		fd := fds[0]
		l, err = net.FileListener(fd)
		if err != nil {
			return fmt.Errorf("couldn't listen on socket: %v", err)
		}
	} else {
		if err = os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("couldn't remove exiting socket: %v", err)
		}
		l, err = net.ListenUnix("unix", &net.UnixAddr{Name: socketPath, Net: "unix"})
		if err != nil {
			return fmt.Errorf("couldn't create AF_UNIX socket: %v", err)
		}
		if err = os.Chmod(socketPath, 0666|os.ModeSocket); err != nil {
			return fmt.Errorf("couldn't set mode on socket: %v", err)
		}
	}

	proxy.listener = l

	return nil
}

func (proxy *proxy) serveNewClient(proto *protocol, newConn net.Conn) {
	newClient := &client{
		proxy: proxy,
		conn:  newConn,
	}

	proto.Serve(newConn, newClient)
}

func (proxy *proxy) serve() {

	// Define the client (runtime/shim) <-> proxy protocol
	proto := newProtocol()
	proto.Handle("hello", helloHandler)
	proto.Handle("attach", attachHandler)
	proto.Handle("bye", byeHandler)
	proto.Handle("allocateIO", allocateIoHandler)
	proto.Handle("hyper", hyperHandler)

	for {
		conn, err := proxy.listener.Accept()
		if err != nil {
			fmt.Fprintln(os.Stderr, "couldn't accept connection:", err)
			continue
		}

		go proxy.serveNewClient(proto, conn)
	}
}

func proxyMain() {
	proxy := newProxy()
	if err := proxy.init(); err != nil {
		fmt.Fprintln(os.Stderr, "init:", err.Error())
		os.Exit(1)
	}
	proxy.serve()
}

func main() {
	proxyMain()
}
<|endoftext|>


<commit_before>package main

//presharding
//compute hash ,find target redis instance,construct new key with hashtag
import (
	"errors"
	"fmt"
	"hash/crc32"
	"net"
)

var (
	crcTable *crc32.Table
	target   RespReaderWriter
	shards   Shards
)

func Hash(key string) uint32 {
	crc := crc32.New(crcTable)
	crc.Write([]byte(key))
	v := crc.Sum32()
	return v
}

type Shard struct {
	Host          string
	Slot0         uint32
	Slot1         uint32
	ShardRespPool *ObjectPool
}

type Shards struct {
	Slots        uint32
	ShardServers []Shard
}

func NewShards() Shards {
	sh := Shards{}
	s1 := Shard{}
	s1.Host = "172.31.6.58:6479"
	s1.Slot0 = 0
	s1.Slot1 = 512
	s2 := Shard{}
	s2.Host = "172.31.6.58:6579"
	s2.Slot0 = 512
	s2.Slot1 = 1024
	s1.ShardRespPool = NewProxyClientPool(s1.Host)
	s2.ShardRespPool = NewProxyClientPool(s2.Host)
	sh.ShardServers = []Shard{s1, s2}
	return sh

}
func (shards Shards) GetPooledObject(hash uint32) (obj *PooledObject, shard Shard, err error) {
	for _, shard := range shards.ShardServers {
		if hash >= shard.Slot0 && hash < shard.Slot1 {
			fmt.Printf("get shard:%v\n", shard)
			obj, er := shard.ShardRespPool.Borrow()
			return obj, shard, er
		}
	}
	return nil, Shard{}, nil
}

func (shards Shards) GetShard(hash uint32) (Shard, error) {
	for i, shard := range shards.ShardServers {
		if hash >= shard.Slot0 && hash < shard.Slot1 {
			return shards.ShardServers[i], nil
		}
	}
	return Shard{}, errors.New("no shard found")
}

func HandleConn(conn net.Conn) {
	//	timeout := time.Now()
	//	timeout.Add(10 *time.Minute )
	//	conn.SetReadDeadline(timeout)
	//conn.
	client := NewRespReadWriter(conn)
	cch := make(chan int) //close flag chan
	dch := client.LoopRead(cch)
	shardMap := make(map[Shard]*PooledObject)
	var lastHash uint32
	defer func() {
		for shard, o := range shardMap {
			//			if o != nil {
			if lastHash >= shard.Slot0 && lastHash < shard.Slot1 {
				o.Broken = true //set last one to broken
				fmt.Printf("set conn to broken,lastHash :%d\n", lastHash)
			}
			o.Release()
			//			}

		}
		client.Close()
	}()
	for {
		select {
		case <-cch:
			return
		case params := <-dch:
			if len(params) == 0 {
				continue
			}
			//fmt.Println(params)
			cmd := params[0].(string)
			if cmd == "PING" {
				client.ProxyWrite("+PONG\r\n")
			} else if cmd == "QUIT" {
				//client closed
				client.ProxyWrite("+OK\r\n")
				client.Close()
				return
			} else {
				if len(params) < 2 {
					fmt.Println(params)
				}
				key := params[1].(string)
				hash := Hash(key) % uint32(1024)
				lastHash = hash
				key = fmt.Sprint(hash) + "_" + key
				//fmt.Println(key)
				s := "*" + fmt.Sprint(len(params)) + "\r\n"
				s += "$" + fmt.Sprint(len(cmd)) + "\r\n" + cmd + "\r\n"
				s += "$" + fmt.Sprint(len(key)) + "\r\n" + key + "\r\n"

				//here should checke nil and empty string
				for i := 2; i < len(params); i++ {
					s += "$" + fmt.Sprint(len(params[i].(string))) + "\r\n" + params[i].(string) + "\r\n"
				}
				shard, err := shards.GetShard(hash)
				if err != nil {
					//handle error
					client.Close()
					return
				}
				if ob, ok := shardMap[shard]; !ok {
					fmt.Println("get new shard")
					ob, err = shard.ShardRespPool.Borrow()
					if err != nil {
						client.Close()
						return
					} else if ob != nil {
						shardMap[shard] = ob
					}
				}

				server := shardMap[shard]
				ss := server.Value.(*RespReaderWriter)
				err = ss.ProxyWrite(s)
				if err != nil {
					fmt.Printf("backend server write error:%v\n", err)
					server.Broken = true
					server.Release()
					return
				}
				resp, err := ss.ProxyRead()
				if err != nil {
					fmt.Printf("backend server read error:%v\n", err)
					server.Broken = true
					//server.Release()
					return
				}
				err = client.ProxyWrite(resp)
				if err != nil {
					fmt.Println("clent error")
					server.Broken = true
					server.Release()
				}
			}

		}

	}

}

func init() {
	shards = NewShards()
}
<commit_msg>quit connection when exit routine<commit_after>package main

//presharding
//compute hash ,find target redis instance,construct new key with hashtag
import (
	"errors"
	"fmt"
	"hash/crc32"
	"net"
)

var (
	crcTable *crc32.Table
	target   RespReaderWriter
	shards   Shards
)

func Hash(key string) uint32 {
	crc := crc32.New(crcTable)
	crc.Write([]byte(key))
	v := crc.Sum32()
	return v
}

type Shard struct {
	Host          string
	Slot0         uint32
	Slot1         uint32
	ShardRespPool *ObjectPool
}

type Shards struct {
	Slots        uint32
	ShardServers []Shard
}

func NewShards() Shards {
	sh := Shards{}
	s1 := Shard{}
	s1.Host = "172.31.6.58:6479"
	s1.Slot0 = 0
	s1.Slot1 = 512
	s2 := Shard{}
	s2.Host = "172.31.6.58:6579"
	s2.Slot0 = 512
	s2.Slot1 = 1024
	s1.ShardRespPool = NewProxyClientPool(s1.Host)
	s2.ShardRespPool = NewProxyClientPool(s2.Host)
	sh.ShardServers = []Shard{s1, s2}
	return sh

}
func (shards Shards) GetPooledObject(hash uint32) (obj *PooledObject, shard Shard, err error) {
	for _, shard := range shards.ShardServers {
		if hash >= shard.Slot0 && hash < shard.Slot1 {
			fmt.Printf("get shard:%v\n", shard)
			obj, er := shard.ShardRespPool.Borrow()
			return obj, shard, er
		}
	}
	return nil, Shard{}, nil
}

func (shards Shards) GetShard(hash uint32) (Shard, error) {
	for i, shard := range shards.ShardServers {
		if hash >= shard.Slot0 && hash < shard.Slot1 {
			return shards.ShardServers[i], nil
		}
	}
	return Shard{}, errors.New("no shard found")
}

func HandleConn(conn net.Conn) {
	//	timeout := time.Now()
	//	timeout.Add(10 *time.Minute )
	//	conn.SetReadDeadline(timeout)
	//conn.
	client := NewRespReadWriter(conn)
	cch := make(chan int) //close flag chan
	dch := client.LoopRead(cch)
	shardMap := make(map[Shard]*PooledObject)
	var lastHash uint32
	defer func() {
		for shard, o := range shardMap {
			if lastHash >= shard.Slot0 && lastHash < shard.Slot1 {
				serverConn:=o.Value.(*RespReaderWriter)
				s:="*1\r\n$4\r\nQUIT\r\n"
				err:=serverConn.ProxyWrite(s)
				if err!=nil{
					fmt.Printf("send quit error %v\n",err)
				}
				o.Broken = true //set last one to broken
				fmt.Printf("set conn to broken,lastHash :%d\n", lastHash)
			}
			o.Release()

		}
		client.Close()
	}()
	for {
		select {
		case <-cch:
			return
		case params := <-dch:
			if len(params) == 0 {
				continue
			}
			cmd := params[0].(string)
			if cmd == "PING" {
				client.ProxyWrite("+PONG\r\n")
			} else if cmd == "QUIT" {
				client.ProxyWrite("+OK\r\n")
				client.Close()
				return
			} else if cmd == "ECHO" {
				s:=fmt.Sprintf("$%d%s\r\n",len(params[1].(string)),params[1].(string))
				client.ProxyWrite(s)
			} else {
				if len(params) < 2 {
					fmt.Println(params)
				}
				key := params[1].(string)
				hash := Hash(key) % uint32(1024)
				lastHash = hash
				key = fmt.Sprint(hash) + "_" + key
				//fmt.Println(key)
				s := "*" + fmt.Sprint(len(params)) + "\r\n"
				s += "$" + fmt.Sprint(len(cmd)) + "\r\n" + cmd + "\r\n"
				s += "$" + fmt.Sprint(len(key)) + "\r\n" + key + "\r\n"

				//here should check nil and empty string
				for i := 2; i < len(params); i++ {
					s += "$" + fmt.Sprint(len(params[i].(string))) + "\r\n" + params[i].(string) + "\r\n"
				}
				shard, err := shards.GetShard(hash)
				if err != nil {
					//handle error
					client.Close()
					return
				}
				if ob, ok := shardMap[shard]; !ok {
					fmt.Println("get new shard")
					ob, err = shard.ShardRespPool.Borrow()
					if err != nil {
						client.Close()
						return
					} else if ob != nil {
						shardMap[shard] = ob
					}
				}

				server := shardMap[shard]
				ss := server.Value.(*RespReaderWriter)
				err = ss.ProxyWrite(s)
				if err != nil {
					fmt.Printf("backend server write error:%v\n", err)
					server.Broken = true
					server.Release()
					return
				}
				resp, err := ss.ProxyRead()
				if err != nil {
					fmt.Printf("backend server read error:%v\n", err)
					server.Broken = true
					return
				}
				err = client.ProxyWrite(resp)
				if err != nil {
					fmt.Println("clent error")
					server.Broken = true
					server.Release()
				}
			}

		}

	}

}

func init() {
	shards = NewShards()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

type Proxy struct {
	config           *config
	contentTypeRegex *regexp.Regexp
	s3               *s3
}

func newProxy(config *config, s3 *s3) *Proxy {
	// Todo, add config error checking

	contentTypeRegex, err := regexp.Compile(config.AllowedContentTypes)
	if err != nil {
		log.Panic(err)
	}

	return &Proxy{
		config:           config,
		contentTypeRegex: contentTypeRegex,
		s3:               s3,
	}
}

func (p Proxy) handler(w http.ResponseWriter, r *http.Request) {
	params := r.URL.Query()

	if err := p.validRequest(params); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
	} else {
		p.proxyRequest(w, params)
	}
}

func (p Proxy) validRequest(params url.Values) error {
	url := params.Get("url")
	path := params.Get("path")
	if url == "" && path == "" {
		return errors.New("No request url or path to proxy")
	}

	return nil
}

func (p Proxy) proxyRequest(w http.ResponseWriter, params url.Values) {
	// TODO, handle missing (or zero) query params
	widthInt, _ := strconv.Atoi(params.Get("width"))
	heightInt, _ := strconv.Atoi(params.Get("height"))

	if path := params.Get("path"); path != "" {
		cachedPath := strings.ToLower(
			fmt.Sprintf("front/%dx%d/%s", widthInt, heightInt, path))

		if body, err := p.s3.read(cachedPath); err == nil {
			// S3 cache hit, return cached body
			log.Printf("[HIT] %s", cachedPath)
			io.Copy(w, body)
		} else {
			// S3 cache miss, get path from S3, resize, return (and write to cache)
			log.Printf("[MISS] %s", cachedPath)
			body, err := p.s3.read(path)
			if err != nil {
				http.Error(w, err.Error(), http.StatusNotFound)
				return
			}
			defer body.Close()

			img, _ := newImg(body)
			img.resize(widthInt, heightInt)

			// Write the resized image (as jpeg) to the http.ResponseWriter
			img.write(w)
			// Write the resized image to s3 cache
			// TODO, handle writing errors?
			go p.s3.write(cachedPath, img)
		}
	} else if url := params.Get("url"); url != "" {
		resp, err := http.Get(url) // http.Get follows up to 10 redirects
		if err != nil {
			http.Error(w, "Could not proxy", http.StatusInternalServerError)
			return
		}
		if resp.StatusCode != 200 {
			http.Error(w, "Could not proxy", resp.StatusCode)
			return
		}
		defer resp.Body.Close()

		if err := p.validResponse(resp); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		if widthInt > 0 && heightInt > 0 {
			img, _ := newImg(resp.Body)
			img.resize(widthInt, heightInt)
			// Write the resized image (as jpeg) to the http.ResponseWriter
			img.write(w)
		} else {
			// Just copy the upstream response to the http.ResponseWriter
			w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
			io.Copy(w, resp.Body)
		}
	}
}

func (p Proxy) validResponse(resp *http.Response) error {
	if !p.contentTypeRegex.MatchString(resp.Header.Get("Content-Type")) {
		return errors.New("Upstream content doesn't match configured allowedContentTypes")
	}

	return nil
}
<commit_msg>Handle image decoding errors, return 500 rather than panic-ing<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

type Proxy struct {
	config           *config
	contentTypeRegex *regexp.Regexp
	s3               *s3
}

func newProxy(config *config, s3 *s3) *Proxy {
	// Todo, add config error checking

	contentTypeRegex, err := regexp.Compile(config.AllowedContentTypes)
	if err != nil {
		log.Panic(err)
	}

	return &Proxy{
		config:           config,
		contentTypeRegex: contentTypeRegex,
		s3:               s3,
	}
}

func (p Proxy) handler(w http.ResponseWriter, r *http.Request) {
	params := r.URL.Query()

	if err := p.validRequest(params); err != nil {
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
	} else {
		p.proxyRequest(w, params)
	}
}

func (p Proxy) validRequest(params url.Values) error {
	url := params.Get("url")
	path := params.Get("path")
	if url == "" && path == "" {
		return errors.New("No request url or path to proxy")
	}

	return nil
}

func (p Proxy) proxyRequest(w http.ResponseWriter, params url.Values) {
	// TODO, handle missing (or zero) query params
	widthInt, _ := strconv.Atoi(params.Get("width"))
	heightInt, _ := strconv.Atoi(params.Get("height"))

	if path := params.Get("path"); path != "" {
		cachedPath := strings.ToLower(
			fmt.Sprintf("front/%dx%d/%s", widthInt, heightInt, path))

		if body, err := p.s3.read(cachedPath); err == nil {
			// S3 cache hit, return cached body
			log.Printf("[HIT] %s", cachedPath)
			io.Copy(w, body)
		} else {
			// S3 cache miss, get path from S3, resize, return (and write to cache)
			log.Printf("[MISS] %s", cachedPath)
			body, err := p.s3.read(path)
			if err != nil {
				http.Error(w, err.Error(), http.StatusNotFound)
				return
			}
			defer body.Close()

			img, err := newImg(body)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			img.resize(widthInt, heightInt)

			// Write the resized image (as jpeg) to the http.ResponseWriter
			img.write(w)
			// Write the resized image to s3 cache
			// TODO, handle writing errors?
			go p.s3.write(cachedPath, img)
		}
	} else if url := params.Get("url"); url != "" {
		resp, err := http.Get(url) // http.Get follows up to 10 redirects
		if err != nil {
			http.Error(w, "Could not proxy", http.StatusInternalServerError)
			return
		}
		if resp.StatusCode != 200 {
			http.Error(w, "Could not proxy", resp.StatusCode)
			return
		}
		defer resp.Body.Close()

		if err := p.validResponse(resp); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		if widthInt > 0 && heightInt > 0 {
			img, err := newImg(resp.Body)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}

			img.resize(widthInt, heightInt)
			// Write the resized image (as jpeg) to the http.ResponseWriter
			img.write(w)
		} else {
			// Just copy the upstream response to the http.ResponseWriter
			w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
			io.Copy(w, resp.Body)
		}
	}
}

func (p Proxy) validResponse(resp *http.Response) error {
	if !p.contentTypeRegex.MatchString(resp.Header.Get("Content-Type")) {
		return errors.New("Upstream content doesn't match configured allowedContentTypes")
	}

	return nil
}
<|endoftext|>


<commit_before>package requestwork

import (
	"context"
	"net/http"
	"net/url"
)

type job struct {
	ctx context.Context
	req *http.Request
	h   func(resp *http.Response, err error) error
	end chan error
}

type result struct {
	resp *http.Response
	err  error
}

const DefaultMaxIdleConnPerHost = 20

func New(threads int) *Worker {

	w := &Worker{
		jobQuene: make(chan *job),
		threads:  threads,
	}

	go w.start()
	return w

}

func NoProxyAllowed(request *http.Request) (*url.URL, error) {
	return nil, nil
}

type Worker struct {
	jobQuene chan *job
	threads  int
}

func (w *Worker) Execute(ctx context.Context, req *http.Request, h func(resp *http.Response, err error) error) (err error) {

	j := &job{ctx, req, h, make(chan error)}
	w.jobQuene <- j
	return <-j.end

}

func (w *Worker) run() {
	tr := &http.Transport{
		Proxy:               NoProxyAllowed,
		Dial:                Dial,
		MaxIdleConnsPerHost: w.threads * DefaultMaxIdleConnPerHost,
	}
	client := &http.Client{
		Transport: tr,
	}
	for j := range w.jobQuene {
		c := make(chan error, 1)
		go func() {
			c <- j.h(client.Do(j.req))
		}()
		select {
		case <-j.ctx.Done():
			tr.CancelRequest(j.req)
			j.end <- j.ctx.Err()
			close(j.end)
		case err := <-c:
			j.end <- err
			close(j.end)
		}
	}

}

func (w *Worker) start() {

	for i := 0; i < w.threads; i++ {
		go w.run()
	}

}
<commit_msg>fix requestwork vendor<commit_after>package requestwork

import (
	"context"
	"net/http"
	"net/url"
)

type job struct {
	ctx     context.Context
	req     *http.Request
	handler func(resp *http.Response, err error) error

	end chan error
}

type result struct {
	resp *http.Response
	err  error
}

const DefaultMaxIdleConnPerHost = 20

func New(threads int) *Worker {

	tr := &http.Transport{
		Proxy:               NoProxyAllowed,
		Dial:                Dial,
		MaxIdleConnsPerHost: threads * DefaultMaxIdleConnPerHost,
	}
	client := &http.Client{
		Transport: tr,
	}
	w := &Worker{
		jobQuene: make(chan *job),
		threads:  threads,
		tr:       tr,
		client:   client,
	}

	go w.start()
	return w

}

func NoProxyAllowed(request *http.Request) (*url.URL, error) {
	return nil, nil
}

type Worker struct {
	jobQuene chan *job
	threads  int
	tr       *http.Transport
	client   *http.Client
}

func (w *Worker) Execute(ctx context.Context, req *http.Request, h func(resp *http.Response, err error) error) (err error) {

	j := &job{ctx, req, h, make(chan error)}
	w.jobQuene <- j
	return <-j.end

}

func (w *Worker) run() {
	for j := range w.jobQuene {
		c := make(chan error, 1)
		go func() {
			c <- j.handler(w.client.Do(j.req))
		}()
		select {
		case <-j.ctx.Done():
			w.tr.CancelRequest(j.req)
			j.end <- j.ctx.Err()
			close(j.end)
		case err := <-c:
			j.end <- err
			close(j.end)
		}
	}

}

func (w *Worker) start() {

	for i := 0; i < w.threads; i++ {
		go w.run()
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

var (
	plugin_long = templates.LongDesc(`
		Provides utilities for interacting with plugins.

		Plugins provide extended functionality that is not part of the major command-line distribution.
		Please refer to the documentation and examples for more information about how write your own plugins.`)

	plugin_list_long = templates.LongDesc(`
		List all available plugin files on a user's PATH.

		Available plugin files are those that are:
		- executable
		- anywhere on the user's PATH
		- begin with "kubectl-"
`)

	ValidPluginFilenamePrefixes = []string{"kubectl"}
)

func NewCmdPlugin(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	cmd := &cobra.Command{
		Use: "plugin [flags]",
		DisableFlagsInUseLine: true,
		Short: i18n.T("Provides utilities for interacting with plugins."),
		Long:  plugin_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args)
		},
	}

	cmd.AddCommand(NewCmdPluginList(f, streams))
	return cmd
}

type PluginListOptions struct {
	Verifier PathVerifier
	NameOnly bool

	genericclioptions.IOStreams
}

// NewCmdPluginList provides a way to list all plugin executables visible to kubectl
func NewCmdPluginList(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := &PluginListOptions{
		IOStreams: streams,
	}

	cmd := &cobra.Command{
		Use:   "list",
		Short: "list all visible plugin executables on a user's PATH",
		Long:  plugin_list_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(cmd))
			cmdutil.CheckErr(o.Run())
		},
	}

	cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path")
	return cmd
}

func (o *PluginListOptions) Complete(cmd *cobra.Command) error {
	o.Verifier = &CommandOverrideVerifier{
		root:        cmd.Root(),
		seenPlugins: make(map[string]string, 0),
	}
	return nil
}

func (o *PluginListOptions) Run() error {
	pluginsFound := false
	isFirstFile := true
	pluginWarnings := 0
	paths := sets.NewString(filepath.SplitList(os.Getenv("PATH"))...)

	for _, dir := range paths.List() {
		files, err := ioutil.ReadDir(dir)
		if err != nil {
			continue
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) {
				continue
			}

			if isFirstFile {
				fmt.Fprintf(o.ErrOut, "The following compatible plugins are available:\n\n")
				pluginsFound = true
				isFirstFile = false
			}

			pluginPath := f.Name()
			if !o.NameOnly {
				pluginPath = filepath.Join(dir, pluginPath)
			}

			fmt.Fprintf(o.Out, "%s\n", pluginPath)
			if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 {
				for _, err := range errs {
					fmt.Fprintf(o.ErrOut, "  - %s\n", err)
					pluginWarnings++
				}
			}
		}
	}

	if !pluginsFound {
		return fmt.Errorf("error: unable to find any kubectl plugins in your PATH")
	}

	if pluginWarnings > 0 {
		fmt.Fprintln(o.ErrOut)
		if pluginWarnings == 1 {
			return fmt.Errorf("one plugin warning was found")
		}
		return fmt.Errorf("%v plugin warnings were found", pluginWarnings)
	}

	return nil
}

// pathVerifier receives a path and determines if it is valid or not
type PathVerifier interface {
	// Verify determines if a given path is valid
	Verify(path string) []error
}

type CommandOverrideVerifier struct {
	root        *cobra.Command
	seenPlugins map[string]string
}

// Verify implements PathVerifier and determines if a given path
// is valid depending on whether or not it overwrites an existing
// kubectl command path, or a previously seen plugin.
func (v *CommandOverrideVerifier) Verify(path string) []error {
	if v.root == nil {
		return []error{fmt.Errorf("unable to verify path with nil root")}
	}

	// extract the plugin binary name
	segs := strings.Split(path, "/")
	binName := segs[len(segs)-1]

	cmdPath := strings.Split(binName, "-")
	if len(cmdPath) > 1 {
		// the first argument is always "kubectl" for a plugin binary
		cmdPath = cmdPath[1:]
	}

	errors := []error{}

	if isExec, err := isExecutable(path); err == nil && !isExec {
		errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path))
	} else if err != nil {
		errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err))
	}

	if existingPath, ok := v.seenPlugins[binName]; ok {
		errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath))
	} else {
		v.seenPlugins[binName] = path
	}

	if cmd, _, err := v.root.Find(cmdPath); err == nil {
		errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath()))
	}

	return errors
}

func isExecutable(fullPath string) (bool, error) {
	info, err := os.Stat(fullPath)
	if err != nil {
		return false, err
	}

	if runtime.GOOS == "windows" {
		fileExt := strings.ToLower(filepath.Ext(fullPath))

		switch fileExt {
		case ".bat", ".cmd", ".com", ".exe":
			return true, nil
		}
		return false, nil
	}

	if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
		return true, nil
	}

	return false, nil
}

func hasValidPrefix(filepath string, validPrefixes []string) bool {
	for _, prefix := range validPrefixes {
		if !strings.HasPrefix(filepath, prefix+"-") {
			continue
		}
		return true
	}
	return false
}
<commit_msg>UPSTREAM: 73599: Add ps1 to windows executable extensions<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

var (
	plugin_long = templates.LongDesc(`
		Provides utilities for interacting with plugins.

		Plugins provide extended functionality that is not part of the major command-line distribution.
		Please refer to the documentation and examples for more information about how write your own plugins.`)

	plugin_list_long = templates.LongDesc(`
		List all available plugin files on a user's PATH.

		Available plugin files are those that are:
		- executable
		- anywhere on the user's PATH
		- begin with "kubectl-"
`)

	ValidPluginFilenamePrefixes = []string{"kubectl"}
)

func NewCmdPlugin(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	cmd := &cobra.Command{
		Use: "plugin [flags]",
		DisableFlagsInUseLine: true,
		Short: i18n.T("Provides utilities for interacting with plugins."),
		Long:  plugin_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.DefaultSubCommandRun(streams.ErrOut)(cmd, args)
		},
	}

	cmd.AddCommand(NewCmdPluginList(f, streams))
	return cmd
}

type PluginListOptions struct {
	Verifier PathVerifier
	NameOnly bool

	genericclioptions.IOStreams
}

// NewCmdPluginList provides a way to list all plugin executables visible to kubectl
func NewCmdPluginList(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := &PluginListOptions{
		IOStreams: streams,
	}

	cmd := &cobra.Command{
		Use:   "list",
		Short: "list all visible plugin executables on a user's PATH",
		Long:  plugin_list_long,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(cmd))
			cmdutil.CheckErr(o.Run())
		},
	}

	cmd.Flags().BoolVar(&o.NameOnly, "name-only", o.NameOnly, "If true, display only the binary name of each plugin, rather than its full path")
	return cmd
}

func (o *PluginListOptions) Complete(cmd *cobra.Command) error {
	o.Verifier = &CommandOverrideVerifier{
		root:        cmd.Root(),
		seenPlugins: make(map[string]string, 0),
	}
	return nil
}

func (o *PluginListOptions) Run() error {
	pluginsFound := false
	isFirstFile := true
	pluginWarnings := 0
	paths := sets.NewString(filepath.SplitList(os.Getenv("PATH"))...)

	for _, dir := range paths.List() {
		files, err := ioutil.ReadDir(dir)
		if err != nil {
			continue
		}

		for _, f := range files {
			if f.IsDir() {
				continue
			}
			if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) {
				continue
			}

			if isFirstFile {
				fmt.Fprintf(o.ErrOut, "The following compatible plugins are available:\n\n")
				pluginsFound = true
				isFirstFile = false
			}

			pluginPath := f.Name()
			if !o.NameOnly {
				pluginPath = filepath.Join(dir, pluginPath)
			}

			fmt.Fprintf(o.Out, "%s\n", pluginPath)
			if errs := o.Verifier.Verify(filepath.Join(dir, f.Name())); len(errs) != 0 {
				for _, err := range errs {
					fmt.Fprintf(o.ErrOut, "  - %s\n", err)
					pluginWarnings++
				}
			}
		}
	}

	if !pluginsFound {
		return fmt.Errorf("error: unable to find any kubectl plugins in your PATH")
	}

	if pluginWarnings > 0 {
		fmt.Fprintln(o.ErrOut)
		if pluginWarnings == 1 {
			return fmt.Errorf("one plugin warning was found")
		}
		return fmt.Errorf("%v plugin warnings were found", pluginWarnings)
	}

	return nil
}

// pathVerifier receives a path and determines if it is valid or not
type PathVerifier interface {
	// Verify determines if a given path is valid
	Verify(path string) []error
}

type CommandOverrideVerifier struct {
	root        *cobra.Command
	seenPlugins map[string]string
}

// Verify implements PathVerifier and determines if a given path
// is valid depending on whether or not it overwrites an existing
// kubectl command path, or a previously seen plugin.
func (v *CommandOverrideVerifier) Verify(path string) []error {
	if v.root == nil {
		return []error{fmt.Errorf("unable to verify path with nil root")}
	}

	// extract the plugin binary name
	segs := strings.Split(path, "/")
	binName := segs[len(segs)-1]

	cmdPath := strings.Split(binName, "-")
	if len(cmdPath) > 1 {
		// the first argument is always "kubectl" for a plugin binary
		cmdPath = cmdPath[1:]
	}

	errors := []error{}

	if isExec, err := isExecutable(path); err == nil && !isExec {
		errors = append(errors, fmt.Errorf("warning: %s identified as a kubectl plugin, but it is not executable", path))
	} else if err != nil {
		errors = append(errors, fmt.Errorf("error: unable to identify %s as an executable file: %v", path, err))
	}

	if existingPath, ok := v.seenPlugins[binName]; ok {
		errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath))
	} else {
		v.seenPlugins[binName] = path
	}

	if cmd, _, err := v.root.Find(cmdPath); err == nil {
		errors = append(errors, fmt.Errorf("warning: %s overwrites existing command: %q", binName, cmd.CommandPath()))
	}

	return errors
}

func isExecutable(fullPath string) (bool, error) {
	info, err := os.Stat(fullPath)
	if err != nil {
		return false, err
	}

	if runtime.GOOS == "windows" {
		fileExt := strings.ToLower(filepath.Ext(fullPath))

		switch fileExt {
		case ".bat", ".cmd", ".com", ".exe", ".ps1":
			return true, nil
		}
		return false, nil
	}

	if m := info.Mode(); !m.IsDir() && m&0111 != 0 {
		return true, nil
	}

	return false, nil
}

func hasValidPrefix(filepath string, validPrefixes []string) bool {
	for _, prefix := range validPrefixes {
		if !strings.HasPrefix(filepath, prefix+"-") {
			continue
		}
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

// Test integration of gkvlite with go-slab, using gkvlite's optional
// ItemValAddRef/DecRef() callbacks to integrate with a slab memory
// allocator.

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"testing"

	"github.com/steveyen/gkvlite"
	"github.com/steveyen/go-slab"
)

var maxOps = flag.Int("ops", 0,
	"max number of ops; 0 means run forever")
var maxItems = flag.Int("n", 10000,
	"max number of items")
var pctSets = flag.Int("sets", 50,
	"percentage of sets; 50 means 50% sets")
var pctDeletes = flag.Int("deletes", 5,
	"percentage of deletes; 5 means 5% deletes")
var pctEvicts = flag.Int("evicts", 5,
	"percentage of evicts; 3 means 3% evicts")
var pctReopens = flag.Int("reopens", 0,
	"percentage of reopens; 0 means 0% reopens")
var useSlab = flag.Bool("useSlab", true,
	"whether to use slab allocator")

func usage() {
	fmt.Fprintf(os.Stderr, "gkvlite slab testing tool\n")
	fmt.Fprintf(os.Stderr, "\nusage: %s [flags] dbFileName\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  dbFileName - file name used during test\n")
	fmt.Fprintf(os.Stderr, "\nexamples:\n")
	fmt.Fprintf(os.Stderr, "  ./slab -n 1000 -sets 30 test.tmp\n")
	fmt.Fprintf(os.Stderr, "\nflags:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "  -h: print this usage/help message\n")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		log.Fatalf("error: missing dbFileName")
	}
	if *pctSets + *pctDeletes + *pctEvicts + *pctReopens > 100 {
		log.Fatalf("error: percentages are > 100%% (%d%%+%d%%+%d%%+%d%%)",
			*pctSets, *pctDeletes, *pctEvicts + *pctReopens)
	}
	run(args[0], *useSlab, *maxOps, *maxItems, *pctSets, *pctDeletes, *pctEvicts, *pctReopens)
}

// Helper function to read a contiguous byte sequence, splitting
// it up into chained buf's of maxBufSize.  The last buf in the
// chain can have length <= maxBufSize.
func readBufChain(arena *slab.Arena, maxBufSize int, r io.ReaderAt, offset int64,
	valLength uint32) ([]byte, error) {
	n := int(valLength)
	if n > maxBufSize {
		n = maxBufSize
	}
	b := arena.Alloc(n)
	_, err := r.ReadAt(b, offset)
	if err != nil {
		arena.DecRef(b)
		return nil, err
	}
	remaining := valLength - uint32(n)
	if remaining > 0 {
		next, err := readBufChain(arena, maxBufSize, r, offset + int64(n), remaining)
		if err != nil {
			arena.DecRef(b)
			return nil, err
		}
		arena.SetNext(b, next)
	}
	return b, nil
}

func setupStoreArena(t *testing.T, maxBufSize int) (
	*slab.Arena, gkvlite.StoreCallbacks) {
	arena := slab.NewArena(48, // The smallest slab class "chunk size" is 48 bytes.
		1024*1024, // Each slab will be 1MB in size.
		2,         // Power of 2 growth in "chunk sizes".
		nil)       // Use default make([]byte) for slab memory.
	if arena == nil {
		t.Errorf("expected arena")
	}

	itemValLength := func(c *gkvlite.Collection, i *gkvlite.Item) int {
		if i.Val == nil {
			if t != nil {
				t.Fatalf("itemValLength on nil i.Val, i: %#v", i)
			} else {
				panic(fmt.Sprintf("itemValLength on nil i.Val, i: %#v", i))
			}
		}
		s := 0
		b := i.Val
		for b != nil {
			s = s + len(b)
			n := arena.GetNext(b)
			if s > len(b) {
				arena.DecRef(b)
			}
			b = n
		}
		return s
	}
	itemValWrite := func(c *gkvlite.Collection,
		i *gkvlite.Item, w io.WriterAt, offset int64) error {
		s := 0
		b := i.Val
		for b != nil {
			_, err := w.WriteAt(b, offset + int64(s))
			if err != nil {
				return err
			}
			s = s + len(b)
			n := arena.GetNext(b)
			if s > len(b) {
				arena.DecRef(b)
			}
			b = n
		}
		return nil
	}
	itemValRead := func(c *gkvlite.Collection,
		i *gkvlite.Item, r io.ReaderAt, offset int64, valLength uint32) error {
		b, err := readBufChain(arena, maxBufSize, r, offset, valLength)
		if err != nil {
			return err
		}
		i.Val = b
		return nil
	}
	itemValAddRef := func(c *gkvlite.Collection, i *gkvlite.Item) {
		if i.Val == nil {
			return
		}
		arena.AddRef(i.Val)
	}
	itemValDecRef := func(c *gkvlite.Collection, i *gkvlite.Item) {
		if i.Val == nil {
			return
		}
		arena.DecRef(i.Val)
	}

	scb := gkvlite.StoreCallbacks{
		ItemValLength: itemValLength,
		ItemValWrite:  itemValWrite,
		ItemValRead:   itemValRead,
		ItemValAddRef: itemValAddRef,
		ItemValDecRef: itemValDecRef,
	}
	return arena, scb
}

func run(fname string, useSlab bool,
	maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens int) {
	os.Remove(fname)
	f, err := os.Create(fname)
	if err != nil {
		panic(fmt.Sprintf("error: could not create file: %s, err: %v", fname, err))
	}

	arena, scb := setupStoreArena(nil, 256)
	if !useSlab {
		arena = nil
		scb = gkvlite.StoreCallbacks{
			ItemValLength: func(c *gkvlite.Collection, i *gkvlite.Item) int {
				return len(i.Val)
			},
		}
	}

	start := func(f *os.File) (*gkvlite.Store, *gkvlite.Collection) {
		s, err := gkvlite.NewStoreEx(f, scb)
		if err != nil || s == nil {
			panic(fmt.Sprintf("error: expected NewStoreEx to work, err: %v", err))
		}
		x := s.SetCollection("x", bytes.Compare)
		if x == nil {
			panic(fmt.Sprintf("error: expected SetColl/GetColl to work"))
		}
		return s, x
	}

	s, x := start(f)

	stop := func() {
		s.Flush()
		s.Close()
		f.Close()
		f = nil
	}

	numGets := 0
	numSets := 0
	numDeletes := 0
	numEvicts := 0
	numReopens := 0

	psd := pctSets + pctDeletes
	psde := pctSets + pctDeletes + pctEvicts
	psder := pctSets + pctDeletes + pctEvicts + pctReopens

	for i := 0; maxOps <= 0 || i < maxOps; i++ {
		kr := rand.Int() % maxItems
		kr4 := kr * kr * kr * kr
		if kr4 > 1000000 {
			kr4 = 1000000
		}
		ks := fmt.Sprintf("%03d", kr)
		k := []byte(ks)
		r := rand.Int() % 100
		if r < pctSets {
			numSets++
			var b []byte
			if arena != nil {
				b = arena.Alloc(kr4)
			} else {
				b = make([]byte, kr4)
			}
			pri := rand.Int31()
			err := x.SetItem(&gkvlite.Item{
				Key:      k,
				Val:      b,
				Priority: pri,
			})
			if err != nil {
				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
			}
			if arena != nil {
				arena.DecRef(b)
			}
		} else if r < psd {
			numDeletes++
			_, err := x.Delete(k)
			if err != nil {
				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
			}
		} else if r < psde {
			numEvicts++
			x.EvictSomeItems()
		} else if r < psder {
			numReopens++
			stop()
			f, _ = os.OpenFile(fname, os.O_RDWR, 0666)
			s, x = start(f)
		} else {
			numGets++
			i, err := x.GetItem(k, true)
			if err != nil {
				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
			}
			if i != nil {
				if scb.ItemValLength(x, i) != kr4 {
					panic(fmt.Sprintf("error: expected len: %d, got %d",
						kr4, scb.ItemValLength(x, i)))
				}
				s.ItemValDecRef(x, i)
			}
		}

		if i % 10000 == 0 {
			log.Printf("i: %d, numGets: %d, numSets: %d, numDeletes: %d" +
				", numEvicts: %d, numReopens: %d\n",
				i, numGets, numSets, numDeletes, numEvicts, numReopens)
		}
	}
}
<commit_msg>Typo with evicts param.<commit_after>package main

// Test integration of gkvlite with go-slab, using gkvlite's optional
// ItemValAddRef/DecRef() callbacks to integrate with a slab memory
// allocator.

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"testing"

	"github.com/steveyen/gkvlite"
	"github.com/steveyen/go-slab"
)

var maxOps = flag.Int("ops", 0,
	"max number of ops; 0 means run forever")
var maxItems = flag.Int("n", 10000,
	"max number of items")
var pctSets = flag.Int("sets", 50,
	"percentage of sets; 50 means 50% sets")
var pctDeletes = flag.Int("deletes", 5,
	"percentage of deletes; 5 means 5% deletes")
var pctEvicts = flag.Int("evicts", 3,
	"percentage of evicts; 3 means 3% evicts")
var pctReopens = flag.Int("reopens", 0,
	"percentage of reopens; 0 means 0% reopens")
var useSlab = flag.Bool("useSlab", true,
	"whether to use slab allocator")

func usage() {
	fmt.Fprintf(os.Stderr, "gkvlite slab testing tool\n")
	fmt.Fprintf(os.Stderr, "\nusage: %s [flags] dbFileName\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  dbFileName - file name used during test\n")
	fmt.Fprintf(os.Stderr, "\nexamples:\n")
	fmt.Fprintf(os.Stderr, "  ./slab -n 1000 -sets 30 test.tmp\n")
	fmt.Fprintf(os.Stderr, "\nflags:\n")
	flag.PrintDefaults()
	fmt.Fprintf(os.Stderr, "  -h: print this usage/help message\n")
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		log.Fatalf("error: missing dbFileName")
	}
	if *pctSets + *pctDeletes + *pctEvicts + *pctReopens > 100 {
		log.Fatalf("error: percentages are > 100%% (%d%%+%d%%+%d%%+%d%%)",
			*pctSets, *pctDeletes, *pctEvicts + *pctReopens)
	}
	run(args[0], *useSlab, *maxOps, *maxItems, *pctSets, *pctDeletes, *pctEvicts, *pctReopens)
}

// Helper function to read a contiguous byte sequence, splitting
// it up into chained buf's of maxBufSize.  The last buf in the
// chain can have length <= maxBufSize.
func readBufChain(arena *slab.Arena, maxBufSize int, r io.ReaderAt, offset int64,
	valLength uint32) ([]byte, error) {
	n := int(valLength)
	if n > maxBufSize {
		n = maxBufSize
	}
	b := arena.Alloc(n)
	_, err := r.ReadAt(b, offset)
	if err != nil {
		arena.DecRef(b)
		return nil, err
	}
	remaining := valLength - uint32(n)
	if remaining > 0 {
		next, err := readBufChain(arena, maxBufSize, r, offset + int64(n), remaining)
		if err != nil {
			arena.DecRef(b)
			return nil, err
		}
		arena.SetNext(b, next)
	}
	return b, nil
}

func setupStoreArena(t *testing.T, maxBufSize int) (
	*slab.Arena, gkvlite.StoreCallbacks) {
	arena := slab.NewArena(48, // The smallest slab class "chunk size" is 48 bytes.
		1024*1024, // Each slab will be 1MB in size.
		2,         // Power of 2 growth in "chunk sizes".
		nil)       // Use default make([]byte) for slab memory.
	if arena == nil {
		t.Errorf("expected arena")
	}

	itemValLength := func(c *gkvlite.Collection, i *gkvlite.Item) int {
		if i.Val == nil {
			if t != nil {
				t.Fatalf("itemValLength on nil i.Val, i: %#v", i)
			} else {
				panic(fmt.Sprintf("itemValLength on nil i.Val, i: %#v", i))
			}
		}
		s := 0
		b := i.Val
		for b != nil {
			s = s + len(b)
			n := arena.GetNext(b)
			if s > len(b) {
				arena.DecRef(b)
			}
			b = n
		}
		return s
	}
	itemValWrite := func(c *gkvlite.Collection,
		i *gkvlite.Item, w io.WriterAt, offset int64) error {
		s := 0
		b := i.Val
		for b != nil {
			_, err := w.WriteAt(b, offset + int64(s))
			if err != nil {
				return err
			}
			s = s + len(b)
			n := arena.GetNext(b)
			if s > len(b) {
				arena.DecRef(b)
			}
			b = n
		}
		return nil
	}
	itemValRead := func(c *gkvlite.Collection,
		i *gkvlite.Item, r io.ReaderAt, offset int64, valLength uint32) error {
		b, err := readBufChain(arena, maxBufSize, r, offset, valLength)
		if err != nil {
			return err
		}
		i.Val = b
		return nil
	}
	itemValAddRef := func(c *gkvlite.Collection, i *gkvlite.Item) {
		if i.Val == nil {
			return
		}
		arena.AddRef(i.Val)
	}
	itemValDecRef := func(c *gkvlite.Collection, i *gkvlite.Item) {
		if i.Val == nil {
			return
		}
		arena.DecRef(i.Val)
	}

	scb := gkvlite.StoreCallbacks{
		ItemValLength: itemValLength,
		ItemValWrite:  itemValWrite,
		ItemValRead:   itemValRead,
		ItemValAddRef: itemValAddRef,
		ItemValDecRef: itemValDecRef,
	}
	return arena, scb
}

func run(fname string, useSlab bool,
	maxOps, maxItems, pctSets, pctDeletes, pctEvicts, pctReopens int) {
	os.Remove(fname)
	f, err := os.Create(fname)
	if err != nil {
		panic(fmt.Sprintf("error: could not create file: %s, err: %v", fname, err))
	}

	arena, scb := setupStoreArena(nil, 256)
	if !useSlab {
		arena = nil
		scb = gkvlite.StoreCallbacks{
			ItemValLength: func(c *gkvlite.Collection, i *gkvlite.Item) int {
				return len(i.Val)
			},
		}
	}

	start := func(f *os.File) (*gkvlite.Store, *gkvlite.Collection) {
		s, err := gkvlite.NewStoreEx(f, scb)
		if err != nil || s == nil {
			panic(fmt.Sprintf("error: expected NewStoreEx to work, err: %v", err))
		}
		x := s.SetCollection("x", bytes.Compare)
		if x == nil {
			panic(fmt.Sprintf("error: expected SetColl/GetColl to work"))
		}
		return s, x
	}

	s, x := start(f)

	stop := func() {
		s.Flush()
		s.Close()
		f.Close()
		f = nil
	}

	numGets := 0
	numSets := 0
	numDeletes := 0
	numEvicts := 0
	numReopens := 0

	psd := pctSets + pctDeletes
	psde := pctSets + pctDeletes + pctEvicts
	psder := pctSets + pctDeletes + pctEvicts + pctReopens

	for i := 0; maxOps <= 0 || i < maxOps; i++ {
		kr := rand.Int() % maxItems
		kr4 := kr * kr * kr * kr
		if kr4 > 1000000 {
			kr4 = 1000000
		}
		ks := fmt.Sprintf("%03d", kr)
		k := []byte(ks)
		r := rand.Int() % 100
		if r < pctSets {
			numSets++
			var b []byte
			if arena != nil {
				b = arena.Alloc(kr4)
			} else {
				b = make([]byte, kr4)
			}
			pri := rand.Int31()
			err := x.SetItem(&gkvlite.Item{
				Key:      k,
				Val:      b,
				Priority: pri,
			})
			if err != nil {
				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
			}
			if arena != nil {
				arena.DecRef(b)
			}
		} else if r < psd {
			numDeletes++
			_, err := x.Delete(k)
			if err != nil {
				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
			}
		} else if r < psde {
			numEvicts++
			x.EvictSomeItems()
		} else if r < psder {
			numReopens++
			stop()
			f, _ = os.OpenFile(fname, os.O_RDWR, 0666)
			s, x = start(f)
		} else {
			numGets++
			i, err := x.GetItem(k, true)
			if err != nil {
				panic(fmt.Sprintf("error: expected nil error, got: %v", err))
			}
			if i != nil {
				if scb.ItemValLength(x, i) != kr4 {
					panic(fmt.Sprintf("error: expected len: %d, got %d",
						kr4, scb.ItemValLength(x, i)))
				}
				s.ItemValDecRef(x, i)
			}
		}

		if i % 10000 == 0 {
			log.Printf("i: %d, numGets: %d, numSets: %d, numDeletes: %d" +
				", numEvicts: %d, numReopens: %d\n",
				i, numGets, numSets, numDeletes, numEvicts, numReopens)
		}
	}
}
<|endoftext|>


<commit_before>package bettermail

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"appengine"
	"appengine/mail"
)

var templates map[string]*Template

func init() {
	templates = loadTemplates()

	http.HandleFunc("/hook", hookHandler)
	http.HandleFunc("/hook-test-harness", hookTestHarnessHandler)
	http.HandleFunc("/test-mail-send", testMailSendHandler)
	http.HandleFunc("/_ah/bounce", bounceHandler)
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	eventType := r.Header.Get("X-Github-Event")
	message, err := handlePayload(eventType, r.Body, c)
	if err != nil {
		c.Errorf("Error %s handling %s payload", err, eventType)
		http.Error(w, "Error handling payload", http.StatusInternalServerError)
		return
	}
	if message == nil {
		fmt.Fprint(w, "Unhandled event type: %s", eventType)
		c.Warningf("Unhandled event type: %s", eventType)
		return
	}
	err = mail.Send(c, message)
	if err != nil {
		c.Errorf("Could not send mail: %s", err)
		http.Error(w, "Could not send mail", http.StatusInternalServerError)
		return
	}
	c.Infof("Sent mail to %s", message.To[0])
	fmt.Fprint(w, "OK")
}

func handlePayload(eventType string, payloadReader io.Reader, c appengine.Context) (*mail.Message, error) {
	decoder := json.NewDecoder(payloadReader)
	if eventType == "push" {
		var payload PushPayload
		err := decoder.Decode(&payload)
		if err != nil {
			return nil, err
		}
		return handlePushPayload(payload, c)
	}
	return nil, nil
}

func handlePushPayload(payload PushPayload, c appengine.Context) (*mail.Message, error) {
	// TODO: allow location to be customized
	location, _ := time.LoadLocation("America/Los_Angeles")

	displayCommits := make([]DisplayCommit, 0)
	for i := range payload.Commits {
		displayCommits = append(displayCommits, newDisplayCommit(&payload.Commits[i], payload.Sender, payload.Repo, location, c))
	}
	branchName := (*payload.Ref)[11:]
	branchUrl := fmt.Sprintf("https://github.com/%s/tree/%s", *payload.Repo.FullName, branchName)
	pushedDate := payload.Repo.PushedAt.In(location)
	// Last link is a link so that the GitHub Gmail extension
	// (https://github.com/muan/github-gmail) will open the diff view.
	extensionUrl := displayCommits[0].URL
	if len(displayCommits) > 1 {
		extensionUrl = *payload.Compare
	}
	var data = map[string]interface{}{
		"Payload":                  payload,
		"Commits":                  displayCommits,
		"BranchName":               branchName,
		"BranchURL":                branchUrl,
		"PushedDisplayDate":        safeFormattedDate(pushedDate.Format(DisplayDateFormat)),
		"PushedDisplayDateTooltip": pushedDate.Format(DisplayDateFullFormat),
		"ExtensionURL":             extensionUrl,
	}
	var mailHtml bytes.Buffer
	if err := templates["push"].Execute(&mailHtml, data); err != nil {
		return nil, err
	}

	senderUserName := *payload.Pusher.Name
	senderName := senderUserName
	// We don't have the display name in the pusher, but usually it's one of the
	// commiters, so get it from there (without having to do any extra API
	// requests)
	for _, commit := range payload.Commits {
		if *commit.Author.Username == senderUserName {
			senderName = *commit.Author.Name
			break
		}
		if *commit.Committer.Username == senderUserName {
			senderName = *commit.Committer.Name
			break
		}
	}

	sender := fmt.Sprintf("%s <%s@better-github-mail.appspotmail.com>", senderName, senderUserName)
	subjectCommit := displayCommits[0]
	subject := fmt.Sprintf("[%s] %s: %s", *payload.Repo.FullName, subjectCommit.ShortSHA, subjectCommit.Title)

	recipient := "eng+commits@quip.com"
	if appengine.IsDevAppServer() {
		recipient = "mihai@quip.com"
	}

	message := &mail.Message{
		Sender:   sender,
		To:       []string{recipient},
		Bcc:      []string{"mihai@quip.com"},
		Subject:  subject,
		HTMLBody: mailHtml.String(),
	}
	return message, nil
}

func hookTestHarnessHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["hook-test-harness"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		eventType := r.FormValue("event_type")
		payload := r.FormValue("payload")
		c := appengine.NewContext(r)

		message, err := handlePayload(eventType, strings.NewReader(payload), c)
		var data = map[string]interface{}{
			"EventType":  eventType,
			"Payload":    payload,
			"Message":    message,
			"MessageErr": err,
		}
		templates["hook-test-harness"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}

func bounceHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if b, err := ioutil.ReadAll(r.Body); err == nil {
		c.Warningf("Bounce: %s", string(b))
	} else {
		c.Warningf("Bounce: <unreadable body>")
	}
}

func testMailSendHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["test-mail-send"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		c := appengine.NewContext(r)
		message := &mail.Message{
			Sender:   r.FormValue("sender"),
			To:       []string{"mihai@quip.com"},
			Subject:  r.FormValue("subject"),
			HTMLBody: r.FormValue("html_body"),
		}
		err := mail.Send(c, message)
		var data = map[string]interface{}{
			"Message": message,
			"SendErr": err,
		}
		templates["test-mail-send"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}
<commit_msg>Stop BCC-ing myself.<commit_after>package bettermail

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"appengine"
	"appengine/mail"
)

var templates map[string]*Template

func init() {
	templates = loadTemplates()

	http.HandleFunc("/hook", hookHandler)
	http.HandleFunc("/hook-test-harness", hookTestHarnessHandler)
	http.HandleFunc("/test-mail-send", testMailSendHandler)
	http.HandleFunc("/_ah/bounce", bounceHandler)
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	eventType := r.Header.Get("X-Github-Event")
	message, err := handlePayload(eventType, r.Body, c)
	if err != nil {
		c.Errorf("Error %s handling %s payload", err, eventType)
		http.Error(w, "Error handling payload", http.StatusInternalServerError)
		return
	}
	if message == nil {
		fmt.Fprint(w, "Unhandled event type: %s", eventType)
		c.Warningf("Unhandled event type: %s", eventType)
		return
	}
	err = mail.Send(c, message)
	if err != nil {
		c.Errorf("Could not send mail: %s", err)
		http.Error(w, "Could not send mail", http.StatusInternalServerError)
		return
	}
	c.Infof("Sent mail to %s", message.To[0])
	fmt.Fprint(w, "OK")
}

func handlePayload(eventType string, payloadReader io.Reader, c appengine.Context) (*mail.Message, error) {
	decoder := json.NewDecoder(payloadReader)
	if eventType == "push" {
		var payload PushPayload
		err := decoder.Decode(&payload)
		if err != nil {
			return nil, err
		}
		return handlePushPayload(payload, c)
	}
	return nil, nil
}

func handlePushPayload(payload PushPayload, c appengine.Context) (*mail.Message, error) {
	// TODO: allow location to be customized
	location, _ := time.LoadLocation("America/Los_Angeles")

	displayCommits := make([]DisplayCommit, 0)
	for i := range payload.Commits {
		displayCommits = append(displayCommits, newDisplayCommit(&payload.Commits[i], payload.Sender, payload.Repo, location, c))
	}
	branchName := (*payload.Ref)[11:]
	branchUrl := fmt.Sprintf("https://github.com/%s/tree/%s", *payload.Repo.FullName, branchName)
	pushedDate := payload.Repo.PushedAt.In(location)
	// Last link is a link so that the GitHub Gmail extension
	// (https://github.com/muan/github-gmail) will open the diff view.
	extensionUrl := displayCommits[0].URL
	if len(displayCommits) > 1 {
		extensionUrl = *payload.Compare
	}
	var data = map[string]interface{}{
		"Payload":                  payload,
		"Commits":                  displayCommits,
		"BranchName":               branchName,
		"BranchURL":                branchUrl,
		"PushedDisplayDate":        safeFormattedDate(pushedDate.Format(DisplayDateFormat)),
		"PushedDisplayDateTooltip": pushedDate.Format(DisplayDateFullFormat),
		"ExtensionURL":             extensionUrl,
	}
	var mailHtml bytes.Buffer
	if err := templates["push"].Execute(&mailHtml, data); err != nil {
		return nil, err
	}

	senderUserName := *payload.Pusher.Name
	senderName := senderUserName
	// We don't have the display name in the pusher, but usually it's one of the
	// commiters, so get it from there (without having to do any extra API
	// requests)
	for _, commit := range payload.Commits {
		if *commit.Author.Username == senderUserName {
			senderName = *commit.Author.Name
			break
		}
		if *commit.Committer.Username == senderUserName {
			senderName = *commit.Committer.Name
			break
		}
	}

	sender := fmt.Sprintf("%s <%s@better-github-mail.appspotmail.com>", senderName, senderUserName)
	subjectCommit := displayCommits[0]
	subject := fmt.Sprintf("[%s] %s: %s", *payload.Repo.FullName, subjectCommit.ShortSHA, subjectCommit.Title)

	recipient := "eng+commits@quip.com"
	if appengine.IsDevAppServer() {
		recipient = "mihai@quip.com"
	}

	message := &mail.Message{
		Sender:   sender,
		To:       []string{recipient},
		Subject:  subject,
		HTMLBody: mailHtml.String(),
	}
	return message, nil
}

func hookTestHarnessHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["hook-test-harness"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		eventType := r.FormValue("event_type")
		payload := r.FormValue("payload")
		c := appengine.NewContext(r)

		message, err := handlePayload(eventType, strings.NewReader(payload), c)
		var data = map[string]interface{}{
			"EventType":  eventType,
			"Payload":    payload,
			"Message":    message,
			"MessageErr": err,
		}
		templates["hook-test-harness"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}

func bounceHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if b, err := ioutil.ReadAll(r.Body); err == nil {
		c.Warningf("Bounce: %s", string(b))
	} else {
		c.Warningf("Bounce: <unreadable body>")
	}
}

func testMailSendHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["test-mail-send"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		c := appengine.NewContext(r)
		message := &mail.Message{
			Sender:   r.FormValue("sender"),
			To:       []string{"mihai@quip.com"},
			Subject:  r.FormValue("subject"),
			HTMLBody: r.FormValue("html_body"),
		}
		err := mail.Send(c, message)
		var data = map[string]interface{}{
			"Message": message,
			"SendErr": err,
		}
		templates["test-mail-send"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}
<|endoftext|>


<commit_before><commit_msg>Format code<commit_after><|endoftext|>


<commit_before>package presence

import (
	mongomodels "koding/db/models"
	"math/rand"
	"socialapi/models"
	"socialapi/workers/common/tests"
	"testing"
	"time"

	mgo "gopkg.in/mgo.v2"

	"github.com/koding/bongo"
	"github.com/koding/cache"
	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPresenceDailyOperations(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		groupName1 := models.RandomGroupName()
		Convey("With given presence data", t, func() {
			p1 := &models.PresenceDaily{
				AccountId: 1,
				GroupName: groupName1,
				// just to give some time between two records
				CreatedAt: time.Now().UTC().Add(-time.Millisecond * 100),
			}
			So(p1.Create(), ShouldBeNil)

			p2 := &models.PresenceDaily{
				AccountId: 1,
				GroupName: groupName1,
				CreatedAt: time.Now().UTC(),
			}
			So(p2.Create(), ShouldBeNil)

			Convey("getPresenceInfoFromDB should work properly", func() {
				pi, err := getPresenceInfoFromDB(&Ping{
					AccountID:     p2.AccountId,
					GroupName:     p2.GroupName,
					paymentStatus: mongomodels.PaymentStatusActive,
				})
				So(err, ShouldBeNil)
				So(pi, ShouldNotBeNil)
				// Why .Unix()? postgres omits after 6 decimal - due to our config.
				// Expected: '2016-09-21 13:44:50.695855774 +0000 UTC'
				// Actual:   '2016-09-21 13:44:50.695856 +0000 UTC'
				So(pi.CreatedAt.UTC().Unix(), ShouldEqual, p2.CreatedAt.UTC().Unix())

				pi2, err := getPresenceInfoFromDB(&Ping{
					AccountID:     p2.AccountId,
					GroupName:     "non_existent_group_name",
					paymentStatus: mongomodels.PaymentStatusActive,
				})
				So(err, ShouldNotBeNil)
				So(err, ShouldEqual, bongo.RecordNotFound)
				So(pi2, ShouldBeNil)

				pi3, err := getPresenceInfoFromDB(&Ping{
					AccountID:     rand.Int63(),
					GroupName:     groupName1,
					paymentStatus: mongomodels.PaymentStatusActive,
				})
				So(err, ShouldNotBeNil)
				So(err, ShouldEqual, bongo.RecordNotFound)
				So(pi3, ShouldBeNil)
			})
		})
	})
}

func TestPresenceDailyVerifyRecord(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		groupName1 := models.RandomGroupName()
		Convey("With given presence data", t, func() {

			Convey("should work properly with non existant data", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     1, // non existing user
					GroupName:     groupName1,
					CreatedAt:     today,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				key := getKey(ping, today)
				_, err := pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				err = verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// it should not be in cache
				_, err = pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
			})

			Convey("should work properly with old existing data", func() {
				today := time.Now().UTC()
				prev := today.Add(-time.Minute)
				ping := &Ping{
					AccountID:     2, // non existing user
					GroupName:     groupName1,
					CreatedAt:     prev,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				So(insertPresenceInfoToDB(ping), ShouldBeNil)

				ping.CreatedAt = today
				err := verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
				So(pd.CreatedAt.UTC().Unix(), ShouldEqual, today.UTC().Unix())
			})

			Convey("should work properly with old deleted data", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     3, // non existing user
					GroupName:     groupName1,
					CreatedAt:     today,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				So(insertPresenceInfoToDB(ping), ShouldBeNil)

				err := (&models.PresenceDaily{}).ProcessByGroupName(groupName1)
				So(err, ShouldBeNil)

				// just to by pass the check in verify
				err = verifyRecord(ping, ping.CreatedAt.Add(-time.Second))
				So(err, ShouldBeNil)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
				So(pd.CreatedAt.UTC().Unix(), ShouldEqual, today.UTC().Unix())
			})
		})
	})
}

func TestPresenceDailyPing(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		groupName1 := models.RandomGroupName()
		Convey("With given presence data", t, func() {

			Convey("should work properly with non existant data", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     1, // non existing user
					GroupName:     groupName1,
					CreatedAt:     today,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				key := getKey(ping, today)
				_, err := pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				err = verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// it should not be in cache
				_, err = pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
			})
		})
	})
}

func TestPresenceCancelledStatus(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		Convey("With given presence data", t, func() {
			account, _, groupSlug := models.CreateRandomGroupDataWithChecks()
			Convey("should work properly with invalid payment status", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     account.Id,
					GroupName:     groupSlug,
					CreatedAt:     today,
					paymentStatus: "invalid",
				}

				err := verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
				So(pd.IsProcessed, ShouldBeTrue)
			})
		})
	})
}

func TestPresenceGetGroupPaymentStatusFromCache(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		Convey("With non existing group", t, func() {
			groupName := models.RandomGroupName()
			Convey("should get err", func() {

				status, err := getGroupPaymentStatusFromCache(groupName)
				So(err, ShouldEqual, mgo.ErrNotFound)
				So(status, ShouldBeEmpty)

				Convey("With existing group", func() {
					_, _, groupSlug := models.CreateRandomGroupDataWithChecks()

					// make sure it is no in the cache
					_, err = groupCache.Get(groupSlug)
					So(err, ShouldEqual, cache.ErrNotFound)

					Convey("should work properly with invalid payment status", func() {
						status, err := getGroupPaymentStatusFromCache(groupSlug)
						So(err, ShouldBeNil)
						So(status, ShouldEqual, "invalid")
					})
				})
			})
		})
	})
}
<commit_msg>go/presence: time checking is difficult, sometimes it fails on wercker due to time diff<commit_after>package presence

import (
	mongomodels "koding/db/models"
	"math/rand"
	"socialapi/models"
	"socialapi/workers/common/tests"
	"testing"
	"time"

	mgo "gopkg.in/mgo.v2"

	"github.com/koding/bongo"
	"github.com/koding/cache"
	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPresenceDailyOperations(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		groupName1 := models.RandomGroupName()
		Convey("With given presence data", t, func() {
			p1 := &models.PresenceDaily{
				AccountId: 1,
				GroupName: groupName1,
				// just to give some time between two records
				CreatedAt: time.Now().UTC().Add(-time.Millisecond * 100),
			}
			So(p1.Create(), ShouldBeNil)

			p2 := &models.PresenceDaily{
				AccountId: 1,
				GroupName: groupName1,
				CreatedAt: time.Now().UTC(),
			}
			So(p2.Create(), ShouldBeNil)

			Convey("getPresenceInfoFromDB should work properly", func() {
				pi, err := getPresenceInfoFromDB(&Ping{
					AccountID:     p2.AccountId,
					GroupName:     p2.GroupName,
					paymentStatus: mongomodels.PaymentStatusActive,
				})
				So(err, ShouldBeNil)
				So(pi, ShouldNotBeNil)
				// Why .Unix()? postgres omits after 6 decimal - due to our config.
				// Expected: '2016-09-21 13:44:50.695855774 +0000 UTC'
				// Actual:   '2016-09-21 13:44:50.695856 +0000 UTC'
				So(pi.CreatedAt.UTC().Unix(), ShouldEqual, p2.CreatedAt.UTC().Unix())

				pi2, err := getPresenceInfoFromDB(&Ping{
					AccountID:     p2.AccountId,
					GroupName:     "non_existent_group_name",
					paymentStatus: mongomodels.PaymentStatusActive,
				})
				So(err, ShouldNotBeNil)
				So(err, ShouldEqual, bongo.RecordNotFound)
				So(pi2, ShouldBeNil)

				pi3, err := getPresenceInfoFromDB(&Ping{
					AccountID:     rand.Int63(),
					GroupName:     groupName1,
					paymentStatus: mongomodels.PaymentStatusActive,
				})
				So(err, ShouldNotBeNil)
				So(err, ShouldEqual, bongo.RecordNotFound)
				So(pi3, ShouldBeNil)
			})
		})
	})
}

func TestPresenceDailyVerifyRecord(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		groupName1 := models.RandomGroupName()
		Convey("With given presence data", t, func() {

			Convey("should work properly with non existant data", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     1, // non existing user
					GroupName:     groupName1,
					CreatedAt:     today,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				key := getKey(ping, today)
				_, err := pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				err = verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// it should not be in cache
				_, err = pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
			})

			Convey("should work properly with old existing data", func() {
				today := time.Now().UTC()
				prev := today.Add(-time.Minute)
				ping := &Ping{
					AccountID:     2, // non existing user
					GroupName:     groupName1,
					CreatedAt:     prev,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				So(insertPresenceInfoToDB(ping), ShouldBeNil)

				ping.CreatedAt = today
				err := verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
				So(pd.CreatedAt.UTC().Unix(), ShouldEqual, today.UTC().Unix())
			})

			Convey("should work properly with old deleted data", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     3, // non existing user
					GroupName:     groupName1,
					CreatedAt:     today,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				So(insertPresenceInfoToDB(ping), ShouldBeNil)

				err := (&models.PresenceDaily{}).ProcessByGroupName(groupName1)
				So(err, ShouldBeNil)

				// just to by pass the check in verify
				err = verifyRecord(ping, ping.CreatedAt.Add(-time.Second))
				So(err, ShouldBeNil)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
				So(pd.CreatedAt.UTC().Unix(), ShouldBeLessThanOrEqualTo, today.UTC().Unix())
			})
		})
	})
}

func TestPresenceDailyPing(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		groupName1 := models.RandomGroupName()
		Convey("With given presence data", t, func() {

			Convey("should work properly with non existant data", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     1, // non existing user
					GroupName:     groupName1,
					CreatedAt:     today,
					paymentStatus: mongomodels.PaymentStatusActive,
				}
				key := getKey(ping, today)
				_, err := pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				err = verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// it should not be in cache
				_, err = pingCache.Get(key)
				So(err, ShouldEqual, cache.ErrNotFound)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
			})
		})
	})
}

func TestPresenceCancelledStatus(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		Convey("With given presence data", t, func() {
			account, _, groupSlug := models.CreateRandomGroupDataWithChecks()
			Convey("should work properly with invalid payment status", func() {
				today := time.Now().UTC()
				ping := &Ping{
					AccountID:     account.Id,
					GroupName:     groupSlug,
					CreatedAt:     today,
					paymentStatus: "invalid",
				}

				err := verifyRecord(ping, today)
				So(err, ShouldBeNil)

				// we should be able to get it from db
				pd, err := getPresenceInfoFromDB(ping)
				So(err, ShouldBeNil)
				So(pd, ShouldNotBeNil)
				So(pd.IsProcessed, ShouldBeTrue)
			})
		})
	})
}

func TestPresenceGetGroupPaymentStatusFromCache(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {
		Convey("With non existing group", t, func() {
			groupName := models.RandomGroupName()
			Convey("should get err", func() {

				status, err := getGroupPaymentStatusFromCache(groupName)
				So(err, ShouldEqual, mgo.ErrNotFound)
				So(status, ShouldBeEmpty)

				Convey("With existing group", func() {
					_, _, groupSlug := models.CreateRandomGroupDataWithChecks()

					// make sure it is no in the cache
					_, err = groupCache.Get(groupSlug)
					So(err, ShouldEqual, cache.ErrNotFound)

					Convey("should work properly with invalid payment status", func() {
						status, err := getGroupPaymentStatusFromCache(groupSlug)
						So(err, ShouldBeNil)
						So(status, ShouldEqual, "invalid")
					})
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"veyron.io/veyron/veyron2"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/security"

	"test_service"
)

type testServiceDispatcher struct {
	cache           interface{}
	errorThrower    interface{}
	cancelCollector interface{}
}

func (sd *testServiceDispatcher) Lookup(suffix, method string) (ipc.Invoker, security.Authorizer, error) {
	if strings.HasPrefix(suffix, "cache") {
		return ipc.ReflectInvoker(sd.cache), nil, nil
	}

	if strings.HasPrefix(suffix, "errorThrower") {
		return ipc.ReflectInvoker(sd.errorThrower), nil, nil
	}

	if strings.HasPrefix(suffix, "cancel") {
		return ipc.ReflectInvoker(sd.cancelCollector), nil, nil
	}

	return ipc.ReflectInvoker(sd.cache), nil, nil
}

func StartServer(r veyron2.Runtime) (ipc.Server, naming.Endpoint, error) {
	// Create a new server instance.
	s, err := r.NewServer()
	if err != nil {
		return nil, nil, fmt.Errorf("failure creating server: %v", err)
	}

	disp := &testServiceDispatcher{
		cache:           test_service.NewServerCache(NewCached()),
		errorThrower:    test_service.NewServerErrorThrower(NewErrorThrower()),
		cancelCollector: test_service.NewServerCancelCollector(NewCancelCollector()),
	}

	// Create an endpoint and begin listening.
	endpoint, err := s.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, nil, fmt.Errorf("error listening to service: %v", err)
	}

	// Publish the services. This will register them in the mount table and
	// maintain the registration until StopServing is called.
	if err := s.Serve("test_service", disp); err != nil {
		return nil, nil, fmt.Errorf("error publishing service '%s': %v", err)
	}

	return s, endpoint, nil
}
<commit_msg>Fixed listen call.<commit_after>package main

import (
	"fmt"
	"strings"

	"veyron.io/veyron/veyron/profiles"
	"veyron.io/veyron/veyron2"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/naming"
	"veyron.io/veyron/veyron2/security"

	"test_service"
)

type testServiceDispatcher struct {
	cache           interface{}
	errorThrower    interface{}
	cancelCollector interface{}
}

func (sd *testServiceDispatcher) Lookup(suffix, method string) (ipc.Invoker, security.Authorizer, error) {
	if strings.HasPrefix(suffix, "cache") {
		return ipc.ReflectInvoker(sd.cache), nil, nil
	}

	if strings.HasPrefix(suffix, "errorThrower") {
		return ipc.ReflectInvoker(sd.errorThrower), nil, nil
	}

	if strings.HasPrefix(suffix, "cancel") {
		return ipc.ReflectInvoker(sd.cancelCollector), nil, nil
	}

	return ipc.ReflectInvoker(sd.cache), nil, nil
}

func StartServer(r veyron2.Runtime) (ipc.Server, naming.Endpoint, error) {
	// Create a new server instance.
	s, err := r.NewServer()
	if err != nil {
		return nil, nil, fmt.Errorf("failure creating server: %v", err)
	}

	disp := &testServiceDispatcher{
		cache:           test_service.NewServerCache(NewCached()),
		errorThrower:    test_service.NewServerErrorThrower(NewErrorThrower()),
		cancelCollector: test_service.NewServerCancelCollector(NewCancelCollector()),
	}

	// Create an endpoint and begin listening.
	endpoint, err := s.Listen(profiles.LocalListenSpec)
	if err != nil {
		return nil, nil, fmt.Errorf("error listening to service: %v", err)
	}

	// Publish the services. This will register them in the mount table and
	// maintain the registration until StopServing is called.
	if err := s.Serve("test_service", disp); err != nil {
		return nil, nil, fmt.Errorf("error publishing service '%s': %v", err)
	}

	return s, endpoint, nil
}
<|endoftext|>


<commit_before>/**
Copyright (c) 2016 The ConnectorDB Contributors
Licensed under the MIT license.
**/
package util

import (
	"os"
	"os/signal"
	"runtime"
	"runtime/debug"
	"sync"
	"syscall"

	log "github.com/Sirupsen/logrus"
)

var (
	closeMutex  = sync.Mutex{}
	closeExiter = sync.Once{}
	closers     = []Closeable{}
	closeWaiter = sync.WaitGroup{}
)

//CloseCall calls a custom function on close
type CloseCall struct {
	Callme func()
}

//Close calls the function
func (c CloseCall) Close() {
	c.Callme()
}

//Closeable is anything that can be closed
type Closeable interface {
	Close()
}

// CloseOnExit closes a resource when the program is exiting.
func CloseOnExit(closeable Closeable) {
	closeExiter.Do(setupCloseOnExit)

	closeMutex.Lock()
	closers = append(closers, closeable)
	closeMutex.Unlock()
}

// SetupCloseOnExit sets up the close on exit code
func setupCloseOnExit() {
	c := make(chan os.Signal, 3)

	if runtime.GOOS == "windows" {
		signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
	} else {
		signal.Notify(c, os.Interrupt)
	}

	go func() {
		for {
			s := <-c
			switch s {
			case syscall.SIGINT, syscall.SIGTERM, os.Interrupt:
				log.Warn("Exiting...")
				closeMutex.Lock()
				closeWaiter.Add(len(closers))
				log.Debugf("Running %d cleanup tasks", len(closers))
				for i := range closers {
					curnum := i

					go func() {
						defer func() {
							if r := recover(); r != nil {
								log.Errorf("Cleanup task panic: %s", debug.Stack())
							}
						}()
						closers[curnum].Close()
						log.Debugf("Done with cleanup task #%d", curnum+1)
						closeWaiter.Done()
					}()

				}
				closeMutex.Unlock()
				closeWaiter.Wait()
				log.Warn("bye!")
				os.Exit(0)
			case syscall.SIGHUP:
				log.Warn("Caught SIGHUP - ignoring")
			}
		}

	}()
}
<commit_msg>Fixed signal handler OS choice<commit_after>/**
Copyright (c) 2016 The ConnectorDB Contributors
Licensed under the MIT license.
**/
package util

import (
	"os"
	"os/signal"
	"runtime"
	"runtime/debug"
	"sync"
	"syscall"

	log "github.com/Sirupsen/logrus"
)

var (
	closeMutex  = sync.Mutex{}
	closeExiter = sync.Once{}
	closers     = []Closeable{}
	closeWaiter = sync.WaitGroup{}
)

//CloseCall calls a custom function on close
type CloseCall struct {
	Callme func()
}

//Close calls the function
func (c CloseCall) Close() {
	c.Callme()
}

//Closeable is anything that can be closed
type Closeable interface {
	Close()
}

// CloseOnExit closes a resource when the program is exiting.
func CloseOnExit(closeable Closeable) {
	closeExiter.Do(setupCloseOnExit)

	closeMutex.Lock()
	closers = append(closers, closeable)
	closeMutex.Unlock()
}

// SetupCloseOnExit sets up the close on exit code
func setupCloseOnExit() {
	c := make(chan os.Signal, 3)

	if runtime.GOOS != "windows" {
		signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP)
	} else {
		signal.Notify(c, os.Interrupt)
	}

	go func() {
		for {
			s := <-c
			switch s {
			case syscall.SIGTERM, os.Interrupt:
				log.Warn("Exiting...")
				closeMutex.Lock()
				closeWaiter.Add(len(closers))
				log.Debugf("Running %d cleanup tasks", len(closers))
				for i := range closers {
					curnum := i

					go func() {
						defer func() {
							if r := recover(); r != nil {
								log.Errorf("Cleanup task panic: %s", debug.Stack())
							}
						}()
						closers[curnum].Close()
						log.Debugf("Done with cleanup task #%d", curnum+1)
						closeWaiter.Done()
					}()

				}
				closeMutex.Unlock()
				closeWaiter.Wait()
				log.Warn("bye!")
				os.Exit(0)
			case syscall.SIGHUP:
				log.Warn("Caught SIGHUP - ignoring")
			}
		}

	}()
}
<|endoftext|>