<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"github.com/simonz05/godis"
)

const HELP string = `
<!DOCTYPE html>

        <title>cdrv.ws</title>

cdrvws(1)                          CDRV.WS                          cdrvws(1)

NAME
    cdrvws: command line url shortener:

SYNOPSIS
    <command> | curl -F 'rvw=<-' http://cdrv.ws

`

const CHARS string = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const BASE uint64 = uint64(len(CHARS))

var redis *godis.Client

func main() {
	connectToRedis()
	http.HandleFunc("/", route)
	startServer()
}

func encode(id uint64) string {
	encoded := ""
	for id > 0 {
		encoded += string(CHARS[id%BASE])
		id = id / BASE
	}
	return encoded
}

func createShortUrl(longurl string) (error, string) {
	shortid, err := redis.Incr("urlId")
	if err != nil {
		return err, ""
	}
	shorturl := encode(uint64(shortid))
	if err := redis.Set(shorturl, longurl); err != nil {
		return err, ""
	}
	return nil, shorturl
}

func expand(shorturl string) (error, string) {
	longurl, err := redis.Get(shorturl)
	if err != nil && err.Error() == "Nonexisting key" {
		return nil, ""
	} else if err != nil {
		return err, ""
	}
	return nil, longurl.String()
}

func connectToRedis() {
	rawurl := os.Getenv("REDISTOGO_URL")
	log.Printf("Redis to go url: %s\n", rawurl)
	redisurl := url.URL{
		User: url.UserPassword("", ""),
	}
	if rawurl != "" {
		var err error
		_, err = (&redisurl).Parse(rawurl)
		if err != nil {
			log.Fatal("Could not parse redis url", err)
		}
	}
	password, _ := redisurl.User.Password()
	log.Printf("Connecting to redis: %s\n", redisurl.String())
	redis = godis.New(redisurl.Host, 0, password)
}

func startServer() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	log.Printf("Starting on %s\n", port)
	err := http.ListenAndServe(":" + port, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func route(w http.ResponseWriter, req *http.Request) {
	if req.Method == "GET" {
		if req.URL.String() == "/" {
			handleHome(w, req)
		} else {
			handleExpand(w, req)
		}
	} else if req.Method == "POST" {
		handleShorten(w, req)
	}
}

func handleHome(w http.ResponseWriter, req *http.Request) {
	fmt.Fprintln(w, HELP)
}

func handleShorten(w http.ResponseWriter, req *http.Request) {
	err, shorturl := createShortUrl(req.FormValue("rvw"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fullurl := url.URL{
		Scheme: "http",
		Host:   req.Host,
		Path:   "/" + shorturl,
	}
	fmt.Fprintln(w, fullurl.String())
}

func handleExpand(w http.ResponseWriter, req *http.Request) {
	shorturl := strings.Trim(req.URL.String(), "/")
	err, longurl := expand(shorturl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else if longurl == "" {
		http.NotFound(w, req)
	} else {
		http.Redirect(w, req, longurl, http.StatusMovedPermanently)
	}
}
<commit_msg>use the returned url from parse<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"github.com/simonz05/godis"
)

const HELP string = `
<!DOCTYPE html>

        <title>cdrv.ws</title>

cdrvws(1)                          CDRV.WS                          cdrvws(1)

NAME
    cdrvws: command line url shortener:

SYNOPSIS
    <command> | curl -F 'rvw=<-' http://cdrv.ws

`

const CHARS string = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const BASE uint64 = uint64(len(CHARS))

var redis *godis.Client

func main() {
	connectToRedis()
	http.HandleFunc("/", route)
	startServer()
}

func encode(id uint64) string {
	encoded := ""
	for id > 0 {
		encoded += string(CHARS[id%BASE])
		id = id / BASE
	}
	return encoded
}

func createShortUrl(longurl string) (error, string) {
	shortid, err := redis.Incr("urlId")
	if err != nil {
		return err, ""
	}
	shorturl := encode(uint64(shortid))
	if err := redis.Set(shorturl, longurl); err != nil {
		return err, ""
	}
	return nil, shorturl
}

func expand(shorturl string) (error, string) {
	longurl, err := redis.Get(shorturl)
	if err != nil && err.Error() == "Nonexisting key" {
		return nil, ""
	} else if err != nil {
		return err, ""
	}
	return nil, longurl.String()
}

func connectToRedis() {
	rawurl := os.Getenv("REDISTOGO_URL")
	log.Printf("Redis to go url: %s\n", rawurl)
	redisurl := url.URL{
		User: url.UserPassword("", ""),
	}
	parsedurl := &redisurl
	if rawurl != "" {
		var err error
		parsedurl, err = parsedurl.Parse(rawurl)
		if err != nil {
			log.Fatal("Could not parse redis url", err)
		}
	}
	password, _ := parsedurl.User.Password()
	log.Printf("Connecting to redis: %s\n", parsedurl.String())
	redis = godis.New(parsedurl.Host, 0, password)
}

func startServer() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	log.Printf("Starting on %s\n", port)
	err := http.ListenAndServe(":" + port, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func route(w http.ResponseWriter, req *http.Request) {
	if req.Method == "GET" {
		if req.URL.String() == "/" {
			handleHome(w, req)
		} else {
			handleExpand(w, req)
		}
	} else if req.Method == "POST" {
		handleShorten(w, req)
	}
}

func handleHome(w http.ResponseWriter, req *http.Request) {
	fmt.Fprintln(w, HELP)
}

func handleShorten(w http.ResponseWriter, req *http.Request) {
	err, shorturl := createShortUrl(req.FormValue("rvw"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fullurl := url.URL{
		Scheme: "http",
		Host:   req.Host,
		Path:   "/" + shorturl,
	}
	fmt.Fprintln(w, fullurl.String())
}

func handleExpand(w http.ResponseWriter, req *http.Request) {
	shorturl := strings.Trim(req.URL.String(), "/")
	err, longurl := expand(shorturl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else if longurl == "" {
		http.NotFound(w, req)
	} else {
		http.Redirect(w, req, longurl, http.StatusMovedPermanently)
	}
}
<|endoftext|>


<commit_before>// Package shredis is a sharded redis client, with the idea of being used as a
// replacement for twemproxy/nutcracker.
//
// Commands are shared by key, and shredis handles the connection logic: you
// hand over the commands, and you checks the indivual command for errors and
// values.
// Commands are sent in as few packets as possible ('pipelined' in redis
// speak), even when they come from multiple goroutines.
//
package shredis

import (
	"sync"
	"time"
)

const (
	// connTimeout is the dial, read, and write timeout.
	connTimeout = 50 * time.Millisecond
)

// Shred controls all connections. Make one with New().
type Shred struct {
	ket       continuum
	conns     map[string]conn
	addrs     map[string]string // just for debugging
	onConnect []*Cmd
	connwg    sync.WaitGroup
}

// Option is an option to New.
type Option func(*Shred)

// OptionAuth is an option to New. It supports the redis AUTH command.
func OptionAuth(pw string) Option {
	return func(s *Shred) {
		s.onConnect = append(s.onConnect, Build("", "AUTH", pw))
	}
}

// New starts all connections to redis daemons.
func New(hosts map[string]string, options ...Option) *Shred {
	var (
		bs []bucket
	)
	s := &Shred{
		conns: map[string]conn{},
		addrs: hosts,
	}
	for _, o := range options {
		o(s)
	}

	for l, h := range hosts {
		bs = append(bs, bucket{Label: l, Weight: 1})

		s.connwg.Add(1)
		c := newConn()
		go func() {
			c.handle(h, s.onConnect)
			s.connwg.Done()
		}()
		s.conns[l] = c
	}
	s.ket = ketamaNew(bs)
	return s
}

// Close closes all connections. Blocks.
func (s *Shred) Close() {
	for _, c := range s.conns {
		c.close()
	}
	s.connwg.Wait()
}

// Exec is the way to execute commands. It is goroutinesafe.
func (s *Shred) Exec(cs ...*Cmd) {
	var (
		wg = sync.WaitGroup{}
		ac = map[conn][]action{}
	)

	// map every action to a connection, collect all actions per connection, and
	// execute them at the same time
	for _, c := range cs {
		wg.Add(1)
		conn := s.conn(c.key)
		ac[conn] = append(ac[conn], action{
			payload: c.payload,
			done: func(c *Cmd) actionCB {
				return func(res interface{}, err error) {
					c.res = res
					c.err = err
					wg.Done()
				}
			}(c),
		})
	}
	for c, vs := range ac {
		c.exec(vs)
	}

	wg.Wait()
}

func (s *Shred) conn(key []byte) conn {
	return s.conns[s.ket.Hash(string(key))]
}

// Addr gives the address for a key. For debugging/testing.
func (s *Shred) Addr(key string) string {
	return s.addrs[s.ket.Hash(key)]
}
<commit_msg>fix a race<commit_after>// Package shredis is a sharded redis client, with the idea of being used as a
// replacement for twemproxy/nutcracker.
//
// Commands are shared by key, and shredis handles the connection logic: you
// hand over the commands, and you checks the indivual command for errors and
// values.
// Commands are sent in as few packets as possible ('pipelined' in redis
// speak), even when they come from multiple goroutines.
//
package shredis

import (
	"sync"
	"time"
)

const (
	// connTimeout is the dial, read, and write timeout.
	connTimeout = 50 * time.Millisecond
)

// Shred controls all connections. Make one with New().
type Shred struct {
	ket       continuum
	conns     map[string]conn
	addrs     map[string]string // just for debugging
	onConnect []*Cmd
	connwg    sync.WaitGroup
}

// Option is an option to New.
type Option func(*Shred)

// OptionAuth is an option to New. It supports the redis AUTH command.
func OptionAuth(pw string) Option {
	return func(s *Shred) {
		s.onConnect = append(s.onConnect, Build("", "AUTH", pw))
	}
}

// New starts all connections to redis daemons.
func New(hosts map[string]string, options ...Option) *Shred {
	var (
		bs []bucket
	)
	s := &Shred{
		conns: map[string]conn{},
		addrs: hosts,
	}
	for _, o := range options {
		o(s)
	}

	for l, h := range hosts {
		bs = append(bs, bucket{Label: l, Weight: 1})

		s.connwg.Add(1)
		c := newConn()
		go func(h string) {
			c.handle(h, s.onConnect)
			s.connwg.Done()
		}(h)
		s.conns[l] = c
	}
	s.ket = ketamaNew(bs)
	return s
}

// Close closes all connections. Blocks.
func (s *Shred) Close() {
	for _, c := range s.conns {
		c.close()
	}
	s.connwg.Wait()
}

// Exec is the way to execute commands. It is goroutinesafe.
func (s *Shred) Exec(cs ...*Cmd) {
	var (
		wg = sync.WaitGroup{}
		ac = map[conn][]action{}
	)

	// map every action to a connection, collect all actions per connection, and
	// execute them at the same time
	for _, c := range cs {
		wg.Add(1)
		conn := s.conn(c.key)
		ac[conn] = append(ac[conn], action{
			payload: c.payload,
			done: func(c *Cmd) actionCB {
				return func(res interface{}, err error) {
					c.res = res
					c.err = err
					wg.Done()
				}
			}(c),
		})
	}
	for c, vs := range ac {
		c.exec(vs)
	}

	wg.Wait()
}

func (s *Shred) conn(key []byte) conn {
	return s.conns[s.ket.Hash(string(key))]
}

// Addr gives the address for a key. For debugging/testing.
func (s *Shred) Addr(key string) string {
	return s.addrs[s.ket.Hash(key)]
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloud9"
	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func TestAccAWSCloud9EnvironmentEc2_basic(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)
	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(envName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"instance_type"},
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(uEnvName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
		},
	})
}

func TestAccAWSCloud9EnvironmentEc2_allFields(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)
	description := fmt.Sprintf("Tf Acc Test %s", rString)
	uDescription := fmt.Sprintf("Tf Acc Test Updated %s", rString)
	userName := fmt.Sprintf("tf_acc_cloud9_env_%s", rString)
	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(envName, description, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"instance_type", "automatic_stop_time_minutes", "subnet_id"},
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(uEnvName, uDescription, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
		},
	})
}

func testAccCheckAWSCloud9EnvironmentEc2Exists(n string, res *cloud9.Environment) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Cloud9 Environment EC2 ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).cloud9conn

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
			}
			return err
		}
		if len(out.Environments) == 0 {
			return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
		}
		env := out.Environments[0]

		*res = *env

		return nil
	}
}

func testAccCheckAWSCloud9EnvironmentEc2Destroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_cloud9_environment_ec2" {
			continue
		}

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return nil
			}
			// :'-(
			if isAWSErr(err, "AccessDeniedException", "is not authorized to access this resource") {
				return nil
			}
			return err
		}
		if len(out.Environments) == 0 {
			return nil
		}

		return fmt.Errorf("Cloud9 Environment EC2 %q still exists.", rs.Primary.ID)
	}
	return nil
}

func testAccPreCheckAWSCloud9(t *testing.T) {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	input := &cloud9.ListEnvironmentsInput{}

	_, err := conn.ListEnvironments(input)

	if testAccPreCheckSkipError(err) {
		t.Skipf("skipping acceptance testing: %s", err)
	}

	if err != nil {
		t.Fatalf("unexpected PreCheck error: %s", err)
	}
}

func testAccAWSCloud9EnvironmentEc2Config(name string) string {
	return fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  instance_type = "t2.micro"
  name          = "%s"
}
`, name)
}

func testAccAWSCloud9EnvironmentEc2AllFieldsConfig(name, description, userName string) string {
	return fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  instance_type               = "t2.micro"
  name                        = "%s"
  description                 = "%s"
  automatic_stop_time_minutes = 60
  subnet_id                   = "${aws_subnet.test.id}"
  owner_arn                   = "${aws_iam_user.test.arn}"
  depends_on                  = ["aws_route_table_association.test"]
}

resource "aws_vpc" "test" {
  cidr_block = "10.10.0.0/16"

  tags = {
    Name = "terraform-testacc-cloud9-environment-ec2-all-fields"
  }
}

resource "aws_subnet" "test" {
  vpc_id     = "${aws_vpc.test.id}"
  cidr_block = "10.10.0.0/19"

  tags = {
    Name = "tf-acc-cloud9-environment-ec2-all-fields"
  }
}

resource "aws_internet_gateway" "test" {
  vpc_id = "${aws_vpc.test.id}"
}

resource "aws_route_table" "test" {
  vpc_id = "${aws_vpc.test.id}"

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = "${aws_internet_gateway.test.id}"
  }
}

resource "aws_route_table_association" "test" {
  subnet_id      = "${aws_subnet.test.id}"
  route_table_id = "${aws_route_table.test.id}"
}

resource "aws_iam_user" "test" {
  name = "%s"
}
`, name, description, userName)
}
<commit_msg>tests/resource/aws_cloud9_environment_ec2: Remove dependency on Default VPC and blacklist usw2-az4 (#11704)<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloud9"
	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func TestAccAWSCloud9EnvironmentEc2_basic(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)
	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(envName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"instance_type", "subnet_id"},
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2Config(uEnvName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
				),
			},
		},
	})
}

func TestAccAWSCloud9EnvironmentEc2_allFields(t *testing.T) {
	var conf cloud9.Environment

	rString := acctest.RandString(8)
	envName := fmt.Sprintf("tf_acc_env_basic_%s", rString)
	uEnvName := fmt.Sprintf("tf_acc_env_basic_updated_%s", rString)
	description := fmt.Sprintf("Tf Acc Test %s", rString)
	uDescription := fmt.Sprintf("Tf Acc Test Updated %s", rString)
	userName := fmt.Sprintf("tf_acc_cloud9_env_%s", rString)
	resourceName := "aws_cloud9_environment_ec2.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t); testAccPreCheckAWSCloud9(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloud9EnvironmentEc2Destroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(envName, description, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", envName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"instance_type", "automatic_stop_time_minutes", "subnet_id"},
			},
			{
				Config: testAccAWSCloud9EnvironmentEc2AllFieldsConfig(uEnvName, uDescription, userName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSCloud9EnvironmentEc2Exists(resourceName, &conf),
					resource.TestCheckResourceAttr(resourceName, "instance_type", "t2.micro"),
					resource.TestCheckResourceAttr(resourceName, "name", uEnvName),
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:[^:]+:cloud9:[^:]+:[^:]+:environment:.+$`)),
					resource.TestMatchResourceAttr(resourceName, "owner_arn", regexp.MustCompile(`^arn:`)),
					resource.TestCheckResourceAttr(resourceName, "type", "ec2"),
				),
			},
		},
	})
}

func testAccCheckAWSCloud9EnvironmentEc2Exists(n string, res *cloud9.Environment) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Cloud9 Environment EC2 ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).cloud9conn

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
			}
			return err
		}
		if len(out.Environments) == 0 {
			return fmt.Errorf("Cloud9 Environment EC2 (%q) not found", rs.Primary.ID)
		}
		env := out.Environments[0]

		*res = *env

		return nil
	}
}

func testAccCheckAWSCloud9EnvironmentEc2Destroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_cloud9_environment_ec2" {
			continue
		}

		out, err := conn.DescribeEnvironments(&cloud9.DescribeEnvironmentsInput{
			EnvironmentIds: []*string{aws.String(rs.Primary.ID)},
		})
		if err != nil {
			if isAWSErr(err, cloud9.ErrCodeNotFoundException, "") {
				return nil
			}
			// :'-(
			if isAWSErr(err, "AccessDeniedException", "is not authorized to access this resource") {
				return nil
			}
			return err
		}
		if len(out.Environments) == 0 {
			return nil
		}

		return fmt.Errorf("Cloud9 Environment EC2 %q still exists.", rs.Primary.ID)
	}
	return nil
}

func testAccPreCheckAWSCloud9(t *testing.T) {
	conn := testAccProvider.Meta().(*AWSClient).cloud9conn

	input := &cloud9.ListEnvironmentsInput{}

	_, err := conn.ListEnvironments(input)

	if testAccPreCheckSkipError(err) {
		t.Skipf("skipping acceptance testing: %s", err)
	}

	if err != nil {
		t.Fatalf("unexpected PreCheck error: %s", err)
	}
}

func testAccAWSCloud9EnvironmentEc2ConfigBase() string {
	return fmt.Sprintf(`
data "aws_availability_zones" "available" {
  # t2.micro instance type is not available in these Availability Zones
  blacklisted_zone_ids = ["usw2-az4"]
  state                = "available"
}

resource "aws_vpc" "test" {
  cidr_block = "10.0.0.0/16"

  tags = {
    Name = "tf-acc-test-cloud9-environment-ec2"
  }
}

resource "aws_subnet" "test" {
  availability_zone = data.aws_availability_zones.available.names[0]
  cidr_block        = "10.0.0.0/24"
  vpc_id            = aws_vpc.test.id

  tags = {
    Name = "tf-acc-test-cloud9-environment-ec2"
  }
}

resource "aws_internet_gateway" "test" {
  vpc_id = aws_vpc.test.id
}

resource "aws_route" "test" {
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.test.id
  route_table_id         = aws_vpc.test.main_route_table_id
}
`)
}

func testAccAWSCloud9EnvironmentEc2Config(name string) string {
	return testAccAWSCloud9EnvironmentEc2ConfigBase() + fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  depends_on = [aws_route.test]

  instance_type = "t2.micro"
  name          = %[1]q
  subnet_id     = aws_subnet.test.id
}
`, name)
}

func testAccAWSCloud9EnvironmentEc2AllFieldsConfig(name, description, userName string) string {
	return testAccAWSCloud9EnvironmentEc2ConfigBase() + fmt.Sprintf(`
resource "aws_cloud9_environment_ec2" "test" {
  depends_on = [aws_route.test]

  automatic_stop_time_minutes = 60
  description                 = %[2]q
  instance_type               = "t2.micro"
  name                        = %[1]q
  owner_arn                   = aws_iam_user.test.arn
  subnet_id                   = aws_subnet.test.id
}

resource "aws_iam_user" "test" {
  name = %[3]q
}
`, name, description, userName)
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/elasticache"
)

func resourceAwsElasticacheParameterGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElasticacheParameterGroupCreate,
		Read:   resourceAwsElasticacheParameterGroupRead,
		Update: resourceAwsElasticacheParameterGroupUpdate,
		Delete: resourceAwsElasticacheParameterGroupDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},
			"family": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "Managed by Terraform",
			},
			"parameter": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: false,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						"value": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
				Set: resourceAwsElasticacheParameterHash,
			},
		},
	}
}

func resourceAwsElasticacheParameterGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	createOpts := elasticache.CreateCacheParameterGroupInput{
		CacheParameterGroupName:   aws.String(d.Get("name").(string)),
		CacheParameterGroupFamily: aws.String(d.Get("family").(string)),
		Description:               aws.String(d.Get("description").(string)),
	}

	log.Printf("[DEBUG] Create Cache Parameter Group: %#v", createOpts)
	_, err := conn.CreateCacheParameterGroup(&createOpts)
	if err != nil {
		return fmt.Errorf("Error creating Cache Parameter Group: %s", err)
	}

	d.Partial(true)
	d.SetPartial("name")
	d.SetPartial("family")
	d.SetPartial("description")
	d.Partial(false)

	d.SetId(*createOpts.CacheParameterGroupName)
	log.Printf("[INFO] Cache Parameter Group ID: %s", d.Id())

	return resourceAwsElasticacheParameterGroupUpdate(d, meta)
}

func resourceAwsElasticacheParameterGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	describeOpts := elasticache.DescribeCacheParameterGroupsInput{
		CacheParameterGroupName: aws.String(d.Id()),
	}

	describeResp, err := conn.DescribeCacheParameterGroups(&describeOpts)
	if err != nil {
		return err
	}

	if len(describeResp.CacheParameterGroups) != 1 ||
		*describeResp.CacheParameterGroups[0].CacheParameterGroupName != d.Id() {
		return fmt.Errorf("Unable to find Parameter Group: %#v", describeResp.CacheParameterGroups)
	}

	d.Set("name", describeResp.CacheParameterGroups[0].CacheParameterGroupName)
	d.Set("family", describeResp.CacheParameterGroups[0].CacheParameterGroupFamily)
	d.Set("description", describeResp.CacheParameterGroups[0].Description)

	// Only include user customized parameters as there's hundreds of system/default ones
	describeParametersOpts := elasticache.DescribeCacheParametersInput{
		CacheParameterGroupName: aws.String(d.Id()),
		Source:                  aws.String("user"),
	}

	describeParametersResp, err := conn.DescribeCacheParameters(&describeParametersOpts)
	if err != nil {
		return err
	}

	d.Set("parameter", flattenElastiCacheParameters(describeParametersResp.Parameters))

	return nil
}

func resourceAwsElasticacheParameterGroupUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	d.Partial(true)

	if d.HasChange("parameter") {
		o, n := d.GetChange("parameter")
		if o == nil {
			o = new(schema.Set)
		}
		if n == nil {
			n = new(schema.Set)
		}

		os := o.(*schema.Set)
		ns := n.(*schema.Set)

		// Expand the "parameter" set to aws-sdk-go compat []elasticacheconn.Parameter
		parameters, err := expandElastiCacheParameters(ns.Difference(os).List())
		if err != nil {
			return err
		}

		if len(parameters) > 0 {
			modifyOpts := elasticache.ModifyCacheParameterGroupInput{
				CacheParameterGroupName: aws.String(d.Get("name").(string)),
				ParameterNameValues:     parameters,
			}

			log.Printf("[DEBUG] Modify Cache Parameter Group: %#v", modifyOpts)
			_, err = conn.ModifyCacheParameterGroup(&modifyOpts)
			if err != nil {
				return fmt.Errorf("Error modifying Cache Parameter Group: %s", err)
			}
		}
		d.SetPartial("parameter")
	}

	d.Partial(false)

	return resourceAwsElasticacheParameterGroupRead(d, meta)
}

func resourceAwsElasticacheParameterGroupDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	return resource.Retry(3*time.Minute, func() *resource.RetryError {
		deleteOpts := elasticache.DeleteCacheParameterGroupInput{
			CacheParameterGroupName: aws.String(d.Id()),
		}
		_, err := conn.DeleteCacheParameterGroup(&deleteOpts)
		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if ok && awsErr.Code() == "CacheParameterGroupNotFoundFault" {
				d.SetId("")
				return nil
			}
			if ok && awsErr.Code() == "InvalidCacheParameterGroupState" {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
}

func resourceAwsElasticacheParameterHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%s-", m["name"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["value"].(string)))

	return hashcode.String(buf.String())
}
<commit_msg>support more than 20 parameters in elasticacheGroupParameterGroup<commit_after>package aws

import (
	"bytes"
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/elasticache"
)

func resourceAwsElasticacheParameterGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElasticacheParameterGroupCreate,
		Read:   resourceAwsElasticacheParameterGroupRead,
		Update: resourceAwsElasticacheParameterGroupUpdate,
		Delete: resourceAwsElasticacheParameterGroupDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},
			"family": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "Managed by Terraform",
			},
			"parameter": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				ForceNew: false,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						"value": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
				Set: resourceAwsElasticacheParameterHash,
			},
		},
	}
}

func resourceAwsElasticacheParameterGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	createOpts := elasticache.CreateCacheParameterGroupInput{
		CacheParameterGroupName:   aws.String(d.Get("name").(string)),
		CacheParameterGroupFamily: aws.String(d.Get("family").(string)),
		Description:               aws.String(d.Get("description").(string)),
	}

	log.Printf("[DEBUG] Create Cache Parameter Group: %#v", createOpts)
	_, err := conn.CreateCacheParameterGroup(&createOpts)
	if err != nil {
		return fmt.Errorf("Error creating Cache Parameter Group: %s", err)
	}

	d.Partial(true)
	d.SetPartial("name")
	d.SetPartial("family")
	d.SetPartial("description")
	d.Partial(false)

	d.SetId(*createOpts.CacheParameterGroupName)
	log.Printf("[INFO] Cache Parameter Group ID: %s", d.Id())

	return resourceAwsElasticacheParameterGroupUpdate(d, meta)
}

func resourceAwsElasticacheParameterGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	describeOpts := elasticache.DescribeCacheParameterGroupsInput{
		CacheParameterGroupName: aws.String(d.Id()),
	}

	describeResp, err := conn.DescribeCacheParameterGroups(&describeOpts)
	if err != nil {
		return err
	}

	if len(describeResp.CacheParameterGroups) != 1 ||
		*describeResp.CacheParameterGroups[0].CacheParameterGroupName != d.Id() {
		return fmt.Errorf("Unable to find Parameter Group: %#v", describeResp.CacheParameterGroups)
	}

	d.Set("name", describeResp.CacheParameterGroups[0].CacheParameterGroupName)
	d.Set("family", describeResp.CacheParameterGroups[0].CacheParameterGroupFamily)
	d.Set("description", describeResp.CacheParameterGroups[0].Description)

	// Only include user customized parameters as there's hundreds of system/default ones
	describeParametersOpts := elasticache.DescribeCacheParametersInput{
		CacheParameterGroupName: aws.String(d.Id()),
		Source:                  aws.String("user"),
	}

	describeParametersResp, err := conn.DescribeCacheParameters(&describeParametersOpts)
	if err != nil {
		return err
	}

	d.Set("parameter", flattenElastiCacheParameters(describeParametersResp.Parameters))

	return nil
}

func resourceAwsElasticacheParameterGroupUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	d.Partial(true)

	if d.HasChange("parameter") {
		o, n := d.GetChange("parameter")
		if o == nil {
			o = new(schema.Set)
		}
		if n == nil {
			n = new(schema.Set)
		}

		os := o.(*schema.Set)
		ns := n.(*schema.Set)

		// Expand the "parameter" set to aws-sdk-go compat []elasticacheconn.Parameter
		parameters, err := expandElastiCacheParameters(ns.Difference(os).List())
		if err != nil {
			return err
		}

		if len(parameters) > 0 {
			// We can only modify 20 parameters at a time, so walk them until
			// we've got them all.
			maxParams := 20
			for parameters != nil {
				paramsToModify := make([]*elasticache.ParameterNameValue, 0)
				if len(parameters) <= maxParams {
					paramsToModify, parameters = parameters[:], nil
				} else {
					paramsToModify, parameters = parameters[:maxParams], parameters[maxParams:]
				}
				modifyOpts := elasticache.ModifyCacheParameterGroupInput{
					CacheParameterGroupName: aws.String(d.Get("name").(string)),
					ParameterNameValues:     paramsToModify,
				}

				log.Printf("[DEBUG] Modify Cache Parameter Group: %#v", modifyOpts)
				_, err = conn.ModifyCacheParameterGroup(&modifyOpts)
				if err != nil {
					return fmt.Errorf("Error modifying Cache Parameter Group: %s", err)
				}
			}
			d.SetPartial("parameter")
		}
	}

	d.Partial(false)

	return resourceAwsElasticacheParameterGroupRead(d, meta)
}

func resourceAwsElasticacheParameterGroupDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).elasticacheconn

	return resource.Retry(3*time.Minute, func() *resource.RetryError {
		deleteOpts := elasticache.DeleteCacheParameterGroupInput{
			CacheParameterGroupName: aws.String(d.Id()),
		}
		_, err := conn.DeleteCacheParameterGroup(&deleteOpts)
		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if ok && awsErr.Code() == "CacheParameterGroupNotFoundFault" {
				d.SetId("")
				return nil
			}
			if ok && awsErr.Code() == "InvalidCacheParameterGroupState" {
				return resource.RetryableError(err)
			}
			return resource.NonRetryableError(err)
		}
		return nil
	})
}

func resourceAwsElasticacheParameterHash(v interface{}) int {
	var buf bytes.Buffer
	m := v.(map[string]interface{})
	buf.WriteString(fmt.Sprintf("%s-", m["name"].(string)))
	buf.WriteString(fmt.Sprintf("%s-", m["value"].(string)))

	return hashcode.String(buf.String())
}
<|endoftext|>


<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package server

import (
	"testing"
	"bytes"
)

type compareTest struct {
	a string
	b string
	expectedResult int
}

func testComparater(t *testing.T, test compareTest, c EventStreamComparer) {
	testComparator(t, test, c)
	testSeparator(t, test, c)
	testSuccessor(t, test.a, c)
	testSuccessor(t, test.b, c)
}

func testSeparator(t *testing.T, test compareTest, c EventStreamComparer) {
	ba := []byte(test.a)
	bb := []byte(test.b)
	res := c.Separator(ba, bb)
	diff := c.Compare(ba, bb)
	if diff >= 0 && bytes.Compare(res, ba) != 0 {
		t.Errorf("Separator was modified when it shouldn't.")
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
	}
	if len(res) > len(ba) {
		t.Errorf("a was lengthened, not shortened.")
	}
	if c.Compare(ba, res) > 0 {
		t.Errorf("The Separator was less than 'a'.")
	}
	if c.Compare(bb, res) <= 0 {
		t.Errorf("The Separator was >= 'a'.")
	}
}

func testComparator(t *testing.T, test compareTest, comparer EventStreamComparer) {
	res := comparer.Compare([]byte(test.a), []byte(test.b))
	if res != test.expectedResult {
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
		t.Errorf("Output was %d. Expected: %d", res,
			test.expectedResult)
	}
}

func testSuccessor(t *testing.T, s string, comparer EventStreamComparer) {
	bs := []byte(s)
	shorter := comparer.Successor(bs)
	if len(shorter) > len(bs) {
		t.Errorf("Successor was longer: %s", s)
	}
	if comparer.Compare(shorter, bs) < 0 {
		t.Errorf("Successor was greater the its origin: %s", s)
	}
}

func TestComparator(t *testing.T) {
	comparer := EventStreamComparer{}
	tests := []compareTest{
		compareTest{"g:a", "g:a", 0},
		compareTest{"g:a", "g:b", -1},
		compareTest{"g:a", "h:a", -1},
		compareTest{"g:a:1", "g:a:1", 0},
		compareTest{"g:a:1", "h:a:1", -1},
		compareTest{"g:a:1", "h:a:2", -1},
		compareTest{"g:a:1", "h:a:11", -1},
		compareTest{"g:a:b:1", "g:a:b:11", -1},

		// Test cases where there are multiple comma keys in the
		// middle.
		compareTest{"g:a:b", "g:a:b", 0},
		compareTest{"g:a:b", "g:b:c", -1},
		compareTest{"g:a:b", "h:a:b", -1},
		compareTest{"g:a:b:1", "g:a:b:1", 0},
		compareTest{"g:a:q:1", "h:a:q:1", -1},
		compareTest{"g:a:b:1", "h:a:b:2", -1},
		compareTest{"g:a:b:1", "h:a:b:11", -1},
	}
	for _, test := range(tests) {
		testComparator(t, test, comparer)
		if test.expectedResult != 0 {
			// Testing the inverted case
			invertedTest := compareTest{
				test.b,
				test.a,
				-test.expectedResult,
			}
			testComparator(t, invertedTest, comparer)
		}
	}
}

<commit_msg>Minor: Name change: compareTest => comparatorTest<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package server

import (
	"testing"
	"bytes"
)

type comparatorTest struct {
	a string
	b string
	expectedResult int
}

func testComparater(t *testing.T, test comparatorTest, c EventStreamComparer) {
	testComparator(t, test, c)
	testSeparator(t, test, c)
	testSuccessor(t, test.a, c)
	testSuccessor(t, test.b, c)
}

func testSeparator(t *testing.T, test comparatorTest, c EventStreamComparer) {
	ba := []byte(test.a)
	bb := []byte(test.b)
	res := c.Separator(ba, bb)
	diff := c.Compare(ba, bb)
	if diff >= 0 && bytes.Compare(res, ba) != 0 {
		t.Errorf("Separator was modified when it shouldn't.")
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
	}
	if len(res) > len(ba) {
		t.Errorf("a was lengthened, not shortened.")
	}
	if c.Compare(ba, res) > 0 {
		t.Errorf("The Separator was less than 'a'.")
	}
	if c.Compare(bb, res) <= 0 {
		t.Errorf("The Separator was >= 'a'.")
	}
}

func testComparator(t *testing.T, test comparatorTest, comparer EventStreamComparer) {
	res := comparer.Compare([]byte(test.a), []byte(test.b))
	if res != test.expectedResult {
		t.Errorf("a: %s", test.a)
		t.Errorf("b: %s", test.b)
		t.Errorf("Output was %d. Expected: %d", res,
			test.expectedResult)
	}
}

func testSuccessor(t *testing.T, s string, comparer EventStreamComparer) {
	bs := []byte(s)
	shorter := comparer.Successor(bs)
	if len(shorter) > len(bs) {
		t.Errorf("Successor was longer: %s", s)
	}
	if comparer.Compare(shorter, bs) < 0 {
		t.Errorf("Successor was greater the its origin: %s", s)
	}
}

func TestComparator(t *testing.T) {
	comparer := EventStreamComparer{}
	tests := []comparatorTest{
		comparatorTest{"g:a", "g:a", 0},
		comparatorTest{"g:a", "g:b", -1},
		comparatorTest{"g:a", "h:a", -1},
		comparatorTest{"g:a:1", "g:a:1", 0},
		comparatorTest{"g:a:1", "h:a:1", -1},
		comparatorTest{"g:a:1", "h:a:2", -1},
		comparatorTest{"g:a:1", "h:a:11", -1},
		comparatorTest{"g:a:b:1", "g:a:b:11", -1},

		// Test cases where there are multiple colon keys in the
		// middle.
		comparatorTest{"g:a:b", "g:a:b", 0},
		comparatorTest{"g:a:b", "g:b:c", -1},
		comparatorTest{"g:a:b", "h:a:b", -1},
		comparatorTest{"g:a:b:1", "g:a:b:1", 0},
		comparatorTest{"g:a:q:1", "h:a:q:1", -1},
		comparatorTest{"g:a:b:1", "h:a:b:2", -1},
		comparatorTest{"g:a:b:1", "h:a:b:11", -1},
	}
	for _, test := range(tests) {
		testComparator(t, test, comparer)
		if test.expectedResult != 0 {
			// Testing the inverted case
			invertedTest := comparatorTest{
				test.b,
				test.a,
				-test.expectedResult,
			}
			testComparator(t, invertedTest, comparer)
		}
	}
}

<|endoftext|>


<commit_before>package server

const ReleaseVersion = "0.4.0"
<commit_msg>chore(server): bump to 0.4.0+git<commit_after>package server

const ReleaseVersion = "0.4.0+git"
<|endoftext|>


<commit_before>package tests

import (
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	"socialapi/request"
	"socialapi/rest"
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/api"
	"socialapi/workers/integration/webhook/services"
	"testing"

	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
)

func newPrepareRequest(email string) *services.ServiceInput {
	return &services.ServiceInput{
		"email":     email,
		"eventName": "emailOpen",
		"message":   "testing it",
		"dataFields": map[string]interface{}{
			"templateId": float64(15120),
			"device":     "Gmail",
			"createdAt":  "2015-04-24 21:35:02 +00:00",
			"campaignId": float64(5654),
			"userAgent":  "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)",
			"ip":         "66.249.93.139",
		},
	}
}

func newPushRequest(channelId int64, groupName string) *api.PushRequest {
	wr := &api.PushRequest{
		GroupName: groupName,
	}
	wr.Body = "hey"
	wr.ChannelId = channelId

	return wr
}

func newBotChannelRequest(nick, groupName string) *api.BotChannelRequest {
	return &api.BotChannelRequest{
		GroupName: groupName,
		Username:  nick,
	}
}

func TestWebhook(t *testing.T) {
	r := runner.New("test webhook")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	channelIntegration := webhook.CreateTestChannelIntegration(t)

	webhook.CreateIterableIntegration(t)

	Convey("We should be able to successfully push message", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		resp, err := rest.GetHistory(channel.Id,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
	})

	Convey("We should be able to successfully fetch bot channel of the user", t, func() {
		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)
		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, models.RandomName())
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)

		So(err, ShouldBeNil)
		So(channelId, ShouldNotEqual, 0)
	})

	Convey("We should be able to successfully push messages via prepare endpoint", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick(models.RandomName())
		So(err, ShouldBeNil)

		err = rest.DoPrepareRequest(newPrepareRequest("xxx@koding.com"), channelIntegration.Token)
		So(err, ShouldNotBeNil)

		err = rest.DoPrepareRequest(newPrepareRequest(account.Nick+"@koding.com"), channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)
		So(err, ShouldBeNil)

		resp, err := rest.GetHistory(channelId,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
	})

}
<commit_msg>webhook: add group name to requests<commit_after>package tests

import (
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	"socialapi/request"
	"socialapi/rest"
	"socialapi/workers/integration/webhook"
	"socialapi/workers/integration/webhook/api"
	"socialapi/workers/integration/webhook/services"
	"testing"

	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
)

func newPrepareRequest(email string, groupName string) *services.ServiceInput {
	return &services.ServiceInput{
		"email":     email,
		"eventName": "emailOpen",
		"message":   "testing it",
		"groupName": groupName,
		"dataFields": map[string]interface{}{
			"templateId": float64(15120),
			"device":     "Gmail",
			"createdAt":  "2015-04-24 21:35:02 +00:00",
			"campaignId": float64(5654),
			"userAgent":  "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)",
			"ip":         "66.249.93.139",
		},
	}
}

func newPushRequest(channelId int64, groupName string) *api.PushRequest {
	wr := &api.PushRequest{
		GroupName: groupName,
	}
	wr.Body = "hey"
	wr.ChannelId = channelId

	return wr
}

func newBotChannelRequest(nick, groupName string) *api.BotChannelRequest {
	return &api.BotChannelRequest{
		GroupName: groupName,
		Username:  nick,
	}
}

func TestWebhook(t *testing.T) {
	r := runner.New("test webhook")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	appConfig := config.MustRead(r.Conf.Path)
	modelhelper.Initialize(appConfig.Mongo)
	defer modelhelper.Close()

	channelIntegration := webhook.CreateTestChannelIntegration(t)

	webhook.CreateIterableIntegration(t)

	Convey("We should be able to successfully push message", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_TOPIC, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		err = rest.DoPushRequest(newPushRequest(channel.Id, channelIntegration.GroupName), channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, channelIntegration.GroupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		resp, err := rest.GetHistory(channel.Id,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
	})

	Convey("We should be able to successfully fetch bot channel of the user", t, func() {
		account, err := models.CreateAccountInBothDbsWithNick("sinan")
		So(err, ShouldBeNil)
		groupName := models.RandomGroupName()
		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, groupName)
		_, err = channel.AddParticipant(account.Id)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, groupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)

		So(err, ShouldBeNil)
		So(channelId, ShouldNotEqual, 0)
	})

	Convey("We should be able to successfully push messages via prepare endpoint", t, func() {

		account, err := models.CreateAccountInBothDbsWithNick(models.RandomName())
		So(err, ShouldBeNil)

		channel := models.CreateTypedGroupedChannelWithTest(account.Id, models.Channel_TYPE_GROUP, channelIntegration.GroupName)
		_, err = channel.AddParticipant(account.Id)

		pr := newPrepareRequest("xxx@koding.com", channelIntegration.GroupName)
		err = rest.DoPrepareRequest(pr, channelIntegration.Token)
		So(err, ShouldNotBeNil)

		pr = newPrepareRequest(account.Nick+"@koding.com", channelIntegration.GroupName)
		err = rest.DoPrepareRequest(pr, channelIntegration.Token)
		So(err, ShouldBeNil)

		ses, err := models.FetchOrCreateSession(account.Nick, channelIntegration.GroupName)
		So(err, ShouldBeNil)
		So(ses, ShouldNotBeNil)

		channelId, err := rest.DoBotChannelRequest(ses.ClientId)
		So(err, ShouldBeNil)

		resp, err := rest.GetHistory(channelId,
			&request.Query{
				AccountId: account.Id,
			},
			ses.ClientId,
		)

		So(err, ShouldBeNil)
		So(len(resp.MessageList), ShouldEqual, 1)
		So(resp.MessageList[0].Message.Body, ShouldEqual, "testing it")
	})

}
<|endoftext|>


<commit_before>// Copyright Banrai LLC. All rights reserved. Use of this source code is
// governed by the license that can be found in the LICENSE file.

package ui

import (
	"database/sql"
	"github.com/Banrai/TeamWork.io/server/database"
	"log"
	"net/http"
	"strings"
)

type NewPostPage struct {
	Title   string
	Alert   *Alert
	Session *database.SESSION
	Person  *database.PERSON
	Keys    []*database.PUBLIC_KEY
}

func PostMessage(w http.ResponseWriter, r *http.Request, db database.DBConnection, opts ...interface{}) {
	alert := new(Alert)
	alert.Message = "You need to create a session to be able to post a new message. If you have already decryted a session code, you can confirm it here"

	if "POST" == r.Method {
		r.ParseForm()

		sessionCode, sessionCodeExists := r.PostForm["session"]
		personCode, personCodeExists := r.PostForm["person"]
		if sessionCodeExists && personCodeExists {
			sessionId := strings.Join(sessionCode, "")
			personId := strings.Join(personCode, "")
			if len(sessionId) > 0 && len(personId) > 0 {

				fn := func(stmt map[string]*sql.Stmt) {
					// remove any expired sessions
					database.CleanupSessions(stmt[database.SESSION_CLEANUP])

					// fetch the session corresponding to this id
					session, sessionErr := database.LookupSession(stmt[database.SESSION_LOOKUP_BY_ID], sessionId)
					if sessionErr != nil {
						alert.AlertType = "alert-danger"
						alert.Icon = "fa-exclamation-triangle"
						alert.Message = OTHER_ERROR
						return
					}

					// attempt to find the person for this session
					person, personErr := database.LookupPerson(stmt[database.PERSON_LOOKUP_BY_ID], session.PersonId)
					if personErr != nil {
						alert.AlertType = "alert-danger"
						alert.Icon = "fa-exclamation-triangle"
						alert.Message = OTHER_ERROR
						return
					}

					if len(person.Id) == 0 {
						alert.AlertType = "alert-danger"
						alert.Icon = "fa-exclamation-triangle"
						alert.Message = UNKNOWN
						return
					}

					if !person.Enabled {
						alert.AlertType = "alert-danger"
						alert.Icon = "fa-exclamation-triangle"
						alert.Message = DISABLED
						return
					}

					// make sure the session matches the person from the form
					if person.Id != personId {
						alert.AlertType = "alert-danger"
						alert.Icon = "fa-exclamation-triangle"
						alert.Message = INVALID_SESSION
						return
					}

					// see if there is a message to post
					messageData, messageDataExists := r.PostForm["message"]
					if !messageDataExists {
						alert.Message = "Please write a message before posting (encryption is optional)"
						return
					}

					// post the message to the database
					message := new(database.MESSAGE)
					message.Message = strings.Join(messageData, "")
					message.PersonId = person.Id
					msgId, msgIdErr := message.Add(stmt[database.MESSAGE_INSERT], MESSAGE_DURATION)
					if msgIdErr != nil {
						alert.AlertType = "alert-danger"
						alert.Icon = "fa-exclamation-triangle"
						alert.Message = "Sorry, your message could not be posted at this time"
						return
					}
					message.Id = msgId

					// add the list of recipients to the message
					recipientList, recipientListExists := r.PostForm["recipients"]
					if recipientListExists {
						people := make([]*database.PERSON, 0)
						for _, recipientEmail := range recipientList {
							recipient, recipientErr := database.LookupPerson(stmt[database.PERSON_LOOKUP_BY_EMAIL], recipientEmail)
							if recipientErr != nil { // for now, but maybe prepare an alert
								log.Println(recipientErr)
							} else {
								people = append(people, recipient)
							}
						}
						if len(people) > 0 {
							for _, msgRecipientErr := range message.AddRecipients(stmt[database.RECIPIENT_INSERT], people) {
								if msgRecipientErr != nil { // for now
									log.Println(msgRecipientErr)
								}
							}
						}
					}

					// success
					//Redirect("/posts")(w, r) // eventually: to the list of recent posts
					// for now
					alert.Message = "Your message has been posted"
					postForm := &NewPostPage{Title: "New Post", Session: session, Person: person}
					NEW_POST_TEMPLATE.Execute(w, postForm)
				}
				database.WithDatabase(db, fn)
			}
		}
	}

	sessionForm := &CreateSessionPage{Title: "New Session", Alert: alert}
	CREATE_SESSION_TEMPLATE.Execute(w, sessionForm)
}
<commit_msg>Applied the Alert helper to post_message.go<commit_after>// Copyright Banrai LLC. All rights reserved. Use of this source code is
// governed by the license that can be found in the LICENSE file.

package ui

import (
	"database/sql"
	"github.com/Banrai/TeamWork.io/server/database"
	"log"
	"net/http"
	"strings"
)

type NewPostPage struct {
	Title   string
	Alert   *Alert
	Session *database.SESSION
	Person  *database.PERSON
	Keys    []*database.PUBLIC_KEY
}

func PostMessage(w http.ResponseWriter, r *http.Request, db database.DBConnection, opts ...interface{}) {
	alert := new(Alert)
	alert.Message = "You need to create a session to be able to post a new message. If you have already decryted a session code, you can confirm it here"

	if "POST" == r.Method {
		r.ParseForm()

		sessionCode, sessionCodeExists := r.PostForm["session"]
		personCode, personCodeExists := r.PostForm["person"]
		if sessionCodeExists && personCodeExists {
			sessionId := strings.Join(sessionCode, "")
			personId := strings.Join(personCode, "")
			if len(sessionId) > 0 && len(personId) > 0 {

				fn := func(stmt map[string]*sql.Stmt) {
					// remove any expired sessions
					database.CleanupSessions(stmt[database.SESSION_CLEANUP])

					// fetch the session corresponding to this id
					session, sessionErr := database.LookupSession(stmt[database.SESSION_LOOKUP_BY_ID], sessionId)
					if sessionErr != nil {
						alert.Update("alert-danger", "fa-exclamation-triangle", OTHER_ERROR)
						return
					}

					// attempt to find the person for this session
					person, personErr := database.LookupPerson(stmt[database.PERSON_LOOKUP_BY_ID], session.PersonId)
					if personErr != nil {
						alert.Update("alert-danger", "fa-exclamation-triangle", OTHER_ERROR)
						return
					}

					if len(person.Id) == 0 {
						alert.Update("alert-danger", "fa-exclamation-triangle", UNKNOWN)
						return
					}

					if !person.Enabled {
						alert.Update("alert-danger", "fa-exclamation-triangle", DISABLED)
						return
					}

					// make sure the session matches the person from the form
					if person.Id != personId {
						alert.Update("alert-danger", "fa-exclamation-triangle", INVALID_SESSION)
						return
					}

					// see if there is a message to post
					messageData, messageDataExists := r.PostForm["message"]
					if !messageDataExists {
						alert.Message = "Please write a message before posting (encryption is optional)"
						return
					}

					// post the message to the database
					message := new(database.MESSAGE)
					message.Message = strings.Join(messageData, "")
					message.PersonId = person.Id
					msgId, msgIdErr := message.Add(stmt[database.MESSAGE_INSERT], MESSAGE_DURATION)
					if msgIdErr != nil {
						alert.Update("alert-danger", "fa-exclamation-triangle", "Sorry, your message could not be posted at this time")
						return
					}
					message.Id = msgId

					// add the list of recipients to the message
					recipientList, recipientListExists := r.PostForm["recipients"]
					if recipientListExists {
						people := make([]*database.PERSON, 0)
						for _, recipientEmail := range recipientList {
							recipient, recipientErr := database.LookupPerson(stmt[database.PERSON_LOOKUP_BY_EMAIL], recipientEmail)
							if recipientErr != nil { // for now, but maybe prepare an alert
								log.Println(recipientErr)
							} else {
								people = append(people, recipient)
							}
						}
						if len(people) > 0 {
							for _, msgRecipientErr := range message.AddRecipients(stmt[database.RECIPIENT_INSERT], people) {
								if msgRecipientErr != nil { // for now
									log.Println(msgRecipientErr)
								}
							}
						}
					}

					// success
					//Redirect("/posts")(w, r) // eventually: to the list of recent posts
					// for now
					alert.Message = "Your message has been posted"
					postForm := &NewPostPage{Title: "New Post", Session: session, Person: person}
					NEW_POST_TEMPLATE.Execute(w, postForm)
				}
				database.WithDatabase(db, fn)
			}
		}
	}

	sessionForm := &CreateSessionPage{Title: "New Session", Alert: alert}
	CREATE_SESSION_TEMPLATE.Execute(w, sessionForm)
}
<|endoftext|>


<commit_before>//////
// This file is part of the Funcgo compiler.
//
// Copyright (c) 2014 Eamonn O'Brien-Strain All rights
// reserved. This program and the accompanying materials are made
// available under the terms of the Eclipse Public License v1.0 which
// accompanies this distribution, and is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// Contributors:
// Eamonn O'Brien-Strain e@obrain.com - initial author
//////

// A symbol table is a mutable state that keeps track of the symbols
// declared, so that the codegenerator can throw exceptions when it
// encounters an undefined symbol.

package symboltable
import "clojure/string"

// Return a new symbol table.
func New() {
	ref({
		"long": TYPE,
		"double": TYPE,
		"boolean": TYPE,
		UNUSED_PACKAGES: set{},
		UNUSED_TYPES: set{}
	})
}

// Add a package symbol to the table.
func PackageImported(st, pkg) {
	dosync(st alter func{
		.. += {
			pkg: PACKAGE,
			UNUSED_PACKAGES: (*st)(UNUSED_PACKAGES) conj pkg
		}
	})
}

// Add a package symbol to the table, but don't require it to be used.
func PackageCreated(st, pkg) {
	dosync(st alter func{.. += {
		pkg: PACKAGE
	}})
}

// Add a package symbol to the table.
func TypeImported(st, typ) {
	dosync(st alter func{.. += {
		typ: TYPE,
		UNUSED_TYPES: (*st)(UNUSED_TYPES) conj typ
	}})
	//dosync{
	//	st := .. += {
	//		typ: TYPE,
	//		UNUSED_TYPES: (*st)(UNUSED_TYPES) conj typ
	//	}
	//}
}

// Add a package symbol to the table.
func TypeCreated(st, typ) {
	dosync(st alter func{.. += {
		typ: TYPE
	}})
	//dosync{
	//	st := .. += {typ: TYPE}
	//}
}

// Has this package been previously been added to the table?
func HasPackage(st, pkg) {
	dosync(st alter func{.. += {
		UNUSED_PACKAGES: (*st)(UNUSED_PACKAGES) disj pkg
	}})
	(*st)(pkg) == PACKAGE
}

// Has this type been previously been added to the table?
func HasType(st, typ) {
	dosync(st alter func{.. += {
		UNUSED_TYPES: (*st)(UNUSED_TYPES) disj typ
	}})
	(*st)(typ) == TYPE
}

// Return a string representation of packages in the table.
func Packages(st) {
	const packages = for [symbol, key] := lazy *st if key == PACKAGE { symbol }
	str("[", ", " string.join packages, "]")
}

// Return a string representation of types in the table.
func Types(st) {
	const packages = for [symbol, key] := lazy *st if key == TYPE { symbol }
	str("[", ", " string.join packages, "]")
}

func CheckAllUsed(st) {
	const (
		pkgs = (*st)(UNUSED_PACKAGES)
		typs = (*st)(UNUSED_TYPES)
	)
	if notEmpty(pkgs) {
		const pkgsS = ", " string.join pkgs
		throw(new Exception(str("Packages imported but never used: [", pkgsS, "]")))
	}
	if notEmpty(typs) {
		const typsS = ", " string.join typs
		throw(new Exception(str("Types imported but never used: [", typsS, "]")))
	}

}
<commit_msg>use new short-form function arg format<commit_after>//////
// This file is part of the Funcgo compiler.
//
// Copyright (c) 2014 Eamonn O'Brien-Strain All rights
// reserved. This program and the accompanying materials are made
// available under the terms of the Eclipse Public License v1.0 which
// accompanies this distribution, and is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// Contributors:
// Eamonn O'Brien-Strain e@obrain.com - initial author
//////

// A symbol table is a mutable state that keeps track of the symbols
// declared, so that the codegenerator can throw exceptions when it
// encounters an undefined symbol.

package symboltable
import "clojure/string"

// Return a new symbol table.
func New() {
	ref({
		"long": TYPE,
		"double": TYPE,
		"boolean": TYPE,
		UNUSED_PACKAGES: set{},
		UNUSED_TYPES: set{}
	})
}

// Add a package symbol to the table.
func PackageImported(st, pkg) {
	dosync(st alter func{
		$1 += {
			pkg: PACKAGE,
			UNUSED_PACKAGES: (*st)(UNUSED_PACKAGES) conj pkg
		}
	})
}

// Add a package symbol to the table, but don't require it to be used.
func PackageCreated(st, pkg) {
	dosync(st alter func{$1 += {
		pkg: PACKAGE
	}})
}

// Add a package symbol to the table.
func TypeImported(st, typ) {
	dosync(st alter func{$1 += {
		typ: TYPE,
		UNUSED_TYPES: (*st)(UNUSED_TYPES) conj typ
	}})
	//dosync{
	//	st := $1 += {
	//		typ: TYPE,
	//		UNUSED_TYPES: (*st)(UNUSED_TYPES) conj typ
	//	}
	//}
}

// Add a package symbol to the table.
func TypeCreated(st, typ) {
	dosync(st alter func{$1 += {
		typ: TYPE
	}})
	//dosync{
	//	st := $1 += {typ: TYPE}
	//}
}

// Has this package been previously been added to the table?
func HasPackage(st, pkg) {
	dosync(st alter func{$1 += {
		UNUSED_PACKAGES: (*st)(UNUSED_PACKAGES) disj pkg
	}})
	(*st)(pkg) == PACKAGE
}

// Has this type been previously been added to the table?
func HasType(st, typ) {
	dosync(st alter func{$1 += {
		UNUSED_TYPES: (*st)(UNUSED_TYPES) disj typ
	}})
	(*st)(typ) == TYPE
}

// Return a string representation of packages in the table.
func Packages(st) {
	const packages = for [symbol, key] := lazy *st if key == PACKAGE { symbol }
	str("[", ", " string.join packages, "]")
}

// Return a string representation of types in the table.
func Types(st) {
	const packages = for [symbol, key] := lazy *st if key == TYPE { symbol }
	str("[", ", " string.join packages, "]")
}

func CheckAllUsed(st) {
	const (
		pkgs = (*st)(UNUSED_PACKAGES)
		typs = (*st)(UNUSED_TYPES)
	)
	if notEmpty(pkgs) {
		const pkgsS = ", " string.join pkgs
		throw(new Exception(str("Packages imported but never used: [", pkgsS, "]")))
	}
	if notEmpty(typs) {
		const typsS = ", " string.join typs
		throw(new Exception(str("Types imported but never used: [", typsS, "]")))
	}

}
<|endoftext|>


<commit_before>package slack

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"sync/atomic"

	"github.com/influxdata/kapacitor"
	"github.com/pkg/errors"
)

type Service struct {
	configValue atomic.Value
	logger      *log.Logger
}

func NewService(c Config, l *log.Logger) *Service {
	s := &Service{
		logger: l,
	}
	s.configValue.Store(c)
	return s
}

func (s *Service) Open() error {
	return nil
}

func (s *Service) Close() error {
	return nil
}

func (s *Service) config() Config {
	return s.configValue.Load().(Config)
}

func (s *Service) Update(newConfig []interface{}) error {
	if l := len(newConfig); l != 1 {
		return fmt.Errorf("expected only one new config object, got %d", l)
	}
	if c, ok := newConfig[0].(Config); !ok {
		return fmt.Errorf("expected config object to be of type %T, got %T", c, newConfig[0])
	} else {
		s.configValue.Store(c)
	}
	return nil
}

func (s *Service) Global() bool {
	c := s.config()
	return c.Global
}

func (s *Service) StateChangesOnly() bool {
	c := s.config()
	return c.StateChangesOnly
}

// slack attachment info
type attachment struct {
	Fallback string `json:"fallback"`
	Color    string `json:"color"`
	Text     string `json:"text"`
}

type testOptions struct {
	Channel   string               `json:"channel"`
	Message   string               `json:"message"`
	Level     kapacitor.AlertLevel `json:"level"`
	Username  string               `json:"username"`
	IconEmoji string               `json:"icon-emoji"`
}

func (s *Service) TestOptions() interface{} {
	c := s.config()
	return &testOptions{
		Channel: c.Channel,
		Message: "test slack message",
		Level:   kapacitor.CritAlert,
	}
}

func (s *Service) Test(options interface{}) error {
	o, ok := options.(*testOptions)
	if !ok {
		return fmt.Errorf("unexpected options type %T", options)
	}
	return s.Alert(o.Channel, o.Message, o.Username, o.IconEmoji, o.Level)
}

func (s *Service) Alert(channel, message, username, iconEmoji string, level kapacitor.AlertLevel) error {
	url, post, err := s.preparePost(channel, message, username, iconEmoji, level)
	if err != nil {
		return err
	}
	resp, err := http.Post(url, "application/json", post)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		type response struct {
			Error string `json:"error"`
		}
		r := &response{Error: fmt.Sprintf("failed to understand Slack response. code: %d content: %s", resp.StatusCode, string(body))}
		b := bytes.NewReader(body)
		dec := json.NewDecoder(b)
		dec.Decode(r)
		return errors.New(r.Error)
	}
	return nil
}

func (s *Service) preparePost(channel, message, username, iconEmoji string, level kapacitor.AlertLevel) (string, io.Reader, error) {
	c := s.config()

	if !c.Enabled {
		return "", nil, errors.New("service is not enabled")
	}
	if channel == "" {
		channel = c.Channel
	}
	var color string
	switch level {
	case kapacitor.WarnAlert:
		color = "warning"
	case kapacitor.CritAlert:
		color = "danger"
	default:
		color = "good"
	}
	a := attachment{
		Fallback: message,
		Text:     message,
		Color:    color,
	}
	postData := make(map[string]interface{})
	postData["as_user"] = false
	postData["channel"] = channel
	postData["text"] = ""
	postData["attachments"] = []attachment{a}

	if username == "" {
		username = c.Username
	}
	postData["username"] = username

	if iconEmoji == "" {
		iconEmoji = c.IconEmoji
	}
	postData["icon_emoji"] = iconEmoji

	var post bytes.Buffer
	enc := json.NewEncoder(&post)
	err := enc.Encode(postData)
	if err != nil {
		return "", nil, err
	}

	return c.URL, &post, nil
}
<commit_msg>Enable markdown in slack attachments #251<commit_after>package slack

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"sync/atomic"

	"github.com/influxdata/kapacitor"
	"github.com/pkg/errors"
)

type Service struct {
	configValue atomic.Value
	logger      *log.Logger
}

func NewService(c Config, l *log.Logger) *Service {
	s := &Service{
		logger: l,
	}
	s.configValue.Store(c)
	return s
}

func (s *Service) Open() error {
	return nil
}

func (s *Service) Close() error {
	return nil
}

func (s *Service) config() Config {
	return s.configValue.Load().(Config)
}

func (s *Service) Update(newConfig []interface{}) error {
	if l := len(newConfig); l != 1 {
		return fmt.Errorf("expected only one new config object, got %d", l)
	}
	if c, ok := newConfig[0].(Config); !ok {
		return fmt.Errorf("expected config object to be of type %T, got %T", c, newConfig[0])
	} else {
		s.configValue.Store(c)
	}
	return nil
}

func (s *Service) Global() bool {
	c := s.config()
	return c.Global
}

func (s *Service) StateChangesOnly() bool {
	c := s.config()
	return c.StateChangesOnly
}

// slack attachment info
type attachment struct {
	Fallback  string   `json:"fallback"`
	Color     string   `json:"color"`
	Text      string   `json:"text"`
	Mrkdwn_in []string `json:"mrkdwn_in"`
}

type testOptions struct {
	Channel   string               `json:"channel"`
	Message   string               `json:"message"`
	Level     kapacitor.AlertLevel `json:"level"`
	Username  string               `json:"username"`
	IconEmoji string               `json:"icon-emoji"`
}

func (s *Service) TestOptions() interface{} {
	c := s.config()
	return &testOptions{
		Channel: c.Channel,
		Message: "test slack message",
		Level:   kapacitor.CritAlert,
	}
}

func (s *Service) Test(options interface{}) error {
	o, ok := options.(*testOptions)
	if !ok {
		return fmt.Errorf("unexpected options type %T", options)
	}
	return s.Alert(o.Channel, o.Message, o.Username, o.IconEmoji, o.Level)
}

func (s *Service) Alert(channel, message, username, iconEmoji string, level kapacitor.AlertLevel) error {
	url, post, err := s.preparePost(channel, message, username, iconEmoji, level)
	if err != nil {
		return err
	}
	resp, err := http.Post(url, "application/json", post)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		type response struct {
			Error string `json:"error"`
		}
		r := &response{Error: fmt.Sprintf("failed to understand Slack response. code: %d content: %s", resp.StatusCode, string(body))}
		b := bytes.NewReader(body)
		dec := json.NewDecoder(b)
		dec.Decode(r)
		return errors.New(r.Error)
	}
	return nil
}

func (s *Service) preparePost(channel, message, username, iconEmoji string, level kapacitor.AlertLevel) (string, io.Reader, error) {
	c := s.config()

	if !c.Enabled {
		return "", nil, errors.New("service is not enabled")
	}
	if channel == "" {
		channel = c.Channel
	}
	var color string
	switch level {
	case kapacitor.WarnAlert:
		color = "warning"
	case kapacitor.CritAlert:
		color = "danger"
	default:
		color = "good"
	}
	a := attachment{
		Fallback:  message,
		Text:      message,
		Color:     color,
		Mrkdwn_in: []string{"text"},
	}
	postData := make(map[string]interface{})
	postData["as_user"] = false
	postData["channel"] = channel
	postData["text"] = ""
	postData["attachments"] = []attachment{a}

	if username == "" {
		username = c.Username
	}
	postData["username"] = username

	if iconEmoji == "" {
		iconEmoji = c.IconEmoji
	}
	postData["icon_emoji"] = iconEmoji

	var post bytes.Buffer
	enc := json.NewEncoder(&post)
	err := enc.Encode(postData)
	if err != nil {
		return "", nil, err
	}

	return c.URL, &post, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"sync"
	"time"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var once sync.Once

func randSeq(n int) string {
	once.Do(func() { rand.Seed(time.Now().UTC().UnixNano()) })
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

type randReader struct{}

func (r randReader) Read(p []byte) (n int, err error) {
	once.Do(func() { rand.Seed(time.Now().UTC().UnixNano()) })
	for i := range p {
		p[i] = byte(rand.Int63() & 0xff)
	}
	return len(p), nil
}

var reader randReader

func TrafficHandler(w http.ResponseWriter, r *http.Request) {
	for i := 0; i < 100; i++ {
		http.Post("http://localhost/pfs/"+randSeq(10), "application/text", reader)
	}
}

// TesterMux creates a multiplexer for a Tester
func TesterMux() *http.ServeMux {
	mux := http.NewServeMux()

	mux.HandleFunc("/traffic", TrafficHandler)
	mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "pong\n") })

	return mux
}

// RunServer runs a master server listening on port 80
func RunServer() {
	http.ListenAndServe(":80", TesterMux())
}

func main() {
	log.SetFlags(log.Lshortfile)
	log.Print("Listening on port 80...")
	RunServer()
}
<commit_msg>First working tester.<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"sync"
	"time"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var once sync.Once

func randSeq(n int) string {
	once.Do(func() { rand.Seed(time.Now().UTC().UnixNano()) })
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

type ConstReader struct{}

func (r ConstReader) Read(p []byte) (n int, err error) {
	for i := range p {
		p[i] = 'a'
	}
	return len(p), nil
}

var reader ConstReader

func TrafficHandler(w http.ResponseWriter, r *http.Request) {
	for i := 0; i < 100; i++ {
		url := "http://172.17.42.1/pfs/" + randSeq(10)
		log.Print("Posting to: ", url)
		resp, err := http.Post(url, "application/text", io.LimitReader(reader, 1<<10))
		if err != nil {
			http.Error(w, err.Error(), 500)
			log.Print(err)
			return
		}
		fmt.Fprint(w, resp)
		io.Copy(w, resp.Body)
	}
	fmt.Fprint(w, "Sent 100 files.\n")
}

// TesterMux creates a multiplexer for a Tester
func TesterMux() *http.ServeMux {
	mux := http.NewServeMux()

	mux.HandleFunc("/traffic", TrafficHandler)
	mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "pong\n") })

	return mux
}

// RunServer runs a master server listening on port 80
func RunServer() {
	http.ListenAndServe(":80", TesterMux())
}

func main() {
	log.SetFlags(log.Lshortfile)
	log.Print("Listening on port 80...")
	RunServer()
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/Shopify/sarama"
	"github.com/cihub/seelog"
	"github.com/wvanbergen/kazoo-go"
)

const (
	REGISTER_CONSUMER_GROUP_RETRY_TIME = 10 * time.Second
	GET_CONSUMER_LIST_RETRY_TIME       = 10 * time.Second
	RUN_PARTITION_MANAGER_RETRY_TIME   = 10 * time.Second
	CONSUMER_LIST_CHANGE_RELOAD_TIME   = 5 * time.Second
	WATCH_INSTANCE_CHANGE_DELAY_TIME   = 5 * time.Second
)

type CallbackManager struct {
	*StartStopControl
	Topics    []string
	GroupName string
	Url       string

	// config
	config          *CallbackItemConfig
	saramaConfig    *sarama.Config
	zookeeperConfig *kazoo.Config

	// zk instances
	kazoo              *kazoo.Kazoo                 // ZK
	kazooGroup         *kazoo.Consumergroup         // ZK ConsumerGroup /consumers/<cgname>/ Object
	kazooGroupInstance *kazoo.ConsumergroupInstance // ZK ConsumerGroup /consumers/<cgname>/ids/<cginstance> Object

	// kafka sarama consumer
	saramaConsumer sarama.Consumer

	// partition manager
	partitionManagers      []*PartitionManager
	partitionManagerRunner *ServiceRunner

	// offset manager
	offsetManager *OffsetManager
}

func NewCallbackManager() *CallbackManager {
	return &CallbackManager{
		StartStopControl: &StartStopControl{},
	}
}

func (this *CallbackManager) Init(config *CallbackItemConfig) error {
	var err error

	this.config = config
	this.Topics = config.Topics
	this.Url = config.Url

	// set group as MD5(Url)
	this.GroupName = getGroupName(this.Url)
	this.saramaConfig = sarama.NewConfig()
	this.zookeeperConfig = kazoo.NewConfig()
	this.zookeeperConfig.Chroot = config.ZkPath
	this.saramaConfig.ClientID = this.GroupName

	// init OffsetManager
	this.offsetManager = NewOffsetManager()
	if err = this.offsetManager.Init(config.OffsetConfig, this); err != nil {
		return err
	}

	this.partitionManagerRunner = NewServiceRunner()

	return nil
}

func (this *CallbackManager) Run() error {
	// mark service as started
	if err := this.ensureStart(); err != nil {
		return err
	}

	defer this.markStop()

	// init zookeeper
	if err := this.connectZookeeper(); err != nil {
		return err
	}

	// init kafka sarama consumer
	if err := this.connectKafka(); err != nil {
		return err
	}

callbackManagerFailoverLoop:
	for {
		if !this.Running() {
			break callbackManagerFailoverLoop
		}

		if err := this.registerConsumergroup(); err != nil {
			time.Sleep(REGISTER_CONSUMER_GROUP_RETRY_TIME)
			continue
		}

		seelog.Infof("Waiting for %v to avoid consumer register rebalance herd",
			WATCH_INSTANCE_CHANGE_DELAY_TIME)
		time.Sleep(WATCH_INSTANCE_CHANGE_DELAY_TIME)

		consumers, consumerChanges, err := this.kazooGroup.WatchInstances()

		if err != nil {
			seelog.Errorf("Failed to get list of registered consumer instances [err:%s]", err)
			time.Sleep(GET_CONSUMER_LIST_RETRY_TIME)
			continue
		}

		seelog.Infof("Currently registered consumers [totalConsumers:%d]", len(consumers))

		// get partitionConsuming assignments
		// start ServiceRunner of PartitionManager
		// TODO refactor this
		if err := this.partitionRun(consumers); err != nil {
			seelog.Errorf("Failed to init partition consumer [err:%s]", err)
			time.Sleep(RUN_PARTITION_MANAGER_RETRY_TIME)
			continue
		}

		select {
		case <-this.WaitForCloseChannel():
			this.partitionManagerRunner.Close()
			break callbackManagerFailoverLoop

		case <-consumerChanges:
			seelog.Infof("Triggering rebalance due to consumer list change")
			this.partitionManagerRunner.Close()
			seelog.Infof("Waiting for %v to avoid consumer inflight rebalance herd",
				CONSUMER_LIST_CHANGE_RELOAD_TIME)
			time.Sleep(CONSUMER_LIST_CHANGE_RELOAD_TIME)
		case <-this.partitionManagerRunner.WaitForExitChannel():
			seelog.Warn("PartitionManager unexpectedly stopped")
		}

		// deregister Consumergroup instance from zookeeper
		if err := this.kazooGroupInstance.Deregister(); err != nil {
			seelog.Errorf("Failed deregistering consumer instance [err:%s]", err)
		} else {
			seelog.Infof("Deregistered consumer instance [instanceId:%s]", this.kazooGroupInstance.ID)
		}
	}

	// sync close offsetManager
	if err := this.offsetManager.Close(); err != nil {
		seelog.Errorf("Failed closing the offset manager [err:%s]", err)
	}

	// close sarama Consumer
	if err := this.saramaConsumer.Close(); err != nil {
		seelog.Errorf("Failed closing the Sarama client [err:%s]", err)
	}

	// close zookeeper connection
	if err := this.kazoo.Close(); err != nil {
		seelog.Errorf("Failed closing the Zookeeper connection [err:%s]", err)
	}

	return nil
}

func (this *CallbackManager) GetOffsetManager() *OffsetManager {
	return this.offsetManager
}

func (this *CallbackManager) GetKazooGroup() *kazoo.Consumergroup {
	return this.kazooGroup
}

func (this *CallbackManager) GetKazooGroupInstance() *kazoo.ConsumergroupInstance {
	return this.kazooGroupInstance
}

func (this *CallbackManager) GetConsumer() sarama.Consumer {
	return this.saramaConsumer
}

func (this *CallbackManager) connectZookeeper() error {
	var err error

	// zookeeper ConsumerGroup instance initialization
	if this.kazoo, err = kazoo.NewKazoo(this.config.Zookeepers, this.zookeeperConfig); err != nil {
		return err
	}
	this.kazooGroup = this.kazoo.Consumergroup(this.GroupName)
	this.kazooGroupInstance = this.kazooGroup.NewInstance()

	return nil
}

func (this *CallbackManager) connectKafka() error {
	var err error

	// kafka consumer initialization
	brokers, err := this.kazoo.BrokerList()
	if err != nil {
		return err
	}

	// connect kafka using sarama Consumer
	if this.saramaConsumer, err = sarama.NewConsumer(brokers, this.saramaConfig); err != nil {
		return err
	}

	return nil
}

func (this *CallbackManager) registerConsumergroup() error {
	// Register Consumergroup zk node
	if exists, err := this.kazooGroup.Exists(); err != nil {
		seelog.Errorf("Failed to check for existence of consumergroup [err:%s]", err)
		return err
	} else if !exists {
		seelog.Infof("Consumergroup does not yet exists, creating [consumergroup:%s] ", this.GroupName)
		if err := this.kazooGroup.Create(); err != nil {
			seelog.Errorf("Failed to create consumergroup in zookeeper [err:%s]", err)
			return err
		}
	}

	// register new kazoo.ConsumerGroup instance
	if err := this.kazooGroupInstance.Register(this.Topics); err != nil {
		seelog.Errorf("Failed to register consumer instance [err:%s]", err)
		return err
	} else {
		seelog.Infof("Consumer instance registered [instanceId:%s]", this.kazooGroupInstance.ID)
	}

	return nil
}

func (this *CallbackManager) partitionRun(consumers kazoo.ConsumergroupInstanceList) error {
	this.partitionManagers = make([]*PartitionManager, 0)

	for _, topic := range this.Topics {
		// Fetch a list of partition IDs
		partitions, err := this.kazoo.Topic(topic).Partitions()
		if err != nil {
			seelog.Errorf("Failed to get list of partitions [topic:%s][err:%s]", this.Topics[0], err)
			return err
		}

		partitionLeaders, err := retrievePartitionLeaders(partitions)
		if err != nil {
			seelog.Errorf("Failed to get leaders of partitions [topic:%s][err:%s]", topic, err)
			return err
		}

		// divide partition for each callback manager instance
		dividedPartitions := dividePartitionsBetweenConsumers(consumers, partitionLeaders)
		myPartitions := dividedPartitions[this.kazooGroupInstance.ID]

		for i := 0; i < len(myPartitions); i++ {
			partitionManager := NewPartitionManager()
			if err := partitionManager.Init(this.config, topic, myPartitions[i].ID, this); err != nil {
				seelog.Criticalf("Init partition manager failed [url:%s][err:%s]", this.Url, err)
				return err
			}
			this.partitionManagers = append(this.partitionManagers, partitionManager)
		}
	}

	this.partitionManagerRunner.RetryTimes = len(this.partitionManagers) * 3
	this.partitionManagerRunner.Prepare()
	if _, err := this.partitionManagerRunner.RunAsync(this.partitionManagers); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add missing offsetManager run in callbackManager run<commit_after>package main

import (
	"time"

	"github.com/Shopify/sarama"
	"github.com/cihub/seelog"
	"github.com/wvanbergen/kazoo-go"
)

const (
	REGISTER_CONSUMER_GROUP_RETRY_TIME = 10 * time.Second
	GET_CONSUMER_LIST_RETRY_TIME       = 10 * time.Second
	RUN_PARTITION_MANAGER_RETRY_TIME   = 10 * time.Second
	CONSUMER_LIST_CHANGE_RELOAD_TIME   = 5 * time.Second
	WATCH_INSTANCE_CHANGE_DELAY_TIME   = 5 * time.Second
)

type CallbackManager struct {
	*StartStopControl
	Topics    []string
	GroupName string
	Url       string

	// config
	config          *CallbackItemConfig
	saramaConfig    *sarama.Config
	zookeeperConfig *kazoo.Config

	// zk instances
	kazoo              *kazoo.Kazoo                 // ZK
	kazooGroup         *kazoo.Consumergroup         // ZK ConsumerGroup /consumers/<cgname>/ Object
	kazooGroupInstance *kazoo.ConsumergroupInstance // ZK ConsumerGroup /consumers/<cgname>/ids/<cginstance> Object

	// kafka sarama consumer
	saramaConsumer sarama.Consumer

	// partition manager
	partitionManagers      []*PartitionManager
	partitionManagerRunner *ServiceRunner

	// offset manager
	offsetManager *OffsetManager
}

func NewCallbackManager() *CallbackManager {
	return &CallbackManager{
		StartStopControl: &StartStopControl{},
	}
}

func (this *CallbackManager) Init(config *CallbackItemConfig) error {
	var err error

	this.config = config
	this.Topics = config.Topics
	this.Url = config.Url

	// set group as MD5(Url)
	this.GroupName = getGroupName(this.Url)
	this.saramaConfig = sarama.NewConfig()
	this.zookeeperConfig = kazoo.NewConfig()
	this.zookeeperConfig.Chroot = config.ZkPath
	this.saramaConfig.ClientID = this.GroupName

	// init OffsetManager
	this.offsetManager = NewOffsetManager()
	if err = this.offsetManager.Init(config.OffsetConfig, this); err != nil {
		return err
	}

	this.partitionManagerRunner = NewServiceRunner()

	return nil
}

func (this *CallbackManager) Run() error {
	// mark service as started
	if err := this.ensureStart(); err != nil {
		return err
	}

	defer this.markStop()

	// init zookeeper
	if err := this.connectZookeeper(); err != nil {
		return err
	}

	// init kafka sarama consumer
	if err := this.connectKafka(); err != nil {
		return err
	}

	go this.offsetManager.Run()

callbackManagerFailoverLoop:
	for {
		if !this.Running() {
			break callbackManagerFailoverLoop
		}

		if err := this.registerConsumergroup(); err != nil {
			time.Sleep(REGISTER_CONSUMER_GROUP_RETRY_TIME)
			continue
		}

		seelog.Infof("Waiting for %v to avoid consumer register rebalance herd",
			WATCH_INSTANCE_CHANGE_DELAY_TIME)
		time.Sleep(WATCH_INSTANCE_CHANGE_DELAY_TIME)

		consumers, consumerChanges, err := this.kazooGroup.WatchInstances()

		if err != nil {
			seelog.Errorf("Failed to get list of registered consumer instances [err:%s]", err)
			time.Sleep(GET_CONSUMER_LIST_RETRY_TIME)
			continue
		}

		seelog.Infof("Currently registered consumers [totalConsumers:%d]", len(consumers))

		// get partitionConsuming assignments
		// start ServiceRunner of PartitionManager
		// TODO refactor this
		if err := this.partitionRun(consumers); err != nil {
			seelog.Errorf("Failed to init partition consumer [err:%s]", err)
			time.Sleep(RUN_PARTITION_MANAGER_RETRY_TIME)
			continue
		}

		select {
		case <-this.WaitForCloseChannel():
			this.partitionManagerRunner.Close()
			break callbackManagerFailoverLoop

		case <-consumerChanges:
			seelog.Infof("Triggering rebalance due to consumer list change")
			this.partitionManagerRunner.Close()
			seelog.Infof("Waiting for %v to avoid consumer inflight rebalance herd",
				CONSUMER_LIST_CHANGE_RELOAD_TIME)
			time.Sleep(CONSUMER_LIST_CHANGE_RELOAD_TIME)
		case <-this.partitionManagerRunner.WaitForExitChannel():
			seelog.Warn("PartitionManager unexpectedly stopped")
		}

		// deregister Consumergroup instance from zookeeper
		if err := this.kazooGroupInstance.Deregister(); err != nil {
			seelog.Errorf("Failed deregistering consumer instance [err:%s]", err)
		} else {
			seelog.Infof("Deregistered consumer instance [instanceId:%s]", this.kazooGroupInstance.ID)
		}
	}

	// sync close offsetManager
	if err := this.offsetManager.Close(); err != nil {
		seelog.Errorf("Failed closing the offset manager [err:%s]", err)
	}

	// close sarama Consumer
	if err := this.saramaConsumer.Close(); err != nil {
		seelog.Errorf("Failed closing the Sarama client [err:%s]", err)
	}

	// close zookeeper connection
	if err := this.kazoo.Close(); err != nil {
		seelog.Errorf("Failed closing the Zookeeper connection [err:%s]", err)
	}

	return nil
}

func (this *CallbackManager) GetOffsetManager() *OffsetManager {
	return this.offsetManager
}

func (this *CallbackManager) GetKazooGroup() *kazoo.Consumergroup {
	return this.kazooGroup
}

func (this *CallbackManager) GetKazooGroupInstance() *kazoo.ConsumergroupInstance {
	return this.kazooGroupInstance
}

func (this *CallbackManager) GetConsumer() sarama.Consumer {
	return this.saramaConsumer
}

func (this *CallbackManager) connectZookeeper() error {
	var err error

	// zookeeper ConsumerGroup instance initialization
	if this.kazoo, err = kazoo.NewKazoo(this.config.Zookeepers, this.zookeeperConfig); err != nil {
		return err
	}
	this.kazooGroup = this.kazoo.Consumergroup(this.GroupName)
	this.kazooGroupInstance = this.kazooGroup.NewInstance()

	return nil
}

func (this *CallbackManager) connectKafka() error {
	var err error

	// kafka consumer initialization
	brokers, err := this.kazoo.BrokerList()
	if err != nil {
		return err
	}

	// connect kafka using sarama Consumer
	if this.saramaConsumer, err = sarama.NewConsumer(brokers, this.saramaConfig); err != nil {
		return err
	}

	return nil
}

func (this *CallbackManager) registerConsumergroup() error {
	// Register Consumergroup zk node
	if exists, err := this.kazooGroup.Exists(); err != nil {
		seelog.Errorf("Failed to check for existence of consumergroup [err:%s]", err)
		return err
	} else if !exists {
		seelog.Infof("Consumergroup does not yet exists, creating [consumergroup:%s] ", this.GroupName)
		if err := this.kazooGroup.Create(); err != nil {
			seelog.Errorf("Failed to create consumergroup in zookeeper [err:%s]", err)
			return err
		}
	}

	// register new kazoo.ConsumerGroup instance
	if err := this.kazooGroupInstance.Register(this.Topics); err != nil {
		seelog.Errorf("Failed to register consumer instance [err:%s]", err)
		return err
	} else {
		seelog.Infof("Consumer instance registered [instanceId:%s]", this.kazooGroupInstance.ID)
	}

	return nil
}

func (this *CallbackManager) partitionRun(consumers kazoo.ConsumergroupInstanceList) error {
	this.partitionManagers = make([]*PartitionManager, 0)

	for _, topic := range this.Topics {
		// Fetch a list of partition IDs
		partitions, err := this.kazoo.Topic(topic).Partitions()
		if err != nil {
			seelog.Errorf("Failed to get list of partitions [topic:%s][err:%s]", this.Topics[0], err)
			return err
		}

		partitionLeaders, err := retrievePartitionLeaders(partitions)
		if err != nil {
			seelog.Errorf("Failed to get leaders of partitions [topic:%s][err:%s]", topic, err)
			return err
		}

		// divide partition for each callback manager instance
		dividedPartitions := dividePartitionsBetweenConsumers(consumers, partitionLeaders)
		myPartitions := dividedPartitions[this.kazooGroupInstance.ID]

		for i := 0; i < len(myPartitions); i++ {
			partitionManager := NewPartitionManager()
			if err := partitionManager.Init(this.config, topic, myPartitions[i].ID, this); err != nil {
				seelog.Criticalf("Init partition manager failed [url:%s][err:%s]", this.Url, err)
				return err
			}
			this.partitionManagers = append(this.partitionManagers, partitionManager)
		}
	}

	this.partitionManagerRunner.RetryTimes = len(this.partitionManagers) * 3
	this.partitionManagerRunner.Prepare()
	if _, err := this.partitionManagerRunner.RunAsync(this.partitionManagers); err != nil {
		return err
	}

	return nil
}
<|endoftext|>