<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub_test

import (
	"fmt"
	"time"

	"cloud.google.com/go/pubsub"
	"golang.org/x/net/context"
	"google.golang.org/api/iterator"
)

func ExampleNewClient() {
	ctx := context.Background()
	_, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	// See the other examples to learn how to use the Client.
}

func ExampleClient_CreateTopic() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	// Create a new topic with the given name.
	topic, err := client.CreateTopic(ctx, "topicName")
	if err != nil {
		// TODO: Handle error.
	}

	_ = topic // TODO: use the topic.
}

func ExampleClient_CreateSubscription() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	// Create a new topic with the given name.
	topic, err := client.CreateTopic(ctx, "topicName")
	if err != nil {
		// TODO: Handle error.
	}

	// Create a new subscription to the previously created topic
	// with the given name.
	sub, err := client.CreateSubscription(ctx, "subName", topic, 10*time.Second, nil)
	if err != nil {
		// TODO: Handle error.
	}

	_ = sub // TODO: use the subscription.
}

func ExampleTopic_Delete() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	topic := client.Topic("topicName")
	if err := topic.Delete(ctx); err != nil {
		// TODO: Handle error.
	}
}

func ExampleTopic_Exists() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	topic := client.Topic("topicName")
	ok, err := topic.Exists(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	if !ok {
		// Topic doesn't exist.
	}
}

func ExampleTopic_Publish() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	topic := client.Topic("topicName")
	defer topic.Stop()
	var results []*pubsub.PublishResult
	r := topic.Publish(ctx, &pubsub.Message{
		Data: []byte("hello world"),
	})
	results = append(results, r)
	// Do other work ...
	for _, r := range results {
		id, err := r.Get(ctx)
		if err != nil {
			// TODO: Handle error.
		}
		fmt.Printf("Published a message with a message ID: %s\n", id)
	}
}

func ExampleTopic_Subscriptions() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	topic := client.Topic("topic-name")
	// List all subscriptions of the topic (maybe of multiple projects).
	for subs := topic.Subscriptions(ctx); ; {
		sub, err := subs.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			// TODO: Handle error.
		}
		_ = sub // TODO: use the subscription.
	}
}

func ExampleSubscription_Delete() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	sub := client.Subscription("subName")
	if err := sub.Delete(ctx); err != nil {
		// TODO: Handle error.
	}
}

func ExampleSubscription_Exists() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	sub := client.Subscription("subName")
	ok, err := sub.Exists(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	if !ok {
		// Subscription doesn't exist.
	}
}

func ExampleSubscription_Config() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	config, err := sub.Config(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	fmt.Println(config)
}

func ExampleSubscription_Receive() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
		// TODO: Handle message.
		// NOTE: May be called concurrently; synchronize access to shared memory.
		m.Ack()
	})
	if err != context.Canceled {
		// TODO: Handle error.
	}
}

func ExampleSubscription_Receive_options() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	// This program is expected to process and acknowledge messages
	// in 5 seconds. If not, Pub/Sub API will assume the message is not
	// acknowledged.
	sub.ReceiveSettings.MaxExtension = 5 * time.Second
	err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
		// TODO: Handle message.
		m.Ack()
	})
	if err != context.Canceled {
		// TODO: Handle error.
	}
}

func ExampleSubscription_ModifyPushConfig() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	if err := sub.ModifyPushConfig(ctx, &pubsub.PushConfig{Endpoint: "https://example.com/push"}); err != nil {
		// TODO: Handle error.
	}
}
<commit_msg>pubsub: improve Receive examples<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub_test

import (
	"fmt"
	"time"

	"cloud.google.com/go/pubsub"
	"golang.org/x/net/context"
	"google.golang.org/api/iterator"
)

func ExampleNewClient() {
	ctx := context.Background()
	_, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	// See the other examples to learn how to use the Client.
}

func ExampleClient_CreateTopic() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	// Create a new topic with the given name.
	topic, err := client.CreateTopic(ctx, "topicName")
	if err != nil {
		// TODO: Handle error.
	}

	_ = topic // TODO: use the topic.
}

func ExampleClient_CreateSubscription() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	// Create a new topic with the given name.
	topic, err := client.CreateTopic(ctx, "topicName")
	if err != nil {
		// TODO: Handle error.
	}

	// Create a new subscription to the previously created topic
	// with the given name.
	sub, err := client.CreateSubscription(ctx, "subName", topic, 10*time.Second, nil)
	if err != nil {
		// TODO: Handle error.
	}

	_ = sub // TODO: use the subscription.
}

func ExampleTopic_Delete() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	topic := client.Topic("topicName")
	if err := topic.Delete(ctx); err != nil {
		// TODO: Handle error.
	}
}

func ExampleTopic_Exists() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	topic := client.Topic("topicName")
	ok, err := topic.Exists(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	if !ok {
		// Topic doesn't exist.
	}
}

func ExampleTopic_Publish() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	topic := client.Topic("topicName")
	defer topic.Stop()
	var results []*pubsub.PublishResult
	r := topic.Publish(ctx, &pubsub.Message{
		Data: []byte("hello world"),
	})
	results = append(results, r)
	// Do other work ...
	for _, r := range results {
		id, err := r.Get(ctx)
		if err != nil {
			// TODO: Handle error.
		}
		fmt.Printf("Published a message with a message ID: %s\n", id)
	}
}

func ExampleTopic_Subscriptions() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	topic := client.Topic("topic-name")
	// List all subscriptions of the topic (maybe of multiple projects).
	for subs := topic.Subscriptions(ctx); ; {
		sub, err := subs.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			// TODO: Handle error.
		}
		_ = sub // TODO: use the subscription.
	}
}

func ExampleSubscription_Delete() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	sub := client.Subscription("subName")
	if err := sub.Delete(ctx); err != nil {
		// TODO: Handle error.
	}
}

func ExampleSubscription_Exists() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}

	sub := client.Subscription("subName")
	ok, err := sub.Exists(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	if !ok {
		// Subscription doesn't exist.
	}
}

func ExampleSubscription_Config() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	config, err := sub.Config(ctx)
	if err != nil {
		// TODO: Handle error.
	}
	fmt.Println(config)
}

func ExampleSubscription_Receive() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
		// TODO: Handle message.
		// NOTE: May be called concurrently; synchronize access to shared memory.
		m.Ack()
	})
	if err != context.Canceled {
		// TODO: Handle error.
	}
}

// This example shows how to configure keepalive so that unacknoweldged messages
// expire quickly, allowing other subscribers to take them.
func ExampleSubscription_Receive_maxExtension() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	// This program is expected to process and acknowledge messages in 30 seconds. If
	// not, the Pub/Sub API will assume the message is not acknowledged.
	sub.ReceiveSettings.MaxExtension = 30 * time.Second
	err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
		// TODO: Handle message.
		m.Ack()
	})
	if err != context.Canceled {
		// TODO: Handle error.
	}
}

// This example shows how to throttle Subscription.Receive, which aims for high
// throughput by default. By limiting the number of messages and/or bytes being
// processed at once, you can bound your program's resource consumption.
func ExampleSubscription_Receive_maxOutstanding() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	sub.ReceiveSettings.MaxOutstandingMessages = 5
	sub.ReceiveSettings.MaxOutstandingBytes = 10e6
	err = sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
		// TODO: Handle message.
		m.Ack()
	})
	if err != context.Canceled {
		// TODO: Handle error.
	}
}

func ExampleSubscription_ModifyPushConfig() {
	ctx := context.Background()
	client, err := pubsub.NewClient(ctx, "project-id")
	if err != nil {
		// TODO: Handle error.
	}
	sub := client.Subscription("subName")
	if err := sub.ModifyPushConfig(ctx, &pubsub.PushConfig{Endpoint: "https://example.com/push"}); err != nil {
		// TODO: Handle error.
	}
}
<|endoftext|>


<commit_before>package quantile

import (
	"testing"
)

func BenchmarkInsert(b *testing.B) {
	b.StopTimer()
	s := NewTargeted(0.01, 0.5, 0.9, 0.99)
	b.StartTimer()
	for i := float64(0); i < float64(b.N); i++ {
		s.Insert(i)
	}
}
<commit_msg>quantile: Add a range of benchmarks including queries<commit_after>package quantile

import (
	"testing"
)

func BenchmarkInsertTargeted(b *testing.B) {
	s := NewTargeted(0.01, 0.5, 0.9, 0.99)
	b.ResetTimer()
	for i := float64(0); i < float64(b.N); i++ {
		s.Insert(i)
	}
}

func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
	s := NewTargeted(0.01, 0.5, 0.9, 0.99)
	s.SetEpsilon(0.0001)
	b.ResetTimer()
	for i := float64(0); i < float64(b.N); i++ {
		s.Insert(i)
	}
}

func BenchmarkInsertBiased(b *testing.B) {
	s := NewBiased()
	b.ResetTimer()
	for i := float64(0); i < float64(b.N); i++ {
		s.Insert(i)
	}
}

func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
	s := NewBiased()
	s.SetEpsilon(0.0001)
	b.ResetTimer()
	for i := float64(0); i < float64(b.N); i++ {
		s.Insert(i)
	}
}

func BenchmarkQuery(b *testing.B) {
	s := NewTargeted(0.01, 0.5, 0.9, 0.99)
	for i := float64(0); i < 1e6; i++ {
		s.Insert(i)
	}
	b.ResetTimer()
	n := float64(b.N)
	for i := float64(0); i < n; i++ {
		s.Query(i / n)
	}
}

func BenchmarkQuerySmallEpsilon(b *testing.B) {
	s := NewTargeted(0.01, 0.5, 0.9, 0.99)
	s.SetEpsilon(0.0001)
	for i := float64(0); i < 1e6; i++ {
		s.Insert(i)
	}
	b.ResetTimer()
	n := float64(b.N)
	for i := float64(0); i < n; i++ {
		s.Query(i / n)
	}
}
<|endoftext|>


<commit_before>package rabbitsmpp

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"time"

	"github.com/streadway/amqp"
	"golang.org/x/net/context"
)

const (
	prefetchCount = 20
	prefetchSize  = 0
	global        = false
)

type Consumer interface {
	Consume() (<-chan Job, <-chan error, error)
	Stop() error
	ID() string
}

type consumer struct {
	*client
	channel Channel
	ctx     context.Context
	cancel  context.CancelFunc
}

func NewConsumer(conf Config) (Consumer, error) {
	c := NewClient(conf).(*client)
	ctx, cancel := context.WithCancel(context.Background())

	return &consumer{
		client:  c,
		channel: nil,
		ctx:     ctx,
		cancel:  cancel,
	}, nil
}

func newConsumerWithContext(ctx context.Context, conf Config) (Consumer, error) {
	c := NewClient(conf).(*client)
	ctx, cancel := context.WithCancel(ctx)

	return &consumer{
		client:  c,
		channel: nil,
		ctx:     ctx,
		cancel:  cancel,
	}, nil
}

func (c *consumer) ID() string {
	return c.client.QueueName()
}

func (c *consumer) bindWithRetry() chan *amqp.Error {
	closeChan, err := c.Bind()
	for err != nil {
		log.Println("Failed to bind consumer:", err)
		time.Sleep(5 * time.Second)
		closeChan, err = c.Bind()
	}
	return closeChan
}

func (c *consumer) getConsumeChannel() (<-chan amqp.Delivery, error) {
	ch, err := c.Channel()
	if err != nil {
		return nil, err
	}
	err = ch.Qos(prefetchCount, prefetchSize, global)
	if err != nil {
		return nil, err
	}

	c.channel = ch

	q, err := c.channel.QueueDeclare(
		c.QueueName(), // name
		true,          // durable
		false,         // delete when unused
		false,         // exclusive
		false,         // no-wait
		nil,           // arguments
	)
	if err != nil {
		return nil, err
	}

	return c.channel.Consume(
		q.Name, // queue
		"",     // consumer
		false,  // auto-ackey
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
}

func (c *consumer) Consume() (<-chan Job, <-chan error, error) {
	if c.channel != nil {
		return nil, nil, errors.New("consumer already active")
	}
	closeChan, err := c.Bind()
	if err != nil {
		return nil, nil, err
	}
	dlvChan, err := c.getConsumeChannel()
	if err != nil {
		return nil, nil, err
	}
	jobChan := make(chan Job)
	errChan := make(chan error)

	go func() {
		defer func() {
			_ = c.channel.Close()
			c.channel = nil
		}()
		defer close(jobChan)
		defer close(errChan)
		for {
			err = c.consume(dlvChan, closeChan, jobChan)
			// if consume returns without an error, means that it was terminated
			// properly, otherwise something went wrong and it needs to restart
			if err == nil {
				log.Printf("EOF consuming for: %s", c.ID())
				return
			}
			errChan <- err
			log.Println("stopped consuming jobs:", err)
			closeChan = c.bindWithRetry()
			dlvChan, err = c.getConsumeChannel()
			for err != nil {
				time.Sleep(5 * time.Second)
				dlvChan, err = c.getConsumeChannel()
			}
		}
	}()

	return jobChan, errChan, nil
}

func (c *consumer) consume(dlvChan <-chan amqp.Delivery, closeChan <-chan *amqp.Error, jobChan chan<- Job) error {
	for {
		select {
		case d := <-dlvChan:
			j := Job{}
			err := json.Unmarshal(d.Body, &j)
			if err != nil {
				return fmt.Errorf("failed to unmarshal PDU: %v", err)
			}
			j.delivery = &d
			jobChan <- j
		case err := <-closeChan:
			return err
		case <-c.ctx.Done():
			return nil
		}
	}
}

func (c *consumer) Stop() error {
	if c.channel == nil {
		return nil
	}
	// Sends the stop signal
	c.cancel()
	return nil
}
<commit_msg>Add functional optional params to constructor<commit_after>package rabbitsmpp

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"time"

	"github.com/streadway/amqp"
	"golang.org/x/net/context"
)

const (
	defaultPrefetchCount = 20
	defaultPrefetchSize  = 0
	defaultGlobalQos     = false
)

type ConsumeOption struct {
	f func(*consumeOptions)
}

type consumeOptions struct {
	prefetchCount int
	prefetchSize  int
	globalQos     bool
}

func SetPrefetchCount(n int) ConsumeOption {
	return ConsumeOption{func(o *consumeOptions) {
		o.prefetchCount = n
	}}
}

func SetPrefetchSize(n int) ConsumeOption {
	return ConsumeOption{func(o *consumeOptions) {
		o.prefetchSize = n
	}}
}

func SetGlobalQos(a bool) ConsumeOption {
	return ConsumeOption{func(o *consumeOptions) {
		o.globalQos = a
	}}
}

type Consumer interface {
	Consume() (<-chan Job, <-chan error, error)
	Stop() error
	ID() string
}

type consumer struct {
	*client
	channel       Channel
	ctx           context.Context
	cancel        context.CancelFunc
	prefetchCount int
	prefetchSize  int
	globalQos     bool
}

func NewConsumer(conf Config, options ...ConsumeOption) (Consumer, error) {
	c := NewClient(conf).(*client)
	ctx, cancel := context.WithCancel(context.Background())
	o := consumeOptions{
		prefetchCount: defaultPrefetchCount,
		prefetchSize:  defaultPrefetchSize,
		globalQos:     defaultGlobalQos,
	}
	for _, option := range options {
		option.f(&o)
	}

	return &consumer{
		client:        c,
		channel:       nil,
		ctx:           ctx,
		cancel:        cancel,
		prefetchCount: o.prefetchCount,
		prefetchSize:  o.prefetchSize,
		globalQos:     o.globalQos,
	}, nil
}

func newConsumerWithContext(ctx context.Context, conf Config) (Consumer, error) {
	c := NewClient(conf).(*client)
	ctx, cancel := context.WithCancel(ctx)

	return &consumer{
		client:  c,
		channel: nil,
		ctx:     ctx,
		cancel:  cancel,
	}, nil
}

func (c *consumer) ID() string {
	return c.client.QueueName()
}

func (c *consumer) bindWithRetry() chan *amqp.Error {
	closeChan, err := c.Bind()
	for err != nil {
		log.Println("Failed to bind consumer:", err)
		time.Sleep(5 * time.Second)
		closeChan, err = c.Bind()
	}
	return closeChan
}

func (c *consumer) getConsumeChannel() (<-chan amqp.Delivery, error) {
	ch, err := c.Channel()
	if err != nil {
		return nil, err
	}
	err = ch.Qos(c.prefetchCount, c.prefetchSize, c.globalQos)
	if err != nil {
		return nil, err
	}

	c.channel = ch

	q, err := c.channel.QueueDeclare(
		c.QueueName(), // name
		true,          // durable
		false,         // delete when unused
		false,         // exclusive
		false,         // no-wait
		nil,           // arguments
	)
	if err != nil {
		return nil, err
	}

	return c.channel.Consume(
		q.Name, // queue
		"",     // consumer
		false,  // auto-ackey
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
}

func (c *consumer) Consume() (<-chan Job, <-chan error, error) {
	if c.channel != nil {
		return nil, nil, errors.New("consumer already active")
	}
	closeChan, err := c.Bind()
	if err != nil {
		return nil, nil, err
	}
	dlvChan, err := c.getConsumeChannel()
	if err != nil {
		return nil, nil, err
	}
	jobChan := make(chan Job)
	errChan := make(chan error)

	go func() {
		defer func() {
			_ = c.channel.Close()
			c.channel = nil
		}()
		defer close(jobChan)
		defer close(errChan)
		for {
			err = c.consume(dlvChan, closeChan, jobChan)
			// if consume returns without an error, means that it was terminated
			// properly, otherwise something went wrong and it needs to restart
			if err == nil {
				log.Printf("EOF consuming for: %s", c.ID())
				return
			}
			errChan <- err
			log.Println("stopped consuming jobs:", err)
			closeChan = c.bindWithRetry()
			dlvChan, err = c.getConsumeChannel()
			for err != nil {
				time.Sleep(5 * time.Second)
				dlvChan, err = c.getConsumeChannel()
			}
		}
	}()

	return jobChan, errChan, nil
}

func (c *consumer) consume(dlvChan <-chan amqp.Delivery, closeChan <-chan *amqp.Error, jobChan chan<- Job) error {
	for {
		select {
		case d := <-dlvChan:
			j := Job{}
			err := json.Unmarshal(d.Body, &j)
			if err != nil {
				return fmt.Errorf("failed to unmarshal PDU: %v", err)
			}
			j.delivery = &d
			jobChan <- j
		case err := <-closeChan:
			return err
		case <-c.ctx.Done():
			return nil
		}
	}
}

func (c *consumer) Stop() error {
	if c.channel == nil {
		return nil
	}
	// Sends the stop signal
	c.cancel()
	return nil
}
<|endoftext|>


<commit_before>package pwr

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/Datadog/zstd"
	"github.com/alecthomas/assert"
	humanize "github.com/dustin/go-humanize"
	"github.com/itchio/wharf/bsdiff"
	"github.com/itchio/wharf/pools/fspool"
	"github.com/itchio/wharf/state"
	"github.com/itchio/wharf/tlc"
)

type zstdCompressor struct{}

func (zc *zstdCompressor) Apply(writer io.Writer, quality int32) (io.Writer, error) {
	return zstd.NewWriterLevel(writer, int(quality)), nil
}

type zstdDecompressor struct{}

func (zd *zstdDecompressor) Apply(reader io.Reader) (io.Reader, error) {
	return zstd.NewReader(reader), nil
}

func init() {
	RegisterCompressor(CompressionAlgorithm_ZSTD, &zstdCompressor{})
	RegisterDecompressor(CompressionAlgorithm_ZSTD, &zstdDecompressor{})
}

func Test_RediffOneSeq(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff when one byte gets changed",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
	})
}

func Test_RediffOneInt(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff when one byte gets changed",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
		partitions: 2,
	})
}

func Test_RediffWorse(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff gets slightly worse",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize*2 + 14},
				{path: "file-1", seed: 0x2},
				{path: "dir2/file-2", seed: 0x3},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize*3 + 14},
				{path: "file-1", seed: 0x2},
				{path: "dir2/file-2", seed: 0x33},
			},
		},
	})
}

func Test_RediffBetter(t *testing.T) {
	for _, partitions := range []int{0, 2, 4, 8} {
		runRediffScenario(t, patchScenario{
			name:         "rediff gets better!",
			touchedFiles: 1,
			deletedFiles: 0,
			v1: testDirSettings{
				entries: []testDirEntry{
					{path: "subdir/file-1", seed: 0x1, size: BlockSize*500 + 14},
					{path: "file-1", seed: 0x2},
					{path: "dir2/file-2", seed: 0x3},
				},
			},
			v2: testDirSettings{
				entries: []testDirEntry{
					{path: "subdir/file-1", seed: 0x1, size: BlockSize*500 + 14, bsmods: []bsmod{
						bsmod{interval: BlockSize/2 + 3, delta: 0x4},
						bsmod{interval: BlockSize/3 + 7, delta: 0x18},
					}},
					{path: "file-1", seed: 0x2},
					{path: "dir2/file-2", seed: 0x33},
				},
			},
			partitions: partitions,
		})
	}
}

func Test_RediffStillBetter(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff gets better even though rsync wasn't that bad",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize*5 + 14},
				{path: "file-1", seed: 0x2, size: BlockSize * 4},
				{path: "dir2/file-2", seed: 0x3},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize * 6, bsmods: []bsmod{
					bsmod{interval: BlockSize/7 + 3, delta: 0x4, max: 4, skip: 20},
					bsmod{interval: BlockSize/13 + 7, delta: 0x18, max: 6, skip: 20},
				}},
				{path: "file-1", chunks: []testDirChunk{
					testDirChunk{size: BlockSize*2 + 3, seed: 0x99},
					testDirChunk{size: BlockSize*1 + 12, seed: 0x2},
				}},
				{path: "dir2/file-2", seed: 0x33},
			},
		},
	})
}

func runRediffScenario(t *testing.T, scenario patchScenario) {
	log := t.Logf

	mainDir, err := ioutil.TempDir("", "rediff")
	assert.NoError(t, err)
	assert.NoError(t, os.MkdirAll(mainDir, 0755))
	defer os.RemoveAll(mainDir)

	v1 := filepath.Join(mainDir, "v1")
	makeTestDir(t, v1, scenario.v1)

	v2 := filepath.Join(mainDir, "v2")
	makeTestDir(t, v2, scenario.v2)

	compression := &CompressionSettings{}
	compression.Algorithm = CompressionAlgorithm_ZSTD
	compression.Quality = 1

	sourceContainer, err := tlc.WalkAny(v2, nil)
	assert.NoError(t, err)

	consumer := &state.Consumer{
		OnMessage: func(level string, message string) {
			// log("[%s] %s", level, message)
		},
	}
	patchBuffer := new(bytes.Buffer)
	signatureBuffer := new(bytes.Buffer)

	func() {
		targetContainer, dErr := tlc.WalkAny(v1, nil)
		assert.NoError(t, dErr)

		targetPool := fspool.New(targetContainer, v1)
		targetSignature, dErr := ComputeSignature(targetContainer, targetPool, consumer)
		assert.NoError(t, dErr)

		pool := fspool.New(sourceContainer, v2)

		dctx := &DiffContext{
			Compression: compression,
			Consumer:    consumer,

			SourceContainer: sourceContainer,
			Pool:            pool,

			TargetContainer: targetContainer,
			TargetSignature: targetSignature,
		}

		assert.NoError(t, dctx.WritePatch(patchBuffer, signatureBuffer))
	}()

	signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
	assert.NoError(t, sErr)

	v1Before := filepath.Join(mainDir, "v1Before")
	cpDir(t, v1, v1Before)

	v1After := filepath.Join(mainDir, "v1After")

	assert.NoError(t, os.RemoveAll(v1Before))
	cpDir(t, v1, v1Before)

	func() {
		actx := &ApplyContext{
			TargetPath: v1Before,
			OutputPath: v1After,

			Consumer: consumer,
		}

		patchReader := bytes.NewReader(patchBuffer.Bytes())

		aErr := actx.ApplyPatch(patchReader)
		assert.NoError(t, aErr)

		signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
		assert.NoError(t, sErr)
		assert.NoError(t, AssertValid(v1After, signature))
		log("Original patch applies cleanly.")
	}()

	func() {
		targetContainer, dErr := tlc.WalkAny(v1, nil)
		assert.NoError(t, dErr)

		bsdiffStats := &bsdiff.DiffStats{}

		rc := &RediffContext{
			TargetPool: fspool.New(targetContainer, v1),
			SourcePool: fspool.New(sourceContainer, v2),

			Consumer:              consumer,
			Compression:           compression,
			SuffixSortConcurrency: 0,
			Partitions:            scenario.partitions,

			BsdiffStats: bsdiffStats,
		}

		patchReader := bytes.NewReader(patchBuffer.Bytes())

		log("Optimizing...")
		aErr := rc.AnalyzePatch(patchReader)
		assert.NoError(t, aErr)

		patchReader.Seek(0, os.SEEK_SET)
		optimizedPatchBuffer := new(bytes.Buffer)

		beforeOptimize := time.Now()
		oErr := rc.OptimizePatch(patchReader, optimizedPatchBuffer)
		assert.NoError(t, oErr)
		log("Optimized patch in %s (spent %s sorting, %s scanning, %s writing)",
			time.Since(beforeOptimize),
			bsdiffStats.TimeSpentSorting,
			bsdiffStats.TimeSpentScanning,
			bsdiffStats.TimeSpentWriting,
		)

		before := patchBuffer.Len()
		after := optimizedPatchBuffer.Len()

		diff := (float64(after) - float64(before)) / float64(before) * 100
		if diff < 0 {
			log("Patch is %.2f%% smaller (%s < %s)", -diff, humanize.IBytes(uint64(after)), humanize.IBytes(uint64(before)))
		} else {
			log("Patch is %.2f%% larger (%s > %s)", diff, humanize.IBytes(uint64(after)), humanize.IBytes(uint64(before)))
		}

		assert.NoError(t, os.RemoveAll(v1Before))
		cpDir(t, v1, v1Before)

		func() {
			actx := &ApplyContext{
				TargetPath: v1Before,
				OutputPath: v1After,

				Consumer: consumer,
			}

			patchReader := bytes.NewReader(optimizedPatchBuffer.Bytes())

			aErr := actx.ApplyPatch(patchReader)
			assert.NoError(t, aErr)

			assert.NoError(t, AssertValid(v1After, signature))
			log("Optimized patch applies cleanly.")
		}()
	}()
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>Make rediff tests smaller again so they pass<commit_after>package pwr

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/Datadog/zstd"
	"github.com/alecthomas/assert"
	humanize "github.com/dustin/go-humanize"
	"github.com/itchio/wharf/bsdiff"
	"github.com/itchio/wharf/pools/fspool"
	"github.com/itchio/wharf/state"
	"github.com/itchio/wharf/tlc"
)

type zstdCompressor struct{}

func (zc *zstdCompressor) Apply(writer io.Writer, quality int32) (io.Writer, error) {
	return zstd.NewWriterLevel(writer, int(quality)), nil
}

type zstdDecompressor struct{}

func (zd *zstdDecompressor) Apply(reader io.Reader) (io.Reader, error) {
	return zstd.NewReader(reader), nil
}

func init() {
	RegisterCompressor(CompressionAlgorithm_ZSTD, &zstdCompressor{})
	RegisterDecompressor(CompressionAlgorithm_ZSTD, &zstdDecompressor{})
}

func Test_RediffOneSeq(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff when one byte gets changed",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
	})
}

func Test_RediffOneInt(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff when one byte gets changed",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "file", data: []byte{
					0x00,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
					0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
				}},
			},
		},
		partitions: 2,
	})
}

func Test_RediffWorse(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff gets slightly worse",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize*2 + 14},
				{path: "file-1", seed: 0x2},
				{path: "dir2/file-2", seed: 0x3},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize*3 + 14},
				{path: "file-1", seed: 0x2},
				{path: "dir2/file-2", seed: 0x33},
			},
		},
	})
}

func Test_RediffBetter(t *testing.T) {
	for _, partitions := range []int{0, 2, 4, 8} {
		runRediffScenario(t, patchScenario{
			name:         "rediff gets better!",
			touchedFiles: 1,
			deletedFiles: 0,
			v1: testDirSettings{
				entries: []testDirEntry{
					{path: "subdir/file-1", seed: 0x1, size: BlockSize*5 + 14},
					{path: "file-1", seed: 0x2},
					{path: "dir2/file-2", seed: 0x3},
				},
			},
			v2: testDirSettings{
				entries: []testDirEntry{
					{path: "subdir/file-1", seed: 0x1, size: BlockSize*5 + 14, bsmods: []bsmod{
						bsmod{interval: BlockSize/2 + 3, delta: 0x4},
						bsmod{interval: BlockSize/3 + 7, delta: 0x18},
					}},
					{path: "file-1", seed: 0x2},
					{path: "dir2/file-2", seed: 0x33},
				},
			},
			partitions: partitions,
		})
	}
}

func Test_RediffStillBetter(t *testing.T) {
	runRediffScenario(t, patchScenario{
		name:         "rediff gets better even though rsync wasn't that bad",
		touchedFiles: 1,
		deletedFiles: 0,
		v1: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize*5 + 14},
				{path: "file-1", seed: 0x2, size: BlockSize * 4},
				{path: "dir2/file-2", seed: 0x3},
			},
		},
		v2: testDirSettings{
			entries: []testDirEntry{
				{path: "subdir/file-1", seed: 0x1, size: BlockSize * 6, bsmods: []bsmod{
					bsmod{interval: BlockSize/7 + 3, delta: 0x4, max: 4, skip: 20},
					bsmod{interval: BlockSize/13 + 7, delta: 0x18, max: 6, skip: 20},
				}},
				{path: "file-1", chunks: []testDirChunk{
					testDirChunk{size: BlockSize*2 + 3, seed: 0x99},
					testDirChunk{size: BlockSize*1 + 12, seed: 0x2},
				}},
				{path: "dir2/file-2", seed: 0x33},
			},
		},
	})
}

func runRediffScenario(t *testing.T, scenario patchScenario) {
	log := t.Logf

	mainDir, err := ioutil.TempDir("", "rediff")
	assert.NoError(t, err)
	assert.NoError(t, os.MkdirAll(mainDir, 0755))
	defer os.RemoveAll(mainDir)

	v1 := filepath.Join(mainDir, "v1")
	makeTestDir(t, v1, scenario.v1)

	v2 := filepath.Join(mainDir, "v2")
	makeTestDir(t, v2, scenario.v2)

	compression := &CompressionSettings{}
	compression.Algorithm = CompressionAlgorithm_ZSTD
	compression.Quality = 1

	sourceContainer, err := tlc.WalkAny(v2, nil)
	assert.NoError(t, err)

	consumer := &state.Consumer{
		OnMessage: func(level string, message string) {
			// log("[%s] %s", level, message)
		},
	}
	patchBuffer := new(bytes.Buffer)
	signatureBuffer := new(bytes.Buffer)

	func() {
		targetContainer, dErr := tlc.WalkAny(v1, nil)
		assert.NoError(t, dErr)

		targetPool := fspool.New(targetContainer, v1)
		targetSignature, dErr := ComputeSignature(targetContainer, targetPool, consumer)
		assert.NoError(t, dErr)

		pool := fspool.New(sourceContainer, v2)

		dctx := &DiffContext{
			Compression: compression,
			Consumer:    consumer,

			SourceContainer: sourceContainer,
			Pool:            pool,

			TargetContainer: targetContainer,
			TargetSignature: targetSignature,
		}

		assert.NoError(t, dctx.WritePatch(patchBuffer, signatureBuffer))
	}()

	signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
	assert.NoError(t, sErr)

	v1Before := filepath.Join(mainDir, "v1Before")
	cpDir(t, v1, v1Before)

	v1After := filepath.Join(mainDir, "v1After")

	assert.NoError(t, os.RemoveAll(v1Before))
	cpDir(t, v1, v1Before)

	func() {
		actx := &ApplyContext{
			TargetPath: v1Before,
			OutputPath: v1After,

			Consumer: consumer,
		}

		patchReader := bytes.NewReader(patchBuffer.Bytes())

		aErr := actx.ApplyPatch(patchReader)
		assert.NoError(t, aErr)

		signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
		assert.NoError(t, sErr)
		assert.NoError(t, AssertValid(v1After, signature))
		log("Original patch applies cleanly.")
	}()

	func() {
		targetContainer, dErr := tlc.WalkAny(v1, nil)
		assert.NoError(t, dErr)

		bsdiffStats := &bsdiff.DiffStats{}

		rc := &RediffContext{
			TargetPool: fspool.New(targetContainer, v1),
			SourcePool: fspool.New(sourceContainer, v2),

			Consumer:              consumer,
			Compression:           compression,
			SuffixSortConcurrency: 0,
			Partitions:            scenario.partitions,

			BsdiffStats: bsdiffStats,
		}

		patchReader := bytes.NewReader(patchBuffer.Bytes())

		log("Optimizing...")
		aErr := rc.AnalyzePatch(patchReader)
		assert.NoError(t, aErr)

		patchReader.Seek(0, os.SEEK_SET)
		optimizedPatchBuffer := new(bytes.Buffer)

		beforeOptimize := time.Now()
		oErr := rc.OptimizePatch(patchReader, optimizedPatchBuffer)
		assert.NoError(t, oErr)
		log("Optimized patch in %s (spent %s sorting, %s scanning, %s writing)",
			time.Since(beforeOptimize),
			bsdiffStats.TimeSpentSorting,
			bsdiffStats.TimeSpentScanning,
			bsdiffStats.TimeSpentWriting,
		)

		before := patchBuffer.Len()
		after := optimizedPatchBuffer.Len()

		diff := (float64(after) - float64(before)) / float64(before) * 100
		if diff < 0 {
			log("Patch is %.2f%% smaller (%s < %s)", -diff, humanize.IBytes(uint64(after)), humanize.IBytes(uint64(before)))
		} else {
			log("Patch is %.2f%% larger (%s > %s)", diff, humanize.IBytes(uint64(after)), humanize.IBytes(uint64(before)))
		}

		assert.NoError(t, os.RemoveAll(v1Before))
		cpDir(t, v1, v1Before)

		func() {
			actx := &ApplyContext{
				TargetPath: v1Before,
				OutputPath: v1After,

				Consumer: consumer,
			}

			patchReader := bytes.NewReader(optimizedPatchBuffer.Bytes())

			aErr := actx.ApplyPatch(patchReader)
			assert.NoError(t, aErr)

			assert.NoError(t, AssertValid(v1After, signature))
			log("Optimized patch applies cleanly.")
		}()
	}()
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"syscall"

	"github.com/bitly/go-nsq"
	"github.com/nulayer/qmd/common"
	"github.com/rcrowley/go-tigertonic"
)

var (
	listen   = flag.String("listen", "0.0.0.0:8080", "listen address")
	queue    = flag.String("queue", "127.0.0.1:4151", "queue address")
	topic    = flag.String("topic", "jobs", "queue topic")
	mux      *tigertonic.TrieServeMux
	producer *nsq.Producer
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: qmd-server [-listen=<listen>] [-queue=<queue>] [-topic=<topic>]")
		flag.PrintDefaults()
	}

	// Register the endpoints.
	mux = tigertonic.NewTrieServeMux()
	mux.Handle("POST", "/job", tigertonic.Marshaled(create))
	mux.Handle("GET", "/job/{id}", tigertonic.Marshaled(get))
}

func main() {
	flag.Parse()

	producer = nsq.NewProducer(*queue, nsq.NewConfig())
	fmt.Printf("Sending to %s\n", producer.String())

	server := tigertonic.NewServer(*listen, mux)
	fmt.Printf("Listening on %s\n", *listen)
	go func() {
		var err error
		err = server.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
	}()

	termChan := make(chan os.Signal, 1)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
	log.Println(<-termChan)
	fmt.Println("Safely shutting down server now...")
	server.Close()
}

// POST /job
func create(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {

	var err error

	data, err := json.Marshal(rq.Scripts)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(data)

	doneChan := make(chan *nsq.ProducerTransaction)
	err = producer.PublishAsync(*topic, data, doneChan, nil)
	if err != nil {
		fmt.Println(err)
	}
	log.Println(<-doneChan)

	return http.StatusCreated, http.Header{
		"Content-Location": {fmt.Sprintf(
			"%s://%s/1.0/job/%s",
			u.Scheme,
			u.Host,
			rq.ID,
		)},
	}, &common.JobResponse{rq.ID, rq.Scripts}, nil
}

// GET /job/{id}
func get(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {

	// TODO: Figure out how to request a job through the queue.

	return http.StatusOK, nil, &common.JobResponse{
		u.Query().Get("id"),
		nil,
	}, nil
}
<commit_msg>Changed queue address for server to sane default.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"syscall"

	"github.com/bitly/go-nsq"
	"github.com/nulayer/qmd/common"
	"github.com/rcrowley/go-tigertonic"
)

var (
	listen   = flag.String("listen", "0.0.0.0:8080", "listen address")
	queue    = flag.String("queue", "127.0.0.1:4150", "queue address")
	topic    = flag.String("topic", "jobs", "queue topic")
	mux      *tigertonic.TrieServeMux
	producer *nsq.Producer
)

func init() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: qmd-server [-listen=<listen>] [-queue=<queue>] [-topic=<topic>]")
		flag.PrintDefaults()
	}

	// Register the endpoints.
	mux = tigertonic.NewTrieServeMux()
	mux.Handle("POST", "/job", tigertonic.Marshaled(create))
	mux.Handle("GET", "/job/{id}", tigertonic.Marshaled(get))
}

func main() {
	flag.Parse()

	producer = nsq.NewProducer(*queue, nsq.NewConfig())
	fmt.Printf("Sending to %s\n", producer.String())

	server := tigertonic.NewServer(*listen, mux)
	fmt.Printf("Listening on %s\n", *listen)
	go func() {
		var err error
		err = server.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
	}()

	termChan := make(chan os.Signal, 1)
	signal.Notify(termChan, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
	log.Println(<-termChan)
	fmt.Println("Safely shutting down server now...")
	server.Close()
}

// POST /job
func create(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {

	var err error

	data, err := json.Marshal(rq.Scripts)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(data)

	doneChan := make(chan *nsq.ProducerTransaction)
	err = producer.PublishAsync(*topic, data, doneChan, nil)
	if err != nil {
		fmt.Println(err)
	}
	log.Println(<-doneChan)

	return http.StatusCreated, http.Header{
		"Content-Location": {fmt.Sprintf(
			"%s://%s/1.0/job/%s",
			u.Scheme,
			u.Host,
			rq.ID,
		)},
	}, &common.JobResponse{rq.ID, rq.Scripts}, nil
}

// GET /job/{id}
func get(u *url.URL, h http.Header, rq *common.JobRequest) (int, http.Header, *common.JobResponse, error) {

	// TODO: Figure out how to request a job through the queue.

	return http.StatusOK, nil, &common.JobResponse{
		u.Query().Get("id"),
		nil,
	}, nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package rafthttp

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
)

const (
	connPerSender = 4
	senderBufSize = connPerSender * 4

	appRespBatchMs = 50

	ConnReadTimeout  = 5 * time.Second
	ConnWriteTimeout = 5 * time.Second
)

type Sender interface {
	// StartStreaming enables streaming in the sender using the given writer,
	// which provides a fast and efficient way to send appendEntry messages.
	StartStreaming(w WriteFlusher, to types.ID, term uint64) (done <-chan struct{}, err error)
	Update(u string)
	// Send sends the data to the remote node. It is always non-blocking.
	// It may be fail to send data if it returns nil error.
	Send(m raftpb.Message) error
	// Stop performs any necessary finalization and terminates the Sender
	// elegantly.
	Stop()

	// Pause pauses the sender. The sender will simply drops all incoming
	// messages without retruning an error.
	Pause()

	// Resume resumes a paused sender.
	Resume()
}

func NewSender(tr http.RoundTripper, u string, cid types.ID, p Processor, fs *stats.FollowerStats, shouldstop chan struct{}) *sender {
	s := &sender{
		tr:         tr,
		u:          u,
		cid:        cid,
		p:          p,
		fs:         fs,
		shouldstop: shouldstop,
		batcher:    NewBatcher(100, appRespBatchMs*time.Millisecond),
		q:          make(chan []byte, senderBufSize),
	}
	s.wg.Add(connPerSender)
	for i := 0; i < connPerSender; i++ {
		go s.handle()
	}
	return s
}

type sender struct {
	tr         http.RoundTripper
	u          string
	cid        types.ID
	p          Processor
	fs         *stats.FollowerStats
	shouldstop chan struct{}

	strmCln   *streamClient
	batcher   *Batcher
	strmSrv   *streamServer
	strmSrvMu sync.Mutex
	q         chan []byte

	paused bool
	mu     sync.RWMutex
	wg     sync.WaitGroup
}

func (s *sender) StartStreaming(w WriteFlusher, to types.ID, term uint64) (<-chan struct{}, error) {
	s.strmSrvMu.Lock()
	defer s.strmSrvMu.Unlock()
	if s.strmSrv != nil {
		// ignore lower-term streaming request
		if term < s.strmSrv.term {
			return nil, fmt.Errorf("out of data streaming request: term %d, request term %d", term, s.strmSrv.term)
		}
		// stop the existing one
		s.strmSrv.stop()
	}
	s.strmSrv = startStreamServer(w, to, term, s.fs)
	return s.strmSrv.stopNotify(), nil
}

func (s *sender) Update(u string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.u = u
}

// TODO (xiangli): reasonable retry logic
func (s *sender) Send(m raftpb.Message) error {
	s.mu.RLock()
	pause := s.paused
	s.mu.RUnlock()
	if pause {
		return nil
	}

	s.maybeStopStream(m.Term)
	if shouldInitStream(m) && !s.hasStreamClient() {
		s.initStream(types.ID(m.From), types.ID(m.To), m.Term)
		s.batcher.Reset(time.Now())
	}
	if canBatch(m) && s.hasStreamClient() {
		if s.batcher.ShouldBatch(time.Now()) {
			return nil
		}
	}
	if canUseStream(m) {
		if ok := s.tryStream(m); ok {
			return nil
		}
	}
	// TODO: don't block. we should be able to have 1000s
	// of messages out at a time.
	data := pbutil.MustMarshal(&m)
	select {
	case s.q <- data:
		return nil
	default:
		log.Printf("sender: reach the maximal serving to %s", s.u)
		return fmt.Errorf("reach maximal serving")
	}
}

func (s *sender) Stop() {
	close(s.q)
	s.wg.Wait()
	s.strmSrvMu.Lock()
	if s.strmSrv != nil {
		s.strmSrv.stop()
	}
	s.strmSrvMu.Unlock()
	if s.strmCln != nil {
		s.strmCln.stop()
	}
}

func (s *sender) Pause() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.paused = true
}

func (s *sender) Resume() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.paused = false
}

func (s *sender) maybeStopStream(term uint64) {
	if s.strmCln != nil && term > s.strmCln.term {
		s.strmCln.stop()
		s.strmCln = nil
	}
	s.strmSrvMu.Lock()
	defer s.strmSrvMu.Unlock()
	if s.strmSrv != nil && term > s.strmSrv.term {
		s.strmSrv.stop()
		s.strmSrv = nil
	}
}

func (s *sender) hasStreamClient() bool {
	return s.strmCln != nil && !s.strmCln.isStopped()
}

func (s *sender) initStream(from, to types.ID, term uint64) {
	strmCln := newStreamClient(from, to, term, s.p)
	s.mu.Lock()
	u := s.u
	s.mu.Unlock()
	if err := strmCln.start(s.tr, u, s.cid); err != nil {
		log.Printf("rafthttp: start stream client error: %v", err)
		return
	}
	s.strmCln = strmCln
}

func (s *sender) tryStream(m raftpb.Message) bool {
	s.strmSrvMu.Lock()
	defer s.strmSrvMu.Unlock()
	if s.strmSrv == nil || m.Term != s.strmSrv.term {
		return false
	}
	if err := s.strmSrv.send(m.Entries); err != nil {
		log.Printf("rafthttp: send stream message error: %v", err)
		s.strmSrv.stop()
		s.strmSrv = nil
		return false
	}
	return true
}

func (s *sender) handle() {
	defer s.wg.Done()
	for d := range s.q {
		start := time.Now()
		err := s.post(d)
		end := time.Now()
		if err != nil {
			s.fs.Fail()
			log.Printf("sender: %v", err)
			continue
		}
		s.fs.Succ(end.Sub(start))
	}
}

// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (s *sender) post(data []byte) error {
	s.mu.RLock()
	req, err := http.NewRequest("POST", s.u, bytes.NewBuffer(data))
	s.mu.RUnlock()
	if err != nil {
		return fmt.Errorf("new request to %s error: %v", s.u, err)
	}
	req.Header.Set("Content-Type", "application/protobuf")
	req.Header.Set("X-Etcd-Cluster-ID", s.cid.String())
	resp, err := s.tr.RoundTrip(req)
	if err != nil {
		return fmt.Errorf("error posting to %q: %v", req.URL.String(), err)
	}
	resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusPreconditionFailed:
		select {
		case s.shouldstop <- struct{}{}:
		default:
		}
		log.Printf("etcdserver: conflicting cluster ID with the target cluster (%s != %s)", resp.Header.Get("X-Etcd-Cluster-ID"), s.cid)
		return nil
	case http.StatusForbidden:
		select {
		case s.shouldstop <- struct{}{}:
		default:
		}
		log.Println("etcdserver: this member has been permanently removed from the cluster")
		log.Println("etcdserver: the data-dir used by this member must be removed so that this host can be re-added with a new member ID")
		return nil
	case http.StatusNoContent:
		return nil
	default:
		return fmt.Errorf("unhandled status %s", http.StatusText(resp.StatusCode))
	}
}
<commit_msg>rafthttp: increase sender buffer size<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package rafthttp

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
)

const (
	connPerSender = 4
	// senderBufSize is the size of sender buffer, which helps hold the
	// temporary network latency.
	// The size ensures that sender does not drop messages when the network
	// is out of work for less than 1 second in good path.
	senderBufSize = 64

	appRespBatchMs = 50

	ConnReadTimeout  = 5 * time.Second
	ConnWriteTimeout = 5 * time.Second
)

type Sender interface {
	// StartStreaming enables streaming in the sender using the given writer,
	// which provides a fast and efficient way to send appendEntry messages.
	StartStreaming(w WriteFlusher, to types.ID, term uint64) (done <-chan struct{}, err error)
	Update(u string)
	// Send sends the data to the remote node. It is always non-blocking.
	// It may be fail to send data if it returns nil error.
	Send(m raftpb.Message) error
	// Stop performs any necessary finalization and terminates the Sender
	// elegantly.
	Stop()

	// Pause pauses the sender. The sender will simply drops all incoming
	// messages without retruning an error.
	Pause()

	// Resume resumes a paused sender.
	Resume()
}

func NewSender(tr http.RoundTripper, u string, cid types.ID, p Processor, fs *stats.FollowerStats, shouldstop chan struct{}) *sender {
	s := &sender{
		tr:         tr,
		u:          u,
		cid:        cid,
		p:          p,
		fs:         fs,
		shouldstop: shouldstop,
		batcher:    NewBatcher(100, appRespBatchMs*time.Millisecond),
		q:          make(chan []byte, senderBufSize),
	}
	s.wg.Add(connPerSender)
	for i := 0; i < connPerSender; i++ {
		go s.handle()
	}
	return s
}

type sender struct {
	tr         http.RoundTripper
	u          string
	cid        types.ID
	p          Processor
	fs         *stats.FollowerStats
	shouldstop chan struct{}

	strmCln   *streamClient
	batcher   *Batcher
	strmSrv   *streamServer
	strmSrvMu sync.Mutex
	q         chan []byte

	paused bool
	mu     sync.RWMutex
	wg     sync.WaitGroup
}

func (s *sender) StartStreaming(w WriteFlusher, to types.ID, term uint64) (<-chan struct{}, error) {
	s.strmSrvMu.Lock()
	defer s.strmSrvMu.Unlock()
	if s.strmSrv != nil {
		// ignore lower-term streaming request
		if term < s.strmSrv.term {
			return nil, fmt.Errorf("out of data streaming request: term %d, request term %d", term, s.strmSrv.term)
		}
		// stop the existing one
		s.strmSrv.stop()
	}
	s.strmSrv = startStreamServer(w, to, term, s.fs)
	return s.strmSrv.stopNotify(), nil
}

func (s *sender) Update(u string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.u = u
}

// TODO (xiangli): reasonable retry logic
func (s *sender) Send(m raftpb.Message) error {
	s.mu.RLock()
	pause := s.paused
	s.mu.RUnlock()
	if pause {
		return nil
	}

	s.maybeStopStream(m.Term)
	if shouldInitStream(m) && !s.hasStreamClient() {
		s.initStream(types.ID(m.From), types.ID(m.To), m.Term)
		s.batcher.Reset(time.Now())
	}
	if canBatch(m) && s.hasStreamClient() {
		if s.batcher.ShouldBatch(time.Now()) {
			return nil
		}
	}
	if canUseStream(m) {
		if ok := s.tryStream(m); ok {
			return nil
		}
	}
	// TODO: don't block. we should be able to have 1000s
	// of messages out at a time.
	data := pbutil.MustMarshal(&m)
	select {
	case s.q <- data:
		return nil
	default:
		log.Printf("sender: reach the maximal serving to %s", s.u)
		return fmt.Errorf("reach maximal serving")
	}
}

func (s *sender) Stop() {
	close(s.q)
	s.wg.Wait()
	s.strmSrvMu.Lock()
	if s.strmSrv != nil {
		s.strmSrv.stop()
	}
	s.strmSrvMu.Unlock()
	if s.strmCln != nil {
		s.strmCln.stop()
	}
}

func (s *sender) Pause() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.paused = true
}

func (s *sender) Resume() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.paused = false
}

func (s *sender) maybeStopStream(term uint64) {
	if s.strmCln != nil && term > s.strmCln.term {
		s.strmCln.stop()
		s.strmCln = nil
	}
	s.strmSrvMu.Lock()
	defer s.strmSrvMu.Unlock()
	if s.strmSrv != nil && term > s.strmSrv.term {
		s.strmSrv.stop()
		s.strmSrv = nil
	}
}

func (s *sender) hasStreamClient() bool {
	return s.strmCln != nil && !s.strmCln.isStopped()
}

func (s *sender) initStream(from, to types.ID, term uint64) {
	strmCln := newStreamClient(from, to, term, s.p)
	s.mu.Lock()
	u := s.u
	s.mu.Unlock()
	if err := strmCln.start(s.tr, u, s.cid); err != nil {
		log.Printf("rafthttp: start stream client error: %v", err)
		return
	}
	s.strmCln = strmCln
}

func (s *sender) tryStream(m raftpb.Message) bool {
	s.strmSrvMu.Lock()
	defer s.strmSrvMu.Unlock()
	if s.strmSrv == nil || m.Term != s.strmSrv.term {
		return false
	}
	if err := s.strmSrv.send(m.Entries); err != nil {
		log.Printf("rafthttp: send stream message error: %v", err)
		s.strmSrv.stop()
		s.strmSrv = nil
		return false
	}
	return true
}

func (s *sender) handle() {
	defer s.wg.Done()
	for d := range s.q {
		start := time.Now()
		err := s.post(d)
		end := time.Now()
		if err != nil {
			s.fs.Fail()
			log.Printf("sender: %v", err)
			continue
		}
		s.fs.Succ(end.Sub(start))
	}
}

// post POSTs a data payload to a url. Returns nil if the POST succeeds,
// error on any failure.
func (s *sender) post(data []byte) error {
	s.mu.RLock()
	req, err := http.NewRequest("POST", s.u, bytes.NewBuffer(data))
	s.mu.RUnlock()
	if err != nil {
		return fmt.Errorf("new request to %s error: %v", s.u, err)
	}
	req.Header.Set("Content-Type", "application/protobuf")
	req.Header.Set("X-Etcd-Cluster-ID", s.cid.String())
	resp, err := s.tr.RoundTrip(req)
	if err != nil {
		return fmt.Errorf("error posting to %q: %v", req.URL.String(), err)
	}
	resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusPreconditionFailed:
		select {
		case s.shouldstop <- struct{}{}:
		default:
		}
		log.Printf("etcdserver: conflicting cluster ID with the target cluster (%s != %s)", resp.Header.Get("X-Etcd-Cluster-ID"), s.cid)
		return nil
	case http.StatusForbidden:
		select {
		case s.shouldstop <- struct{}{}:
		default:
		}
		log.Println("etcdserver: this member has been permanently removed from the cluster")
		log.Println("etcdserver: the data-dir used by this member must be removed so that this host can be re-added with a new member ID")
		return nil
	case http.StatusNoContent:
		return nil
	default:
		return fmt.Errorf("unhandled status %s", http.StatusText(resp.StatusCode))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"sync"

	"github.com/anaminus/drill"
	"github.com/anaminus/drill/filesys"
	"github.com/anaminus/drill/filesys/markdown"
	"github.com/anaminus/rbxmk/fragments"
	"github.com/anaminus/rbxmk/rbxmk/term"
	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/extension"
	terminal "golang.org/x/term"
)

// Language determines the language of documentation text.
var Language = "en-us"

// panicLanguage writes the languages that have been embedded, then panics.
func panicLanguage() {
	langs := make([]string, 0, len(fragments.Languages))
	for lang := range fragments.Languages {
		langs = append(langs, lang)
	}
	sort.Strings(langs)
	if len(langs) == 0 {
		fmt.Fprintln(os.Stderr, "no languages are embedded")
	} else {
		fmt.Fprintln(os.Stderr, "the following languages are embedded:")
		for _, lang := range langs {
			fmt.Fprintf(os.Stderr, "\t%s\n", lang)
		}
	}
	panic(fmt.Sprintf("unsupported language %q", Language))
}

func initDocs() drill.Node {
	termWidth, _, _ := terminal.GetSize(int(os.Stdout.Fd()))
	lang, ok := fragments.Languages[Language]
	if !ok {
		panicLanguage()
	}
	f, err := filesys.NewFS(lang, filesys.Handlers{
		{Pattern: "*.md", Func: markdown.NewHandler(
			goldmark.WithRenderer(term.Renderer{Width: termWidth, TabSize: 4}),
			goldmark.WithExtensions(
				extension.Table,
				extension.Footnote,
			),
		)},
	})
	if err != nil {
		panic(err)
	}
	node := f.UnorderedChild(Language)
	if node == nil {
		panicLanguage()
	}
	return node
}

var docfs = initDocs()
var docMut sync.RWMutex
var docCache = map[string]*markdown.Node{}
var docFailed = map[string]struct{}{}
var docSeen = map[string]struct{}{}

// DocFragments returns a list of requested fragments.
func DocFragments() []string {
	docMut.RLock()
	defer docMut.RUnlock()
	frags := make([]string, 0, len(docSeen))
	for frag := range docSeen {
		frags = append(frags, frag)
	}
	sort.Strings(frags)
	return frags
}

// docString extracts and formats the fragment from the given node. Panics if
// the node was not found.
func docString(fragpath string, node drill.Node) string {
	if node == nil {
		docFailed[fragpath] = struct{}{}
		return "{" + Language + ":" + fragpath + "}"
	}
	return strings.TrimSpace(node.Fragment())
}

// UnresolvedFragments writes to stderr a list of fragment references that
// failed to resolve. Panics if any references failed.
func UnresolvedFragments() {
	if len(docFailed) == 0 {
		return
	}
	refs := make([]string, 0, len(docFailed))
	for ref := range docFailed {
		refs = append(refs, ref)
	}
	sort.Strings(refs)
	var s strings.Builder
	fmt.Fprintf(&s, "unresolved fragments for %q:\n", Language)
	for _, ref := range refs {
		fmt.Fprintf(&s, "\t%s\n", ref)
	}
	fmt.Fprintf(&s, "\nversion: %s", VersionString())
	panic(s.String())
}

// parseFragPath receives a fragment path and converts it to a file path.
//
// A fragment path is like a regular file path, except that filesep is the
// separator that descends into a file. After the first occurrence of filesep,
// the preceding element is appended with the given suffix, and then descending
// continues.
//
// For example, with ".md" and ":":
//
//     libraries/roblox/types/Region3:Properties/CFrame/Description
//
// is split into the following elements:
//
//     libraries, roblox, types, Region3.md, Properties, CFrame, Description
//
// The file portion of the fragment path is converted to lowercase.
func parseFragPath(s, suffix string, filesep rune) []string {
	if s == "" {
		return []string{}
	}
	i := strings.IndexRune(s, filesep)
	if i < 0 {
		return strings.Split(strings.ToLower(s)+".md", "/")
	}
	items := make([]string, 0, strings.Count(s, "/")+1)
	items = append(items, strings.Split(strings.ToLower(s[:i])+".md", "/")...)
	items = append(items, strings.Split(s[i+1:], "/")...)
	return items
}

const filesep = ':'
const suffix = ".md"

func Doc(fragpath string) string {
	docMut.Lock()
	defer docMut.Unlock()
	var n drill.Node = docfs
	var path string
	docSeen[fragpath] = struct{}{}
	names := parseFragPath(fragpath, suffix, filesep)
	for _, name := range names {
		if name == "" {
			return docString(fragpath, nil)
		}
		path += "/" + name
		if node, ok := docCache[path]; ok {
			n = node
		} else {
			switch v := n.(type) {
			case drill.UnorderedBranch:
				n = v.UnorderedChild(name)
			default:
				return docString(fragpath, nil)
			}
			if node, ok := n.(*markdown.Node); ok {
				docCache[path] = node
			}
		}
	}
	return docString(fragpath, n)
}
<commit_msg>Fix parseFragPath not using suffix argument.<commit_after>package main

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"sync"

	"github.com/anaminus/drill"
	"github.com/anaminus/drill/filesys"
	"github.com/anaminus/drill/filesys/markdown"
	"github.com/anaminus/rbxmk/fragments"
	"github.com/anaminus/rbxmk/rbxmk/term"
	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/extension"
	terminal "golang.org/x/term"
)

// Language determines the language of documentation text.
var Language = "en-us"

// panicLanguage writes the languages that have been embedded, then panics.
func panicLanguage() {
	langs := make([]string, 0, len(fragments.Languages))
	for lang := range fragments.Languages {
		langs = append(langs, lang)
	}
	sort.Strings(langs)
	if len(langs) == 0 {
		fmt.Fprintln(os.Stderr, "no languages are embedded")
	} else {
		fmt.Fprintln(os.Stderr, "the following languages are embedded:")
		for _, lang := range langs {
			fmt.Fprintf(os.Stderr, "\t%s\n", lang)
		}
	}
	panic(fmt.Sprintf("unsupported language %q", Language))
}

func initDocs() drill.Node {
	termWidth, _, _ := terminal.GetSize(int(os.Stdout.Fd()))
	lang, ok := fragments.Languages[Language]
	if !ok {
		panicLanguage()
	}
	f, err := filesys.NewFS(lang, filesys.Handlers{
		{Pattern: "*.md", Func: markdown.NewHandler(
			goldmark.WithRenderer(term.Renderer{Width: termWidth, TabSize: 4}),
			goldmark.WithExtensions(
				extension.Table,
				extension.Footnote,
			),
		)},
	})
	if err != nil {
		panic(err)
	}
	node := f.UnorderedChild(Language)
	if node == nil {
		panicLanguage()
	}
	return node
}

var docfs = initDocs()
var docMut sync.RWMutex
var docCache = map[string]*markdown.Node{}
var docFailed = map[string]struct{}{}
var docSeen = map[string]struct{}{}

// DocFragments returns a list of requested fragments.
func DocFragments() []string {
	docMut.RLock()
	defer docMut.RUnlock()
	frags := make([]string, 0, len(docSeen))
	for frag := range docSeen {
		frags = append(frags, frag)
	}
	sort.Strings(frags)
	return frags
}

// docString extracts and formats the fragment from the given node. Panics if
// the node was not found.
func docString(fragpath string, node drill.Node) string {
	if node == nil {
		docFailed[fragpath] = struct{}{}
		return "{" + Language + ":" + fragpath + "}"
	}
	return strings.TrimSpace(node.Fragment())
}

// UnresolvedFragments writes to stderr a list of fragment references that
// failed to resolve. Panics if any references failed.
func UnresolvedFragments() {
	if len(docFailed) == 0 {
		return
	}
	refs := make([]string, 0, len(docFailed))
	for ref := range docFailed {
		refs = append(refs, ref)
	}
	sort.Strings(refs)
	var s strings.Builder
	fmt.Fprintf(&s, "unresolved fragments for %q:\n", Language)
	for _, ref := range refs {
		fmt.Fprintf(&s, "\t%s\n", ref)
	}
	fmt.Fprintf(&s, "\nversion: %s", VersionString())
	panic(s.String())
}

// parseFragPath receives a fragment path and converts it to a file path.
//
// A fragment path is like a regular file path, except that filesep is the
// separator that descends into a file. After the first occurrence of filesep,
// the preceding element is appended with the given suffix, and then descending
// continues.
//
// For example, with ".md" and ":":
//
//     libraries/roblox/types/Region3:Properties/CFrame/Description
//
// is split into the following elements:
//
//     libraries, roblox, types, Region3.md, Properties, CFrame, Description
//
// The file portion of the fragment path is converted to lowercase.
func parseFragPath(s, suffix string, filesep rune) []string {
	if s == "" {
		return []string{}
	}
	i := strings.IndexRune(s, filesep)
	if i < 0 {
		return strings.Split(strings.ToLower(s)+suffix, "/")
	}
	items := make([]string, 0, strings.Count(s, "/")+1)
	items = append(items, strings.Split(strings.ToLower(s[:i])+suffix, "/")...)
	items = append(items, strings.Split(s[i+1:], "/")...)
	return items
}

const filesep = ':'
const suffix = ".md"

func Doc(fragpath string) string {
	docMut.Lock()
	defer docMut.Unlock()
	var n drill.Node = docfs
	var path string
	docSeen[fragpath] = struct{}{}
	names := parseFragPath(fragpath, suffix, filesep)
	for _, name := range names {
		if name == "" {
			return docString(fragpath, nil)
		}
		path += "/" + name
		if node, ok := docCache[path]; ok {
			n = node
		} else {
			switch v := n.(type) {
			case drill.UnorderedBranch:
				n = v.UnorderedChild(name)
			default:
				return docString(fragpath, nil)
			}
			if node, ok := n.(*markdown.Node); ok {
				docCache[path] = node
			}
		}
	}
	return docString(fragpath, n)
}
<|endoftext|>


<commit_before>package reflect

import (
	"fmt"
	stdref "reflect"
)

var NotStructErr = fmt.Errorf("not a struct or pointer to struct")

func GetFieldNamesP(s interface{}) []string {
	a, err := GetFieldNames(s)
	if err != nil {
		panic(err)
	}
	return a
}

// Returns a field names using reflect package
func GetFieldNames(s interface{}) ([]string, error) {
	t := stdref.TypeOf(s)
	if t.Kind() == stdref.Ptr {
		t = t.Elem()
	}
	if kind := t.Kind(); kind != stdref.Struct {
		return nil, NotStructErr
	}
	var ret []string
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		ret = append(ret, field.Name)
	}
	return ret, nil
}
<commit_msg>add some description<commit_after>package reflect

import (
	"fmt"
	stdref "reflect"
)

var NotStructErr = fmt.Errorf("not a struct or pointer to struct")

// GetFieldNames returns a field names of s, argument must be either struct
// or pointer to struct.
func GetFieldNames(s interface{}) ([]string, error) {
	t := stdref.TypeOf(s)
	if t.Kind() == stdref.Ptr {
		t = t.Elem()
	}
	if kind := t.Kind(); kind != stdref.Struct {
		return nil, NotStructErr
	}
	var ret []string
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		ret = append(ret, field.Name)
	}
	return ret, nil
}

// GetFieldNamesP is a GetFieldNames but panics on error.
func GetFieldNamesP(s interface{}) []string {
	a, err := GetFieldNames(s)
	if err != nil {
		panic(err)
	}
	return a
}
<|endoftext|>


<commit_before>package main

const releaseVersion = "0.4.3"
<commit_msg>release_version: bump to 0.4.4<commit_after>package main

const releaseVersion = "0.4.4"
<|endoftext|>


<commit_before>package render

import (
	"github.com/indraniel/weaver/assets"

	"github.com/arschles/go-bindata-html-template"
	"github.com/shurcooL/github_flavored_markdown"

	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

type Document struct {
	InputFile string
	BaseName  string
	Title     string
	RawBody   string
	HTMLBody  template.HTML
	FullHTML  string
	Date      time.Time
	Updated   time.Time
	Params    map[string]string
}

func NewDocument(inputFile string) *Document {
	basename := path.Base(inputFile)
	doc := &Document{
		InputFile: inputFile,
		BaseName:  basename,
		Date:      time.Now(),
		Updated:   time.Now(),
		Params:    make(map[string]string),
	}
	return doc
}

func (d *Document) ParseDocument() {
	data, err := ioutil.ReadFile(d.InputFile)
	if err != nil {
		log.Fatalf("Couldn't read file: '%s': %s", d.InputFile, err)
	}
	startContentLine := d.parseHeader(data)
	d.parseContents(data, startContentLine)
}

func (d *Document) parseHeader(data []byte) int {
	lines := strings.Split(string(data), "\n")

	headerState := 0
	startMdContentLine := 0
	for i, line := range lines {
		line = strings.TrimSpace(line)

		// in the header section: from --- to ---
		if headerState == 1 {
			colonIndex := strings.Index(line, ":")
			if colonIndex > 0 {
				key := strings.TrimSpace(line[:colonIndex])
				value := strings.TrimSpace(line[colonIndex+1:])
				// remove surrounding quotes
				value = strings.Trim(value, "\"")
				switch key {
				case "Title":
					d.Title = value
				case "Date":
					d.Date, _ =
						time.Parse("2006-01-02", value)
				case "Updated":
					d.Updated, _ =
						time.Parse("2006-01-02", value)
				default:
					d.Params[key] = value
				}
			}
		} else if headerState >= 2 {
			startMdContentLine = i
			break
		}

		if strings.HasPrefix(line, "---") {
			headerState += 1
		}
	}

	return startMdContentLine
}

func (d *Document) parseContents(data []byte, startLine int) {
	lines := strings.Split(string(data), "\n")
	contents := strings.Join(lines[startLine:], "\n")
	d.RawBody = contents
}

func (d *Document) RenderHTML(outDir string) {
	text := []byte(d.RawBody)
	d.HTMLBody = template.HTML(github_flavored_markdown.Markdown(text))
	base := strings.TrimSuffix(d.InputFile, filepath.Ext(d.InputFile))
	htmlFile := strings.Join([]string{base, "html"}, ".")
	htmlPath := filepath.Join(outDir, htmlFile)

	f, err := os.Create(htmlPath)
	if err != nil {
		log.Fatalf("Couldn't open '%s' : %s\n", htmlPath, err)
	}
	defer f.Close()

	baseTemplate := "views/base.html"
	t := assets.NewView(baseTemplate).GetTemplate()
	t, err = t.ParseFiles(baseTemplate)
	if err != nil {
		log.Fatalf("Template '%s' parse error: %s\n", baseTemplate, err)
	}
	err = t.Execute(f, d)
	if err != nil {
		log.Fatalf("Template render error: %s\n", err)
	}

	// alternative approach:
	// http://stackoverflow.com/questions/23124008/how-can-i-render-markdown-to-a-golang-templatehtml-or-tmpl-with-blackfriday
}
<commit_msg>+ use the original html/template<commit_after>package render

import (
	"github.com/indraniel/weaver/assets"

	"github.com/shurcooL/github_flavored_markdown"

	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

type Document struct {
	InputFile string
	BaseName  string
	Title     string
	RawBody   string
	HTMLBody  template.HTML
	FullHTML  string
	Date      time.Time
	Updated   time.Time
	Params    map[string]string
}

func NewDocument(inputFile string) *Document {
	basename := path.Base(inputFile)
	doc := &Document{
		InputFile: inputFile,
		BaseName:  basename,
		Date:      time.Now(),
		Updated:   time.Now(),
		Params:    make(map[string]string),
	}
	return doc
}

func (d *Document) ParseDocument() {
	data, err := ioutil.ReadFile(d.InputFile)
	if err != nil {
		log.Fatalf("Couldn't read file: '%s': %s", d.InputFile, err)
	}
	startContentLine := d.parseHeader(data)
	d.parseContents(data, startContentLine)
}

func (d *Document) parseHeader(data []byte) int {
	lines := strings.Split(string(data), "\n")

	headerState := 0
	startMdContentLine := 0
	for i, line := range lines {
		line = strings.TrimSpace(line)

		// in the header section: from --- to ---
		if headerState == 1 {
			colonIndex := strings.Index(line, ":")
			if colonIndex > 0 {
				key := strings.TrimSpace(line[:colonIndex])
				value := strings.TrimSpace(line[colonIndex+1:])
				// remove surrounding quotes
				value = strings.Trim(value, "\"")
				switch key {
				case "Title":
					d.Title = value
				case "Date":
					d.Date, _ =
						time.Parse("2006-01-02", value)
				case "Updated":
					d.Updated, _ =
						time.Parse("2006-01-02", value)
				default:
					d.Params[key] = value
				}
			}
		} else if headerState >= 2 {
			startMdContentLine = i
			break
		}

		if strings.HasPrefix(line, "---") {
			headerState += 1
		}
	}

	return startMdContentLine
}

func (d *Document) parseContents(data []byte, startLine int) {
	lines := strings.Split(string(data), "\n")
	contents := strings.Join(lines[startLine:], "\n")
	d.RawBody = contents
}

func (d *Document) RenderHTML(outDir string) {
	text := []byte(d.RawBody)
	d.HTMLBody = template.HTML(github_flavored_markdown.Markdown(text))
	base := strings.TrimSuffix(d.InputFile, filepath.Ext(d.InputFile))
	htmlFile := strings.Join([]string{base, "html"}, ".")
	htmlPath := filepath.Join(outDir, htmlFile)

	f, err := os.Create(htmlPath)
	if err != nil {
		log.Fatalf("Couldn't open '%s' : %s\n", htmlPath, err)
	}
	defer f.Close()

	baseTemplate := "views/base.html"
	t := assets.NewView(baseTemplate).GetTemplate()
	t, err = t.ParseFiles(baseTemplate)
	if err != nil {
		log.Fatalf("Template '%s' parse error: %s\n", baseTemplate, err)
	}
	err = t.Execute(f, d)
	if err != nil {
		log.Fatalf("Template render error: %s\n", err)
	}

	// alternative approach:
	// http://stackoverflow.com/questions/23124008/how-can-i-render-markdown-to-a-golang-templatehtml-or-tmpl-with-blackfriday
}
<|endoftext|>


<commit_before>package request

import (
	"net/url"
	"strings"

	"github.com/evalphobia/go-log-wrapper/log"
	"github.com/evalphobia/goreq-wrapper/request"
	"github.com/mitchellh/mapstructure"
)

var _ = log.Nothing

// CallGET sends GET request
func CallGET(url string, params interface{}, result interface{}) error {
	return call("GET", url, params, result)
}

// call sends http request to `url` with `params` and set reqponse to `result`
func call(method, url string, params interface{}, result interface{}) error {
	b, err := request.DSN{
		Method: method,
		Uri:    url,
	}.Param(params).Call()
	if err != nil {
		return err
	}
	body, err := b.ToString()
	if err != nil {
		return err
	}
	m := parseToMap(body)
	return assignFromMap(m, result, "url")
}

// parseToMap converts response string data to map
func parseToMap(str string) map[string]interface{} {
	m := make(map[string]interface{})
	values := strings.Split(str, "&")
	for _, value := range values {
		v := strings.Split(value, "=")
		if len(v) != 2 {
			continue
		}
		m[v[0]] = urlUnescape(v[1])
	}
	return m
}

// assignFromMap set data from map to struct
func assignFromMap(m interface{}, result interface{}, tagName string) error {
	config := &mapstructure.DecoderConfig{
		Metadata: nil,
		Result:   result,
		TagName:  tagName,
	}
	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		return err
	}
	return decoder.Decode(m)
}

func urlUnescape(v string) string {
	unescaped, _ := url.QueryUnescape(v)
	return unescaped
}
<commit_msg>Modified request method<commit_after>package request

import (
	"net/url"
	"strings"

	"github.com/evalphobia/go-log-wrapper/log"
	"github.com/evalphobia/goreq-wrapper/request"
	"github.com/mitchellh/mapstructure"
)

var _ = log.Nothing

// CallGET sends GET request
func CallGET(url string, params interface{}, result interface{}) error {
	return call("GET", url, params, result)
}

// call sends http request to `url` with `params` and set reqponse to `result`
func call(method, url string, params interface{}, result interface{}) error {
	dsn := request.DSN{
		Method: method,
		Uri:    url,
	}
	b, err := dsn.Param(params).Call()
	if err != nil {
		return err
	}
	body, err := b.ToString()
	if err != nil {
		return err
	}
	m := parseToMap(body)
	return assignFromMap(m, result, "url")
}

// parseToMap converts response string data to map
func parseToMap(str string) map[string]interface{} {
	m := make(map[string]interface{})
	values := strings.Split(str, "&")
	for _, value := range values {
		v := strings.Split(value, "=")
		if len(v) != 2 {
			continue
		}
		m[v[0]] = urlUnescape(v[1])
	}
	return m
}

// assignFromMap set data from map to struct
func assignFromMap(m interface{}, result interface{}, tagName string) error {
	config := &mapstructure.DecoderConfig{
		Metadata: nil,
		Result:   result,
		TagName:  tagName,
	}
	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		return err
	}
	return decoder.Decode(m)
}

func urlUnescape(v string) string {
	unescaped, _ := url.QueryUnescape(v)
	return unescaped
}
<|endoftext|>


<commit_before>package requests

import "fmt"

// ImportCollection imports data to a collection
type ImportCollection struct {
	CollectionName string
	Data           []byte
}

func (c *ImportCollection) Path() string {
	return fmt.Sprintf("/_api/import/?type=auto&collection=%s", c.CollectionName)
}

func (c *ImportCollection) Method() string {
	return "POST"
}

func (c *ImportCollection) Generate() []byte {
	return c.Data
}
<commit_msg>Add more collection import options<commit_after>package requests

// ImportCollection imports data to a collection
type ImportCollection struct {
	CollectionName string
	Data           []byte
	FromPrefix     string
	ToPrefix       string
	Overwrite      bool
	WaitForSync    bool
	OnDuplicate    string
	Complete       bool
	Details        bool
}

func (c *ImportCollection) Path() string {
	path := "/_api/import/?type=auto&collection=" + c.CollectionName

	if c.FromPrefix != "" {
		path += "&fromPrefix=" + c.FromPrefix
	}

	if c.ToPrefix != "" {
		path += "&toPrefix=" + c.ToPrefix
	}

	if c.Overwrite {
		path += "&overwrite=yes"
	}

	if c.WaitForSync {
		path += "&waitForSync=yes"
	}

	if c.OnDuplicate != "" {
		path += "&onDuplicate=" + c.OnDuplicate
	}

	if c.Complete {
		path += "&complete=yes"
	}

	if c.Details {
		path += "&details=yes"
	}

	return path
}

func (c *ImportCollection) Method() string {
	return "POST"
}

func (c *ImportCollection) Generate() []byte {
	return c.Data
}
<|endoftext|>


<commit_before>package resource

import (
	"encoding/json"
	"io"
	"net/http"
	"regexp"
	"sort"
	"strings"

	"github.com/qor/qor"
)

func convertMapToMetaValues(values map[string]interface{}, metaors []Metaor) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorMap := make(map[string]Metaor)
	for _, metaor := range metaors {
		metaorMap[metaor.GetName()] = metaor
	}

	for key, value := range values {
		var metaValue *MetaValue
		metaor := metaorMap[key]

		switch result := value.(type) {
		case map[string]interface{}:
			if children, err := convertMapToMetaValues(result, metaor.GetMetas()); err == nil {
				metaValue = &MetaValue{Name: key, Meta: metaor, MetaValues: children}
			}
		case []interface{}:
			for _, r := range result {
				if mr, ok := r.(map[string]interface{}); ok {
					if children, err := convertMapToMetaValues(mr, metaor.GetMetas()); err == nil {
						metaValue := &MetaValue{Name: key, Meta: metaor, MetaValues: children}
						metaValues.Values = append(metaValues.Values, metaValue)
					}
				} else {
					metaValue := &MetaValue{Name: key, Value: result, Meta: metaor}
					metaValues.Values = append(metaValues.Values, metaValue)
					break
				}
			}
		default:
			metaValue = &MetaValue{Name: key, Value: value, Meta: metaor}
		}

		if metaValue != nil {
			metaValues.Values = append(metaValues.Values, metaValue)
		}
	}
	return metaValues, nil
}

// ConvertJSONToMetaValues convert json to meta values
func ConvertJSONToMetaValues(reader io.Reader, metaors []Metaor) (*MetaValues, error) {
	var (
		err     error
		values  = map[string]interface{}{}
		decoder = json.NewDecoder(reader)
	)

	if err = decoder.Decode(&values); err == nil {
		return convertMapToMetaValues(values, metaors)
	}
	return nil, err
}

var (
	isCurrentLevel = regexp.MustCompile("^[^.]+$")
	isNextLevel    = regexp.MustCompile(`^(([^.\[\]]+)(\[\d+\])?)(?:\.([^.]+)+)$`)
)

// ConvertFormToMetaValues convert form to meta values
func ConvertFormToMetaValues(request *http.Request, metaors []Metaor, prefix string) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorsMap := map[string]Metaor{}
	convertedNextLevel := map[string]bool{}
	for _, metaor := range metaors {
		metaorsMap[metaor.GetName()] = metaor
	}

	newMetaValue := func(key string, value interface{}) {
		if strings.HasPrefix(key, prefix) {
			var metaValue *MetaValue
			key = strings.TrimPrefix(key, prefix)

			if matches := isCurrentLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[0]
				metaValue = &MetaValue{Name: name, Value: value, Meta: metaorsMap[name]}
			} else if matches := isNextLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[1]
				if _, ok := convertedNextLevel[name]; !ok {
					var metaors []Metaor
					convertedNextLevel[name] = true
					metaor := metaorsMap[matches[2]]
					if metaor != nil {
						metaors = metaor.GetMetas()
					}

					if children, err := ConvertFormToMetaValues(request, metaors, prefix+name+"."); err == nil {
						metaValue = &MetaValue{Name: matches[2], Meta: metaor, MetaValues: children}
					}
				}
			}

			if metaValue != nil {
				metaValues.Values = append(metaValues.Values, metaValue)
			}
		}
	}

	var sortedFormKeys []string
	for key := range request.Form {
		sortedFormKeys = append(sortedFormKeys, key)
	}
	sort.Strings(sortedFormKeys)

	for _, key := range sortedFormKeys {
		newMetaValue(key, request.Form[key])
	}

	if request.MultipartForm != nil {
		sortedFormKeys = []string{}
		for key := range request.MultipartForm.File {
			sortedFormKeys = append(sortedFormKeys, key)
		}
		sort.Strings(sortedFormKeys)

		for _, key := range sortedFormKeys {
			newMetaValue(key, request.MultipartForm.File[key])
		}
	}
	return metaValues, nil
}

// Decode decode context to result according to resource definition
func Decode(context *qor.Context, result interface{}, res Resourcer) error {
	var errors qor.Errors
	var err error
	var metaValues *MetaValues
	metaors := res.GetMetas([]string{})

	if strings.Contains(context.Request.Header.Get("Content-Type"), "json") {
		metaValues, err = ConvertJSONToMetaValues(context.Request.Body, metaors)
		context.Request.Body.Close()
	} else {
		metaValues, err = ConvertFormToMetaValues(context.Request, metaors, "QorResource.")
	}

	errors.AddError(err)
	errors.AddError(DecodeToResource(res, result, metaValues, context).Start())
	return errors
}
<commit_msg>Fix decode nested collection for virtual attributes<commit_after>package resource

import (
	"encoding/json"
	"io"
	"net/http"
	"regexp"
	"sort"
	"strings"

	"github.com/qor/qor"
)

func convertMapToMetaValues(values map[string]interface{}, metaors []Metaor) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorMap := make(map[string]Metaor)
	for _, metaor := range metaors {
		metaorMap[metaor.GetName()] = metaor
	}

	for key, value := range values {
		var metaValue *MetaValue
		metaor := metaorMap[key]

		switch result := value.(type) {
		case map[string]interface{}:
			if children, err := convertMapToMetaValues(result, metaor.GetMetas()); err == nil {
				metaValue = &MetaValue{Name: key, Meta: metaor, MetaValues: children}
			}
		case []interface{}:
			for _, r := range result {
				if mr, ok := r.(map[string]interface{}); ok {
					if children, err := convertMapToMetaValues(mr, metaor.GetMetas()); err == nil {
						metaValue := &MetaValue{Name: key, Meta: metaor, MetaValues: children}
						metaValues.Values = append(metaValues.Values, metaValue)
					}
				} else {
					metaValue := &MetaValue{Name: key, Value: result, Meta: metaor}
					metaValues.Values = append(metaValues.Values, metaValue)
					break
				}
			}
		default:
			metaValue = &MetaValue{Name: key, Value: value, Meta: metaor}
		}

		if metaValue != nil {
			metaValues.Values = append(metaValues.Values, metaValue)
		}
	}
	return metaValues, nil
}

// ConvertJSONToMetaValues convert json to meta values
func ConvertJSONToMetaValues(reader io.Reader, metaors []Metaor) (*MetaValues, error) {
	var (
		err     error
		values  = map[string]interface{}{}
		decoder = json.NewDecoder(reader)
	)

	if err = decoder.Decode(&values); err == nil {
		return convertMapToMetaValues(values, metaors)
	}
	return nil, err
}

var (
	isCurrentLevel = regexp.MustCompile("^[^.]+$")
	isNextLevel    = regexp.MustCompile(`^(([^.\[\]]+)(\[\d+\])?)(?:(\.[^.]+)+)$`)
)

// ConvertFormToMetaValues convert form to meta values
func ConvertFormToMetaValues(request *http.Request, metaors []Metaor, prefix string) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorsMap := map[string]Metaor{}
	convertedNextLevel := map[string]bool{}
	for _, metaor := range metaors {
		metaorsMap[metaor.GetName()] = metaor
	}

	newMetaValue := func(key string, value interface{}) {
		if strings.HasPrefix(key, prefix) {
			var metaValue *MetaValue
			key = strings.TrimPrefix(key, prefix)

			if matches := isCurrentLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[0]
				metaValue = &MetaValue{Name: name, Value: value, Meta: metaorsMap[name]}
			} else if matches := isNextLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[1]
				if _, ok := convertedNextLevel[name]; !ok {
					var metaors []Metaor
					convertedNextLevel[name] = true
					metaor := metaorsMap[matches[2]]
					if metaor != nil {
						metaors = metaor.GetMetas()
					}

					if children, err := ConvertFormToMetaValues(request, metaors, prefix+name+"."); err == nil {
						metaValue = &MetaValue{Name: matches[2], Meta: metaor, MetaValues: children}
					}
				}
			}

			if metaValue != nil {
				metaValues.Values = append(metaValues.Values, metaValue)
			}
		}
	}

	var sortedFormKeys []string
	for key := range request.Form {
		sortedFormKeys = append(sortedFormKeys, key)
	}
	sort.Strings(sortedFormKeys)

	for _, key := range sortedFormKeys {
		newMetaValue(key, request.Form[key])
	}

	if request.MultipartForm != nil {
		sortedFormKeys = []string{}
		for key := range request.MultipartForm.File {
			sortedFormKeys = append(sortedFormKeys, key)
		}
		sort.Strings(sortedFormKeys)

		for _, key := range sortedFormKeys {
			newMetaValue(key, request.MultipartForm.File[key])
		}
	}
	return metaValues, nil
}

// Decode decode context to result according to resource definition
func Decode(context *qor.Context, result interface{}, res Resourcer) error {
	var errors qor.Errors
	var err error
	var metaValues *MetaValues
	metaors := res.GetMetas([]string{})

	if strings.Contains(context.Request.Header.Get("Content-Type"), "json") {
		metaValues, err = ConvertJSONToMetaValues(context.Request.Body, metaors)
		context.Request.Body.Close()
	} else {
		metaValues, err = ConvertFormToMetaValues(context.Request, metaors, "QorResource.")
	}

	errors.AddError(err)
	errors.AddError(DecodeToResource(res, result, metaValues, context).Start())
	return errors
}
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package generator

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"text/template"
)

const inceptionMainTemplate = `
// DO NOT EDIT!
// Code generated by ffjson <https://github.com/pquerna/ffjson>
// DO NOT EDIT!

package main

import (
	"github.com/pquerna/ffjson/inception"
	"{{.ImportName}}"
)

{{$pn := .PackageName}}

func main() {
	i := ffjsoninception.NewInception()
{{range .StructNames}}
	i.Add(&{{$pn}}.{{.}}{})
{{end}}
	i.Execute()
}
`

type templateCtx struct {
	StructNames []string
	ImportName  string
	PackageName string
}

type InceptionMain struct {
	inputPath    string
	tempMainPath string
	tempMain     *os.File
}

func NewInceptionMain(inputPath string) *InceptionMain {
	return &InceptionMain{
		inputPath: inputPath,
	}
}

func getImportName(inputPath string) (string, error) {
	p, err := filepath.Abs(inputPath)
	if err != nil {
		return "", err
	}

	dpath := filepath.Dir(p)

	gpath, err := filepath.Abs(os.Getenv("GOPATH"))
	if err != nil {
		return "", err
	}

	rel, err := filepath.Rel(gpath, dpath)
	if err != nil {
		return "", err
	}

	if rel[:4] != "src/" {
		return "", errors.New(fmt.Sprintf("Could not find source directory: GOPATH=%q REL=%q", gpath, rel))
	}
	return rel[4:], nil
}

func (im *InceptionMain) Generate(packageName string, si []*StructInfo) error {
	var err error

	importName, err := getImportName(im.inputPath)

	if err != nil {
		return nil
	}

	im.tempMain, err = ioutil.TempFile("", "ffjson-inception")
	if err != nil {
		return err
	}

	sn := make([]string, 0)
	for _, st := range si {
		sn = append(sn, st.Name)
	}

	tc := &templateCtx{
		ImportName:  importName,
		PackageName: packageName,
		StructNames: sn,
	}

	t := template.Must(template.New("inception.go").Parse(inceptionMainTemplate))
	err = t.Execute(im.tempMain, tc)
	if err != nil {
		return err
	}

	println(im.tempMain.Name())

	return nil
}

func (im *InceptionMain) Run(outputPath string) error {
	return nil
}
<commit_msg>don't need a pointer to the struct type<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package generator

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"text/template"
)

const inceptionMainTemplate = `
// DO NOT EDIT!
// Code generated by ffjson <https://github.com/pquerna/ffjson>
// DO NOT EDIT!

package main

import (
	"github.com/pquerna/ffjson/inception"
	"{{.ImportName}}"
)

{{$pn := .PackageName}}

func main() {
	i := ffjsoninception.NewInception()
{{range .StructNames}}
	i.Add({{$pn}}.{{.}}{})
{{end}}
	i.Execute()
}
`

type templateCtx struct {
	StructNames []string
	ImportName  string
	PackageName string
}

type InceptionMain struct {
	inputPath    string
	tempMainPath string
	tempMain     *os.File
}

func NewInceptionMain(inputPath string) *InceptionMain {
	return &InceptionMain{
		inputPath: inputPath,
	}
}

func getImportName(inputPath string) (string, error) {
	p, err := filepath.Abs(inputPath)
	if err != nil {
		return "", err
	}

	dpath := filepath.Dir(p)

	gpath, err := filepath.Abs(os.Getenv("GOPATH"))
	if err != nil {
		return "", err
	}

	rel, err := filepath.Rel(gpath, dpath)
	if err != nil {
		return "", err
	}

	if rel[:4] != "src/" {
		return "", errors.New(fmt.Sprintf("Could not find source directory: GOPATH=%q REL=%q", gpath, rel))
	}
	return rel[4:], nil
}

func (im *InceptionMain) Generate(packageName string, si []*StructInfo) error {
	var err error

	importName, err := getImportName(im.inputPath)

	if err != nil {
		return nil
	}

	im.tempMain, err = ioutil.TempFile("", "ffjson-inception")
	if err != nil {
		return err
	}

	sn := make([]string, 0)
	for _, st := range si {
		sn = append(sn, st.Name)
	}

	tc := &templateCtx{
		ImportName:  importName,
		PackageName: packageName,
		StructNames: sn,
	}

	t := template.Must(template.New("inception.go").Parse(inceptionMainTemplate))
	err = t.Execute(im.tempMain, tc)
	if err != nil {
		return err
	}

	println(im.tempMain.Name())

	return nil
}

func (im *InceptionMain) Run(outputPath string) error {
	return nil
}
<|endoftext|>