<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"testing"

	"github.com/stretchr/testify/assert"
)

var testClient *Client
var testCommands map[string]string

func setupCommands() {
	testCommands = map[string]string{
		"createdb": "createdb",
		"psql":     "psql",
		"dropdb":   "dropdb",
	}

	if onWindows() {
		for k, v := range testCommands {
			testCommands[k] = v + ".exe"
		}
	}
}

func onWindows() bool {
	return runtime.GOOS == "windows"
}

func setup() {
	out, err := exec.Command(testCommands["createdb"], "-U", "postgres", "-h", "localhost", "booktown").CombinedOutput()

	if err != nil {
		fmt.Println("Database creation failed:", string(out))
		fmt.Println("Error:", err)
		os.Exit(1)
	}

	out, err = exec.Command(testCommands["psql"], "-U", "postgres", "-h", "localhost", "-f", "./sql/booktown.sql", "booktown").CombinedOutput()

	if err != nil {
		fmt.Println("Database import failed:", string(out))
		fmt.Println("Error:", err)
		os.Exit(1)
	}
}

func setupClient() {
	testClient, _ = NewClientFromUrl("postgres://postgres@localhost/booktown?sslmode=disable")
}

func teardownClient() {
	if testClient != nil {
		testClient.db.Close()
	}
}

func teardown() {
	out, err := exec.Command(testCommands["dropdb"], "-U", "postgres", "-h", "localhost", "booktown").CombinedOutput()

	if err != nil {
		fmt.Println(string(out))
		fmt.Println("Error:", err)
	}
}

func test_NewClientFromUrl(t *testing.T) {
	url := "postgres://postgres@localhost/booktown?sslmode=disable"
	client, err := NewClientFromUrl(url)

	if err != nil {
		defer client.db.Close()
	}

	assert.Equal(t, nil, err)
	assert.Equal(t, url, client.connectionString)
}

func test_Test(t *testing.T) {
	assert.Equal(t, nil, testClient.Test())
}

func test_Info(t *testing.T) {
	res, err := testClient.Info()

	assert.Equal(t, nil, err)
	assert.NotEqual(t, nil, res)
}

func test_Databases(t *testing.T) {
	res, err := testClient.Databases()

	assert.Equal(t, nil, err)
	assert.Contains(t, res, "booktown")
	assert.Contains(t, res, "postgres")
}

func test_Tables(t *testing.T) {
	res, err := testClient.Tables()

	expected := []string{
		"alternate_stock",
		"authors",
		"book_backup",
		"book_queue",
		"books",
		"customers",
		"daily_inventory",
		"distinguished_authors",
		"editions",
		"employees",
		"favorite_authors",
		"favorite_books",
		"money_example",
		"my_list",
		"numeric_values",
		"publishers",
		"recent_shipments",
		"schedules",
		"shipments",
		"states",
		"stock",
		"stock_backup",
		"stock_view",
		"subjects",
		"text_sorting",
	}

	assert.Equal(t, nil, err)
	assert.Equal(t, expected, res)
}

func test_Table(t *testing.T) {
	res, err := testClient.Table("books")

	columns := []string{
		"column_name",
		"data_type",
		"is_nullable",
		"character_maximum_length",
		"character_set_catalog",
		"column_default",
	}

	assert.Equal(t, nil, err)
	assert.Equal(t, columns, res.Columns)
	assert.Equal(t, 4, len(res.Rows))
}

func test_TableRows(t *testing.T) {
	res, err := testClient.TableRows("books", RowsOptions{})

	assert.Equal(t, nil, err)
	assert.Equal(t, 4, len(res.Columns))
	assert.Equal(t, 15, len(res.Rows))
}

func test_TableInfo(t *testing.T) {
	res, err := testClient.TableInfo("books")

	assert.Equal(t, nil, err)
	assert.Equal(t, 4, len(res.Columns))
	assert.Equal(t, 1, len(res.Rows))
}

func test_TableIndexes(t *testing.T) {
	res, err := testClient.TableIndexes("books")

	assert.Equal(t, nil, err)
	assert.Equal(t, 2, len(res.Columns))
	assert.Equal(t, 2, len(res.Rows))
}

func test_Query(t *testing.T) {
	res, err := testClient.Query("SELECT * FROM books")

	assert.Equal(t, nil, err)
	assert.Equal(t, 4, len(res.Columns))
	assert.Equal(t, 15, len(res.Rows))
}

func test_QueryError(t *testing.T) {
	res, err := testClient.Query("SELCT * FROM books")

	assert.NotEqual(t, nil, err)
	assert.Equal(t, "pq: syntax error at or near \"SELCT\"", err.Error())
	assert.Equal(t, true, res == nil)
}

func test_QueryInvalidTable(t *testing.T) {
	res, err := testClient.Query("SELECT * FROM books2")

	assert.NotEqual(t, nil, err)
	assert.Equal(t, "pq: relation \"books2\" does not exist", err.Error())
	assert.Equal(t, true, res == nil)
}

func test_ResultCsv(t *testing.T) {
	res, _ := testClient.Query("SELECT * FROM books ORDER BY id ASC LIMIT 1")
	csv := res.CSV()

	expected := "id,title,author_id,subject_id\n156,The Tell-Tale Heart,115,9\n"

	assert.Equal(t, expected, string(csv))
}

func test_History(t *testing.T) {
	_, err := testClient.Query("SELECT * FROM books")
	query := testClient.history[len(testClient.history)-1].Query

	assert.Equal(t, nil, err)
	assert.Equal(t, "SELECT * FROM books", query)
}

func test_HistoryError(t *testing.T) {
	_, err := testClient.Query("SELECT * FROM books123")
	query := testClient.history[len(testClient.history)-1].Query

	assert.NotEqual(t, nil, err)
	assert.NotEqual(t, "SELECT * FROM books123", query)
}

func TestAll(t *testing.T) {
	setupCommands()
	teardown()
	setup()
	setupClient()

	test_NewClientFromUrl(t)
	test_Test(t)
	test_Info(t)
	test_Databases(t)
	test_Tables(t)
	test_Table(t)
	test_TableRows(t)
	test_TableInfo(t)
	test_TableIndexes(t)
	test_Query(t)
	test_QueryError(t)
	test_QueryInvalidTable(t)
	test_ResultCsv(t)
	test_History(t)
	test_HistoryError(t)

	teardownClient()
	teardown()
}
<commit_msg>Skip postgres tests on windows for now<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"testing"

	"github.com/stretchr/testify/assert"
)

var testClient *Client
var testCommands map[string]string

func setupCommands() {
	testCommands = map[string]string{
		"createdb": "createdb",
		"psql":     "psql",
		"dropdb":   "dropdb",
	}

	if onWindows() {
		for k, v := range testCommands {
			testCommands[k] = v + ".exe"
		}
	}
}

func onWindows() bool {
	return runtime.GOOS == "windows"
}

func setup() {
	out, err := exec.Command(testCommands["createdb"], "-U", "postgres", "-h", "localhost", "booktown").CombinedOutput()

	if err != nil {
		fmt.Println("Database creation failed:", string(out))
		fmt.Println("Error:", err)
		os.Exit(1)
	}

	out, err = exec.Command(testCommands["psql"], "-U", "postgres", "-h", "localhost", "-f", "./sql/booktown.sql", "booktown").CombinedOutput()

	if err != nil {
		fmt.Println("Database import failed:", string(out))
		fmt.Println("Error:", err)
		os.Exit(1)
	}
}

func setupClient() {
	testClient, _ = NewClientFromUrl("postgres://postgres@localhost/booktown?sslmode=disable")
}

func teardownClient() {
	if testClient != nil {
		testClient.db.Close()
	}
}

func teardown() {
	out, err := exec.Command(testCommands["dropdb"], "-U", "postgres", "-h", "localhost", "booktown").CombinedOutput()

	if err != nil {
		fmt.Println(string(out))
		fmt.Println("Error:", err)
	}
}

func test_NewClientFromUrl(t *testing.T) {
	url := "postgres://postgres@localhost/booktown?sslmode=disable"
	client, err := NewClientFromUrl(url)

	if err != nil {
		defer client.db.Close()
	}

	assert.Equal(t, nil, err)
	assert.Equal(t, url, client.connectionString)
}

func test_Test(t *testing.T) {
	assert.Equal(t, nil, testClient.Test())
}

func test_Info(t *testing.T) {
	res, err := testClient.Info()

	assert.Equal(t, nil, err)
	assert.NotEqual(t, nil, res)
}

func test_Databases(t *testing.T) {
	res, err := testClient.Databases()

	assert.Equal(t, nil, err)
	assert.Contains(t, res, "booktown")
	assert.Contains(t, res, "postgres")
}

func test_Tables(t *testing.T) {
	res, err := testClient.Tables()

	expected := []string{
		"alternate_stock",
		"authors",
		"book_backup",
		"book_queue",
		"books",
		"customers",
		"daily_inventory",
		"distinguished_authors",
		"editions",
		"employees",
		"favorite_authors",
		"favorite_books",
		"money_example",
		"my_list",
		"numeric_values",
		"publishers",
		"recent_shipments",
		"schedules",
		"shipments",
		"states",
		"stock",
		"stock_backup",
		"stock_view",
		"subjects",
		"text_sorting",
	}

	assert.Equal(t, nil, err)
	assert.Equal(t, expected, res)
}

func test_Table(t *testing.T) {
	res, err := testClient.Table("books")

	columns := []string{
		"column_name",
		"data_type",
		"is_nullable",
		"character_maximum_length",
		"character_set_catalog",
		"column_default",
	}

	assert.Equal(t, nil, err)
	assert.Equal(t, columns, res.Columns)
	assert.Equal(t, 4, len(res.Rows))
}

func test_TableRows(t *testing.T) {
	res, err := testClient.TableRows("books", RowsOptions{})

	assert.Equal(t, nil, err)
	assert.Equal(t, 4, len(res.Columns))
	assert.Equal(t, 15, len(res.Rows))
}

func test_TableInfo(t *testing.T) {
	res, err := testClient.TableInfo("books")

	assert.Equal(t, nil, err)
	assert.Equal(t, 4, len(res.Columns))
	assert.Equal(t, 1, len(res.Rows))
}

func test_TableIndexes(t *testing.T) {
	res, err := testClient.TableIndexes("books")

	assert.Equal(t, nil, err)
	assert.Equal(t, 2, len(res.Columns))
	assert.Equal(t, 2, len(res.Rows))
}

func test_Query(t *testing.T) {
	res, err := testClient.Query("SELECT * FROM books")

	assert.Equal(t, nil, err)
	assert.Equal(t, 4, len(res.Columns))
	assert.Equal(t, 15, len(res.Rows))
}

func test_QueryError(t *testing.T) {
	res, err := testClient.Query("SELCT * FROM books")

	assert.NotEqual(t, nil, err)
	assert.Equal(t, "pq: syntax error at or near \"SELCT\"", err.Error())
	assert.Equal(t, true, res == nil)
}

func test_QueryInvalidTable(t *testing.T) {
	res, err := testClient.Query("SELECT * FROM books2")

	assert.NotEqual(t, nil, err)
	assert.Equal(t, "pq: relation \"books2\" does not exist", err.Error())
	assert.Equal(t, true, res == nil)
}

func test_ResultCsv(t *testing.T) {
	res, _ := testClient.Query("SELECT * FROM books ORDER BY id ASC LIMIT 1")
	csv := res.CSV()

	expected := "id,title,author_id,subject_id\n156,The Tell-Tale Heart,115,9\n"

	assert.Equal(t, expected, string(csv))
}

func test_History(t *testing.T) {
	_, err := testClient.Query("SELECT * FROM books")
	query := testClient.history[len(testClient.history)-1].Query

	assert.Equal(t, nil, err)
	assert.Equal(t, "SELECT * FROM books", query)
}

func test_HistoryError(t *testing.T) {
	_, err := testClient.Query("SELECT * FROM books123")
	query := testClient.history[len(testClient.history)-1].Query

	assert.NotEqual(t, nil, err)
	assert.NotEqual(t, "SELECT * FROM books123", query)
}

func TestAll(t *testing.T) {
	if onWindows() {
		// Dont have access to windows machines at the moment...
		return
	}

	setupCommands()
	teardown()
	setup()
	setupClient()

	test_NewClientFromUrl(t)
	test_Test(t)
	test_Info(t)
	test_Databases(t)
	test_Tables(t)
	test_Table(t)
	test_TableRows(t)
	test_TableInfo(t)
	test_TableIndexes(t)
	test_Query(t)
	test_QueryError(t)
	test_QueryInvalidTable(t)
	test_ResultCsv(t)
	test_History(t)
	test_HistoryError(t)

	teardownClient()
	teardown()
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package environs

import (
	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/storage"
	"github.com/juju/juju/tools"
)

// StartInstanceParams holds parameters for the
// InstanceBroker.StartInstance method.
type StartInstanceParams struct {
	// Constraints is a set of constraints on
	// the kind of instance to create.
	Constraints constraints.Value

	// Tools is a list of tools that may be used
	// to start a Juju agent on the machine.
	Tools tools.List

	// InstanceConfig describes the machine's configuration.
	InstanceConfig *instancecfg.InstanceConfig

	// Placement, if non-empty, contains an environment-specific
	// placement directive that may be used to decide how the
	// instance should be started.
	Placement string

	// DistributionGroup, if non-nil, is a function
	// that returns a slice of instance.Ids that belong
	// to the same distribution group as the machine
	// being provisioned. The InstanceBroker may use
	// this information to distribute instances for
	// high availability.
	DistributionGroup func() ([]instance.Id, error)

	// Volumes is a set of parameters for volumes that should be created.
	//
	// StartInstance need not check the value of the Attachment field,
	// as it is guaranteed that any volumes in this list are designated
	// for attachment to the instance being started.
	Volumes []storage.VolumeParams

	// NetworkInfo is an optional list of network interface details,
	// necessary to configure on the instance.
	NetworkInfo []network.InterfaceInfo

	// SubnetsToZones is an optional map of provider-specific subnet
	// id to a list of availability zone names the subnet is available
	// in. It is only populated when valid positive spaces constraints
	// are present.
	SubnetsToZones map[network.Id][]string
}

// StartInstanceResult holds the result of an
// InstanceBroker.StartInstance method call.
type StartInstanceResult struct {
	// Instance is an interface representing a cloud instance.
	Instance instance.Instance

	// HardwareCharacteristics represents the hardware characteristics
	// of the newly created instance.
	Hardware *instance.HardwareCharacteristics

	// NetworkInfo contains information about how to configure network
	// interfaces on the instance. Depending on the provider, this
	// might be the same StartInstanceParams.NetworkInfo or may be
	// modified as needed.
	NetworkInfo []network.InterfaceInfo

	// Volumes contains a list of volumes created, each one having the
	// same Name as one of the VolumeParams in StartInstanceParams.Volumes.
	// VolumeAttachment information is reported separately.
	Volumes []storage.Volume

	// VolumeAttachments contains a attachment-specific information about
	// volumes that were attached to the started instance.
	VolumeAttachments []storage.VolumeAttachment
}

// TODO(wallyworld) - we want this in the environs/instance package but import loops
// stop that from being possible right now.
type InstanceBroker interface {
	// StartInstance asks for a new instance to be created, associated with
	// the provided config in machineConfig. The given config describes the juju
	// state for the new instance to connect to. The config MachineNonce, which must be
	// unique within an environment, is used by juju to protect against the
	// consequences of multiple instances being started with the same machine
	// id.
	StartInstance(args StartInstanceParams) (*StartInstanceResult, error)

	// StopInstances shuts down the instances with the specified IDs.
	// Unknown instance IDs are ignored, to enable idempotency.
	StopInstances(...instance.Id) error

	// AllInstances returns all instances currently known to the broker.
	AllInstances() ([]instance.Instance, error)

	// MaintainInstance is used to run actions on jujud startup for existing
	// instances. It is currently only used to ensure that LXC hosts have the
	// correct network configuration.
	MaintainInstance(args StartInstanceParams) error
}
<commit_msg>Add a Config field to environs.StartInstanceResult.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package environs

import (
	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/storage"
	"github.com/juju/juju/tools"
)

// StartInstanceParams holds parameters for the
// InstanceBroker.StartInstance method.
type StartInstanceParams struct {
	// Constraints is a set of constraints on
	// the kind of instance to create.
	Constraints constraints.Value

	// Tools is a list of tools that may be used
	// to start a Juju agent on the machine.
	Tools tools.List

	// InstanceConfig describes the machine's configuration.
	InstanceConfig *instancecfg.InstanceConfig

	// Placement, if non-empty, contains an environment-specific
	// placement directive that may be used to decide how the
	// instance should be started.
	Placement string

	// DistributionGroup, if non-nil, is a function
	// that returns a slice of instance.Ids that belong
	// to the same distribution group as the machine
	// being provisioned. The InstanceBroker may use
	// this information to distribute instances for
	// high availability.
	DistributionGroup func() ([]instance.Id, error)

	// Volumes is a set of parameters for volumes that should be created.
	//
	// StartInstance need not check the value of the Attachment field,
	// as it is guaranteed that any volumes in this list are designated
	// for attachment to the instance being started.
	Volumes []storage.VolumeParams

	// NetworkInfo is an optional list of network interface details,
	// necessary to configure on the instance.
	NetworkInfo []network.InterfaceInfo

	// SubnetsToZones is an optional map of provider-specific subnet
	// id to a list of availability zone names the subnet is available
	// in. It is only populated when valid positive spaces constraints
	// are present.
	SubnetsToZones map[network.Id][]string
}

// StartInstanceResult holds the result of an
// InstanceBroker.StartInstance method call.
type StartInstanceResult struct {
	// Instance is an interface representing a cloud instance.
	Instance instance.Instance

	// Config holds the environment config to be used for any further
	// operations, if the instance is for a state server.
	Config *config.Config

	// HardwareCharacteristics represents the hardware characteristics
	// of the newly created instance.
	Hardware *instance.HardwareCharacteristics

	// NetworkInfo contains information about how to configure network
	// interfaces on the instance. Depending on the provider, this
	// might be the same StartInstanceParams.NetworkInfo or may be
	// modified as needed.
	NetworkInfo []network.InterfaceInfo

	// Volumes contains a list of volumes created, each one having the
	// same Name as one of the VolumeParams in StartInstanceParams.Volumes.
	// VolumeAttachment information is reported separately.
	Volumes []storage.Volume

	// VolumeAttachments contains a attachment-specific information about
	// volumes that were attached to the started instance.
	VolumeAttachments []storage.VolumeAttachment
}

// TODO(wallyworld) - we want this in the environs/instance package but import loops
// stop that from being possible right now.
type InstanceBroker interface {
	// StartInstance asks for a new instance to be created, associated with
	// the provided config in machineConfig. The given config describes the juju
	// state for the new instance to connect to. The config MachineNonce, which must be
	// unique within an environment, is used by juju to protect against the
	// consequences of multiple instances being started with the same machine
	// id.
	StartInstance(args StartInstanceParams) (*StartInstanceResult, error)

	// StopInstances shuts down the instances with the specified IDs.
	// Unknown instance IDs are ignored, to enable idempotency.
	StopInstances(...instance.Id) error

	// AllInstances returns all instances currently known to the broker.
	AllInstances() ([]instance.Instance, error)

	// MaintainInstance is used to run actions on jujud startup for existing
	// instances. It is currently only used to ensure that LXC hosts have the
	// correct network configuration.
	MaintainInstance(args StartInstanceParams) error
}
<|endoftext|>


<commit_before>package consolidation

import (
	"context"

	"github.com/grafana/metrictank/schema"
)

// ConsolidateContext wraps a Consolidate() call with a context.Context condition
func ConsolidateContext(ctx context.Context, in []schema.Point, aggNum uint32, consolidator Consolidator) []schema.Point {
	select {
	case <-ctx.Done():
		//request canceled
		return nil
	default:
	}
	return Consolidate(in, aggNum, consolidator)
}

// ConsolidateNudged consolidates points in a "mostly-stable" way, meaning if you run the same function again so that the input
// receives new points at the end and old points get removed at the beginning, we keep picking the same points to consolidate together,
// except for cases where there's a few points and a low MaxDataPoints. See nudgeMaybe()
// interval is the interval between the input points
func ConsolidateNudged(points []schema.Point, interval, maxDataPoints uint32, consolidator Consolidator) ([]schema.Point, uint32) {
	aggNum := AggEvery(uint32(len(points)), maxDataPoints)
	points = nudgeMaybe(points, aggNum, interval)
	points = Consolidate(points, aggNum, consolidator)
	return points, interval * aggNum
}

// Consolidate consolidates `in`, aggNum points at a time via the given function
// note: the returned slice repurposes in's backing array.
// it will always aggregate aggNum-sized groups of points together, with the timestamp of the last of them, and it always starts at the beginning,
// possibly having a point at the end that didn't incorporate as much data
func Consolidate(in []schema.Point, aggNum uint32, consolidator Consolidator) []schema.Point {
	num := int(aggNum)
	aggFunc := GetAggFunc(consolidator)

	// let's see if the input data is a perfect fit for the requested aggNum
	// (e.g. no remainder). This case is the easiest to handle
	outLen := len(in) / num
	cleanLen := num * outLen
	if len(in) == cleanLen {
		out := in[0:outLen]
		var outI, nextI int
		for inI := 0; inI < cleanLen; inI = nextI {
			nextI = inI + num
			out[outI] = schema.Point{Val: aggFunc(in[inI:nextI]), Ts: in[nextI-1].Ts}
			outI += 1
		}
		return out
	}

	// the fit is not perfect: first process all the aggNum sized groups:
	outLen += 1
	out := in[0:outLen]
	var outI, nextI int
	for inI := 0; inI < cleanLen; inI = nextI {
		nextI = inI + num
		out[outI] = schema.Point{Val: aggFunc(in[inI:nextI]), Ts: in[nextI-1].Ts}
		outI += 1
	}

	// we have some leftover points that didn't get aggregated yet because they're fewer than aggNum.
	// we must also aggregate it and add it, and the timestamp of this point must be what it would have been
	// if the group would have been complete, i.e. points in the consolidation output should be evenly spaced.
	// obviously we can only figure out the interval if we have at least 2 points
	var lastTs uint32
	if len(in) == 1 {
		lastTs = in[0].Ts
	} else {
		interval := in[len(in)-1].Ts - in[len(in)-2].Ts
		// len 10, cleanLen 9, num 3 -> 3*4 values supposedly -> "in[11].Ts" -> in[9].Ts + 2*interval
		lastTs = in[cleanLen].Ts + (aggNum-1)*interval
	}
	out[outI] = schema.Point{Val: aggFunc(in[cleanLen:]), Ts: lastTs}
	return out
}

// returns how many points should be aggregated together so that you end up with as many points as possible,
// but never more than maxPoints
func AggEvery(numPoints, maxPoints uint32) uint32 {
	if numPoints == 0 {
		return 1
	}
	return (numPoints + maxPoints - 1) / maxPoints
}

func nudgeMaybe(points []schema.Point, aggNum, interval uint32) []schema.Point {
	// note that the amount of points to strip by nudging is always < 1 postAggInterval's worth.
	// there's 2 important considerations here:
	// 1) we shouldn't make any too drastic alterations of the timerange returned compared to the requested time range
	// 2) the stripping effort shouldn't significantly alter the output otherwise things get confusing
	// these 2 remarks boil down to "the amount of points stripped should be a small fraction of the amount of input points"
	// we use this simple heuristic:
	// only nudge if we have points > 2 * postAggInterval's worth where "postAggInterval's worth is aggNum points"
	//
	// this also assures that in the special case where people request MaxDataPoints=1 we will always consolidate
	// all points together and don't trim a significant amount of the points
	// that are expected to go into the aggregation
	// e.g. consider a case where we have points with ts 140,150,160,170
	// aggNum = aggEvery(4/1) = 4, postAggInterval is thus 40.
	// strict application of nudging would return 1 point with ts=200 (aggregation of all points 170-200 which is 1 point)
	// and strip the first 3 points,
	// which is not what we want. since we only have a small set of points, better to incorporate all points into 1 bucket with ts 170.
	// note that in this case (where we don't nudge) the timestamps in output are not cleanly divisible by postAggInterval

	// we only start stripping if we have more than 2*4=8 points
	// see the unit tests which explore cases like this (TestConsolidateNudgedNoTrimDueToNotManyPoints)
	if len(points) > int(2*aggNum) {
		_, num := nudge(points[0].Ts, interval, aggNum)
		points = points[num:]
	}
	return points
}

// Nudge computes the parameters for nudging:
// * the diff to add to the old start point to compute the timestamp of the new start point
// * number of points to strip from the start.
// let's say a series has points A,B,C,D and we must consolidate with numAgg=2.
// if we wait a step, point E appears into the visible window and A will slide out of the window.
// there's a few approaches you can take wrt such changes across refreshes:
// 1) naive old approach:
//    on first load return consolidate(A,B), consolidate(C,D)
//    after a step, return consolidate(B,C), consolidate(D,E)
//    => this looks weird across refreshes:
//       both the values as well as the timestamps change everywhere, points jump around on the chart
// 2) graphite-style nudging: trim a few of the first points away as needed, such that the first TS
//    will be the first one to fill a "full" aggregation bucket. (note: assumes input is quantized!)
//    on first load return consolidate(A,B), consolidate(C,D)
//    after a step, return consolidate(C,D), consolidate(E)
//    => same points are always consolidated together, no jumping around.
//    => simple to understand, but not the most performant (fetches/processes some points needlessly)
//    => tends to introduce emptyness in graphs right after the requested start.
//       (because Grafana plots from requested start, not returned start, and so there will be some empty space
//       where we trimmed points)
// 3) similar to 2, but don't trim, rather consolidate the leftovers both on the start and at the end.
//    on first load return consolidate(A,B), consolidate(C,D)
//    after a step, return consolidate(B), consolidate(C,D), consolidate(E)
//    => same points are always consolidated together, no jumping around.
//    => only datapoint up front and at the end may jump around, but not the vast majority
//    => no discarding of points
//    => requires a large code change though, as it makes it harder to honor MaxDataPoints.
//       e.g. MDP=1, you have 5 points and aggNum is 5, if alignment is improper, it would cause
//       2 output points, so we would have to rewrite a lot of code, no longer compute AggNum in advance etc
//
// note that with all 3 approaches, we always consolidate leftovers at the end together, so with any approach
// the last point may jump around (see Consolidate function)
// for now, and for simplicity we just implement the 2nd approach. it's also the only one that assures MDP is strictly
// honored (see last point of approach 3, which also affects approach 1)
func nudge(start, preAggInterval, aggNum uint32) (uint32, int) {
	postAggInterval := preAggInterval * aggNum
	var num int
	var diff uint32
	// two important principles here:
	// 1) aggregation buckets have timestamps that are cleanly divisible by postAggInterval.
	// 2) we never make data pretend to be able to predict the future,
	//    e.g. when aggregating a spike, the spike should never move to an earlier timestamp
	//    in other words, data should only ever move to a later timestamp when being aggregated, never to the past.
	// it follows that the start of (first point going into) an aggregation bucket
	// is the point that has a timestamp of 1 preAggInterval after the cleanly-divisible-by-postAggInterval timestamp.

	// let's clarify all this with an example.
	// consider: preAggInterval = 5, aggNum = 4 => postAggInterval = 20
	// points marked with a * are the first points going into an aggregation interval

	// input   consolidate into    output points
	// 15      ▼
	// 20        ►                 20
	// 25 *    ▼
	// 30      ▼
	// 35      ▼
	// 40        ►                 40
	// 45 *    ▼
	// 50      ▼
	// 55      ▼
	// 60        ►                 60
	// 65 *    ▼
	// 70      ▼
	// 75      ▼
	//                             80

	// with our nudging approach, we strip all the points that do not form a full aggregation bucket.
	// IOW the desired first point must be the the first point of a bucket.
	// in the example above, it means we strip the first 2 points and start at ts=25 (which will go into ts=40).

	// move start until it maps to the first point of an aggregation bucket
	remainder := (start - preAggInterval) % postAggInterval
	if remainder > 0 {
		diff = postAggInterval - remainder
		num = int(diff / preAggInterval)
	}
	// following the above example:
	// remainder = (15 - 5) % 20 = 10
	// diff     = 20 - 10 = 10 // start will be moved up by 10s, from 15 to 25
	// num      = 10/5 = 2 // which means we will strip the first two points

	return diff, num
}
<commit_msg>consolidation.Consolidate needs a valid consolidator. make explicit<commit_after>package consolidation

import (
	"context"

	"github.com/grafana/metrictank/schema"
)

// ConsolidateContext wraps a Consolidate() call with a context.Context condition
func ConsolidateContext(ctx context.Context, in []schema.Point, aggNum uint32, consolidator Consolidator) []schema.Point {
	select {
	case <-ctx.Done():
		//request canceled
		return nil
	default:
	}
	return Consolidate(in, aggNum, consolidator)
}

// ConsolidateNudged consolidates points in a "mostly-stable" way, meaning if you run the same function again so that the input
// receives new points at the end and old points get removed at the beginning, we keep picking the same points to consolidate together,
// except for cases where there's a few points and a low MaxDataPoints. See nudgeMaybe()
// interval is the interval between the input points
func ConsolidateNudged(points []schema.Point, interval, maxDataPoints uint32, consolidator Consolidator) ([]schema.Point, uint32) {
	aggNum := AggEvery(uint32(len(points)), maxDataPoints)
	points = nudgeMaybe(points, aggNum, interval)
	points = Consolidate(points, aggNum, consolidator)
	return points, interval * aggNum
}

// Consolidate consolidates `in`, aggNum points at a time via the given function
// note: the returned slice repurposes in's backing array.
// it will always aggregate aggNum-sized groups of points together, with the timestamp of the last of them, and it always starts at the beginning,
// possibly having a point at the end that didn't incorporate as much data
func Consolidate(in []schema.Point, aggNum uint32, consolidator Consolidator) []schema.Point {
	if consolidator == None {
		panic("Consolidate called with consolidation.None. this should never happen")
	}
	num := int(aggNum)
	aggFunc := GetAggFunc(consolidator)

	// let's see if the input data is a perfect fit for the requested aggNum
	// (e.g. no remainder). This case is the easiest to handle
	outLen := len(in) / num
	cleanLen := num * outLen
	if len(in) == cleanLen {
		out := in[0:outLen]
		var outI, nextI int
		for inI := 0; inI < cleanLen; inI = nextI {
			nextI = inI + num
			out[outI] = schema.Point{Val: aggFunc(in[inI:nextI]), Ts: in[nextI-1].Ts}
			outI += 1
		}
		return out
	}

	// the fit is not perfect: first process all the aggNum sized groups:
	outLen += 1
	out := in[0:outLen]
	var outI, nextI int
	for inI := 0; inI < cleanLen; inI = nextI {
		nextI = inI + num
		out[outI] = schema.Point{Val: aggFunc(in[inI:nextI]), Ts: in[nextI-1].Ts}
		outI += 1
	}

	// we have some leftover points that didn't get aggregated yet because they're fewer than aggNum.
	// we must also aggregate it and add it, and the timestamp of this point must be what it would have been
	// if the group would have been complete, i.e. points in the consolidation output should be evenly spaced.
	// obviously we can only figure out the interval if we have at least 2 points
	var lastTs uint32
	if len(in) == 1 {
		lastTs = in[0].Ts
	} else {
		interval := in[len(in)-1].Ts - in[len(in)-2].Ts
		// len 10, cleanLen 9, num 3 -> 3*4 values supposedly -> "in[11].Ts" -> in[9].Ts + 2*interval
		lastTs = in[cleanLen].Ts + (aggNum-1)*interval
	}
	out[outI] = schema.Point{Val: aggFunc(in[cleanLen:]), Ts: lastTs}
	return out
}

// returns how many points should be aggregated together so that you end up with as many points as possible,
// but never more than maxPoints
func AggEvery(numPoints, maxPoints uint32) uint32 {
	if numPoints == 0 {
		return 1
	}
	return (numPoints + maxPoints - 1) / maxPoints
}

func nudgeMaybe(points []schema.Point, aggNum, interval uint32) []schema.Point {
	// note that the amount of points to strip by nudging is always < 1 postAggInterval's worth.
	// there's 2 important considerations here:
	// 1) we shouldn't make any too drastic alterations of the timerange returned compared to the requested time range
	// 2) the stripping effort shouldn't significantly alter the output otherwise things get confusing
	// these 2 remarks boil down to "the amount of points stripped should be a small fraction of the amount of input points"
	// we use this simple heuristic:
	// only nudge if we have points > 2 * postAggInterval's worth where "postAggInterval's worth is aggNum points"
	//
	// this also assures that in the special case where people request MaxDataPoints=1 we will always consolidate
	// all points together and don't trim a significant amount of the points
	// that are expected to go into the aggregation
	// e.g. consider a case where we have points with ts 140,150,160,170
	// aggNum = aggEvery(4/1) = 4, postAggInterval is thus 40.
	// strict application of nudging would return 1 point with ts=200 (aggregation of all points 170-200 which is 1 point)
	// and strip the first 3 points,
	// which is not what we want. since we only have a small set of points, better to incorporate all points into 1 bucket with ts 170.
	// note that in this case (where we don't nudge) the timestamps in output are not cleanly divisible by postAggInterval

	// we only start stripping if we have more than 2*4=8 points
	// see the unit tests which explore cases like this (TestConsolidateNudgedNoTrimDueToNotManyPoints)
	if len(points) > int(2*aggNum) {
		_, num := nudge(points[0].Ts, interval, aggNum)
		points = points[num:]
	}
	return points
}

// Nudge computes the parameters for nudging:
// * the diff to add to the old start point to compute the timestamp of the new start point
// * number of points to strip from the start.
// let's say a series has points A,B,C,D and we must consolidate with numAgg=2.
// if we wait a step, point E appears into the visible window and A will slide out of the window.
// there's a few approaches you can take wrt such changes across refreshes:
// 1) naive old approach:
//    on first load return consolidate(A,B), consolidate(C,D)
//    after a step, return consolidate(B,C), consolidate(D,E)
//    => this looks weird across refreshes:
//       both the values as well as the timestamps change everywhere, points jump around on the chart
// 2) graphite-style nudging: trim a few of the first points away as needed, such that the first TS
//    will be the first one to fill a "full" aggregation bucket. (note: assumes input is quantized!)
//    on first load return consolidate(A,B), consolidate(C,D)
//    after a step, return consolidate(C,D), consolidate(E)
//    => same points are always consolidated together, no jumping around.
//    => simple to understand, but not the most performant (fetches/processes some points needlessly)
//    => tends to introduce emptyness in graphs right after the requested start.
//       (because Grafana plots from requested start, not returned start, and so there will be some empty space
//       where we trimmed points)
// 3) similar to 2, but don't trim, rather consolidate the leftovers both on the start and at the end.
//    on first load return consolidate(A,B), consolidate(C,D)
//    after a step, return consolidate(B), consolidate(C,D), consolidate(E)
//    => same points are always consolidated together, no jumping around.
//    => only datapoint up front and at the end may jump around, but not the vast majority
//    => no discarding of points
//    => requires a large code change though, as it makes it harder to honor MaxDataPoints.
//       e.g. MDP=1, you have 5 points and aggNum is 5, if alignment is improper, it would cause
//       2 output points, so we would have to rewrite a lot of code, no longer compute AggNum in advance etc
//
// note that with all 3 approaches, we always consolidate leftovers at the end together, so with any approach
// the last point may jump around (see Consolidate function)
// for now, and for simplicity we just implement the 2nd approach. it's also the only one that assures MDP is strictly
// honored (see last point of approach 3, which also affects approach 1)
func nudge(start, preAggInterval, aggNum uint32) (uint32, int) {
	postAggInterval := preAggInterval * aggNum
	var num int
	var diff uint32
	// two important principles here:
	// 1) aggregation buckets have timestamps that are cleanly divisible by postAggInterval.
	// 2) we never make data pretend to be able to predict the future,
	//    e.g. when aggregating a spike, the spike should never move to an earlier timestamp
	//    in other words, data should only ever move to a later timestamp when being aggregated, never to the past.
	// it follows that the start of (first point going into) an aggregation bucket
	// is the point that has a timestamp of 1 preAggInterval after the cleanly-divisible-by-postAggInterval timestamp.

	// let's clarify all this with an example.
	// consider: preAggInterval = 5, aggNum = 4 => postAggInterval = 20
	// points marked with a * are the first points going into an aggregation interval

	// input   consolidate into    output points
	// 15      ▼
	// 20        ►                 20
	// 25 *    ▼
	// 30      ▼
	// 35      ▼
	// 40        ►                 40
	// 45 *    ▼
	// 50      ▼
	// 55      ▼
	// 60        ►                 60
	// 65 *    ▼
	// 70      ▼
	// 75      ▼
	//                             80

	// with our nudging approach, we strip all the points that do not form a full aggregation bucket.
	// IOW the desired first point must be the the first point of a bucket.
	// in the example above, it means we strip the first 2 points and start at ts=25 (which will go into ts=40).

	// move start until it maps to the first point of an aggregation bucket
	remainder := (start - preAggInterval) % postAggInterval
	if remainder > 0 {
		diff = postAggInterval - remainder
		num = int(diff / preAggInterval)
	}
	// following the above example:
	// remainder = (15 - 5) % 20 = 10
	// diff     = 20 - 10 = 10 // start will be moved up by 10s, from 15 to 25
	// num      = 10/5 = 2 // which means we will strip the first two points

	return diff, num
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/gitpods/gitpods/authorization"
	"github.com/gitpods/gitpods/cmd"
	"github.com/gitpods/gitpods/repository"
	"github.com/gitpods/gitpods/resolver"
	"github.com/gitpods/gitpods/session"
	"github.com/gitpods/gitpods/storage"
	"github.com/gitpods/gitpods/user"
	"github.com/go-chi/chi"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics"
	"github.com/go-kit/kit/metrics/prometheus"
	_ "github.com/lib/pq"
	graphql "github.com/neelance/graphql-go"
	"github.com/neelance/graphql-go/relay"
	"github.com/oklog/oklog/pkg/group"
	prom "github.com/prometheus/client_golang/prometheus"
	jaeger "github.com/uber/jaeger-client-go"
	"github.com/uber/jaeger-client-go/config"
	"github.com/urfave/cli"
)

type apiConf struct {
	HTTPAddr        string
	HTTPPrivateAddr string
	APIPrefix       string
	DatabaseDriver  string
	DatabaseDSN     string
	LogJSON         bool
	LogLevel        string
	Secret          string
	StorageGRPCURL  string
	TracingURL      string
}

var (
	apiConfig = apiConf{}

	apiFlags = []cli.Flag{
		cli.StringFlag{
			Name:        cmd.FlagAPIPrefix,
			EnvVar:      cmd.EnvAPIPrefix,
			Usage:       "The prefix the api is serving from, default: /",
			Value:       "/",
			Destination: &apiConfig.APIPrefix,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDriver,
			EnvVar:      cmd.EnvDatabaseDriver,
			Usage:       "The database driver to use: memory & postgres",
			Value:       "postgres",
			Destination: &apiConfig.DatabaseDriver,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDSN,
			EnvVar:      cmd.EnvDatabaseDSN,
			Usage:       "The database connection data",
			Destination: &apiConfig.DatabaseDSN,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPAddr,
			EnvVar:      cmd.EnvHTTPAddr,
			Usage:       "The address gitpods API runs on",
			Value:       ":3020",
			Destination: &apiConfig.HTTPAddr,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPPrivateAddr,
			EnvVar:      cmd.EnvHTTPPrivateAddr,
			Usage:       "The address gitpods runs a http server only for internal access",
			Value:       ":3021",
			Destination: &apiConfig.HTTPPrivateAddr,
		},
		cli.BoolFlag{
			Name:        cmd.FlagLogJSON,
			EnvVar:      cmd.EnvLogJSON,
			Usage:       "The logger will log json lines",
			Destination: &apiConfig.LogJSON,
		},
		cli.StringFlag{
			Name:        cmd.FlagLogLevel,
			EnvVar:      cmd.EnvLogLevel,
			Usage:       "The log level to filter logs with before printing",
			Value:       "info",
			Destination: &apiConfig.LogLevel,
		},
		cli.StringFlag{
			Name:        cmd.FlagSecret,
			EnvVar:      cmd.EnvSecret,
			Usage:       "This secret is going to be used to generate cookies",
			Destination: &apiConfig.Secret,
		},
		cli.StringFlag{
			Name:        cmd.FlagStorageGRPCURL,
			EnvVar:      cmd.EnvStorageGRPCURL,
			Usage:       "The storage's gprc url to connect with",
			Destination: &apiConfig.StorageGRPCURL,
		},
		cli.StringFlag{
			Name:        cmd.FlagTracingURL,
			EnvVar:      cmd.EnvTracingURL,
			Usage:       "The url to send spans for tracing to",
			Destination: &apiConfig.TracingURL,
		},
	}
)

func apiAction(c *cli.Context) error {
	if apiConfig.Secret == "" {
		return errors.New("the secret for the api can't be empty")
	}

	logger := cmd.NewLogger(apiConfig.LogJSON, apiConfig.LogLevel)
	logger = log.WithPrefix(logger, "app", c.App.Name)

	apiMetrics := apiMetrics()

	if apiConfig.TracingURL != "" {
		traceConfig := config.Configuration{
			Sampler: &config.SamplerConfig{
				Type:  jaeger.SamplerTypeConst,
				Param: 1,
			},
			Reporter: &config.ReporterConfig{
				LocalAgentHostPort: apiConfig.TracingURL,
			},
		}

		traceCloser, err := traceConfig.InitGlobalTracer(c.App.Name)
		if err != nil {
			return err
		}
		defer traceCloser.Close()

		level.Info(logger).Log(
			"msg", "tracing enabled",
			"addr", apiConfig.TracingURL,
		)
	} else {
		level.Info(logger).Log("msg", "tracing is disabled, no url given")
	}

	//
	// Stores
	//
	var (
		repositories repository.Store
		sessions     session.Store
		users        user.Store
	)

	switch apiConfig.DatabaseDriver {
	default:
		db, err := sql.Open("postgres", apiConfig.DatabaseDSN)
		if err != nil {
			return err
		}
		defer db.Close()

		users = user.NewPostgresStore(db)
		sessions = session.NewPostgresStore(db)
		repositories = repository.NewPostgresStore(db)
	}

	//
	// Storage
	//
	storageClient, err := storage.NewClient(apiConfig.StorageGRPCURL)
	if err != nil {
		return err
	}

	//
	// Services
	//
	var ss session.Service
	ss = session.NewService(sessions)
	ss = session.NewMetricsService(ss, apiMetrics.SessionsCreated, apiMetrics.SessionsCleared)
	ss = session.NewTracingService(ss)

	var as authorization.Service
	as = authorization.NewService(users.(authorization.Store), ss)
	as = authorization.NewLoggingService(log.WithPrefix(logger, "service", "authorization"), as)
	as = authorization.NewMetricsService(apiMetrics.LoginAttempts, as)
	as = authorization.NewTracingService(as)

	var us user.Service
	us = user.NewService(users)
	us = user.NewLoggingService(log.WithPrefix(logger, "service", "user"), us)
	us = user.NewTracingService(us)

	var rs repository.Service
	rs = repository.NewService(repositories, storageClient)
	rs = repository.NewLoggingService(log.WithPrefix(logger, "service", "repository"), rs)
	rs = repository.NewTracingService(rs)

	//
	// Resolvers
	//
	res := &resolver.Resolver{
		RepositoryResolver: resolver.NewRepository(rs, us),
		TreeResolver:       resolver.NewTree(rs),
		UserResolver:       resolver.NewUser(rs, us),
	}
	schema := graphql.MustParseSchema(resolver.Schema, res)

	//
	// Router
	//
	router := chi.NewRouter()
	{
		router.Use(cmd.NewRequestLogger(logger))

		// Wrap the router inside a Router handler to make it possible to listen on / or on /api.
		// Change via APIPrefix.
		router.Route(apiConfig.APIPrefix, func(router chi.Router) {
			router.Get("/", func(w http.ResponseWriter, r *http.Request) {
				w.Write(page)
			})

			router.Mount("/authorize", authorization.NewHandler(as))

			router.Group(func(router chi.Router) {
				router.Use(session.Authorized(ss))
				router.Mount("/query", &relay.Handler{Schema: schema})
			})
		})

		if apiConfig.APIPrefix != "/" {
			router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
				fmt.Fprintln(w, "API is available at ", apiConfig.APIPrefix)
			})
		}

		router.NotFound(func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte(`{"error":"Not Found"}`))
		})
	}

	server := &http.Server{
		Addr:    apiConfig.HTTPAddr,
		Handler: router,
	}

	privateRouter := chi.NewRouter()
	privateRouter.Get("/healthz", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, http.StatusText(http.StatusOK))
	})
	privateRouter.Mount("/metrics", prom.UninstrumentedHandler())
	privateRouter.Get("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "0.0.0") // TODO: Return json
	})

	privateServer := &http.Server{
		Addr:    apiConfig.HTTPPrivateAddr,
		Handler: privateRouter,
	}

	var gr group.Group
	{
		gr.Add(func() error {
			dur := time.Minute
			level.Info(logger).Log("msg", "starting session cleaner", "interval", dur)
			for {
				if _, err := ss.ClearSessions(context.TODO()); err != nil {
					return err
				}
				time.Sleep(dur)
			}
		}, func(err error) {
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting gitpods api",
				"addr", apiConfig.HTTPAddr,
			)
			return server.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := server.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "http server shutdown gracefully")
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting internal gitpods api",
				"addr", apiConfig.HTTPPrivateAddr,
			)
			return privateServer.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := privateServer.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown internal http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "internal http server shutdown gracefully")
		})
	}

	return gr.Run()
}

type APIMetrics struct {
	LoginAttempts   metrics.Counter
	SessionsCreated metrics.Counter
	SessionsCleared metrics.Counter
}

func apiMetrics() *APIMetrics {
	namespace := "gitpods"

	return &APIMetrics{
		LoginAttempts: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "authentication",
			Name:      "login_attempts_total",
			Help:      "Number of login attempts that succeeded and failed",
		}, []string{"status"}),
		SessionsCreated: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "created_total",
			Help:      "Number of created sessions",
		}, []string{}),
		SessionsCleared: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "cleared_total",
			Help:      "Number of cleared sessions",
		}, []string{}),
	}
}

var page = []byte(`
<!DOCTYPE html>

		<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.7.8/graphiql.css" />

Loading...

`)
<commit_msg>Add NoCache middleware to /query endpoint<commit_after>package main

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/gitpods/gitpods/authorization"
	"github.com/gitpods/gitpods/cmd"
	"github.com/gitpods/gitpods/repository"
	"github.com/gitpods/gitpods/resolver"
	"github.com/gitpods/gitpods/session"
	"github.com/gitpods/gitpods/storage"
	"github.com/gitpods/gitpods/user"
	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics"
	"github.com/go-kit/kit/metrics/prometheus"
	_ "github.com/lib/pq"
	graphql "github.com/neelance/graphql-go"
	"github.com/neelance/graphql-go/relay"
	"github.com/oklog/oklog/pkg/group"
	prom "github.com/prometheus/client_golang/prometheus"
	jaeger "github.com/uber/jaeger-client-go"
	"github.com/uber/jaeger-client-go/config"
	"github.com/urfave/cli"
)

type apiConf struct {
	HTTPAddr        string
	HTTPPrivateAddr string
	APIPrefix       string
	DatabaseDriver  string
	DatabaseDSN     string
	LogJSON         bool
	LogLevel        string
	Secret          string
	StorageGRPCURL  string
	TracingURL      string
}

var (
	apiConfig = apiConf{}

	apiFlags = []cli.Flag{
		cli.StringFlag{
			Name:        cmd.FlagAPIPrefix,
			EnvVar:      cmd.EnvAPIPrefix,
			Usage:       "The prefix the api is serving from, default: /",
			Value:       "/",
			Destination: &apiConfig.APIPrefix,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDriver,
			EnvVar:      cmd.EnvDatabaseDriver,
			Usage:       "The database driver to use: memory & postgres",
			Value:       "postgres",
			Destination: &apiConfig.DatabaseDriver,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDSN,
			EnvVar:      cmd.EnvDatabaseDSN,
			Usage:       "The database connection data",
			Destination: &apiConfig.DatabaseDSN,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPAddr,
			EnvVar:      cmd.EnvHTTPAddr,
			Usage:       "The address gitpods API runs on",
			Value:       ":3020",
			Destination: &apiConfig.HTTPAddr,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPPrivateAddr,
			EnvVar:      cmd.EnvHTTPPrivateAddr,
			Usage:       "The address gitpods runs a http server only for internal access",
			Value:       ":3021",
			Destination: &apiConfig.HTTPPrivateAddr,
		},
		cli.BoolFlag{
			Name:        cmd.FlagLogJSON,
			EnvVar:      cmd.EnvLogJSON,
			Usage:       "The logger will log json lines",
			Destination: &apiConfig.LogJSON,
		},
		cli.StringFlag{
			Name:        cmd.FlagLogLevel,
			EnvVar:      cmd.EnvLogLevel,
			Usage:       "The log level to filter logs with before printing",
			Value:       "info",
			Destination: &apiConfig.LogLevel,
		},
		cli.StringFlag{
			Name:        cmd.FlagSecret,
			EnvVar:      cmd.EnvSecret,
			Usage:       "This secret is going to be used to generate cookies",
			Destination: &apiConfig.Secret,
		},
		cli.StringFlag{
			Name:        cmd.FlagStorageGRPCURL,
			EnvVar:      cmd.EnvStorageGRPCURL,
			Usage:       "The storage's gprc url to connect with",
			Destination: &apiConfig.StorageGRPCURL,
		},
		cli.StringFlag{
			Name:        cmd.FlagTracingURL,
			EnvVar:      cmd.EnvTracingURL,
			Usage:       "The url to send spans for tracing to",
			Destination: &apiConfig.TracingURL,
		},
	}
)

func apiAction(c *cli.Context) error {
	if apiConfig.Secret == "" {
		return errors.New("the secret for the api can't be empty")
	}

	logger := cmd.NewLogger(apiConfig.LogJSON, apiConfig.LogLevel)
	logger = log.WithPrefix(logger, "app", c.App.Name)

	apiMetrics := apiMetrics()

	if apiConfig.TracingURL != "" {
		traceConfig := config.Configuration{
			Sampler: &config.SamplerConfig{
				Type:  jaeger.SamplerTypeConst,
				Param: 1,
			},
			Reporter: &config.ReporterConfig{
				LocalAgentHostPort: apiConfig.TracingURL,
			},
		}

		traceCloser, err := traceConfig.InitGlobalTracer(c.App.Name)
		if err != nil {
			return err
		}
		defer traceCloser.Close()

		level.Info(logger).Log(
			"msg", "tracing enabled",
			"addr", apiConfig.TracingURL,
		)
	} else {
		level.Info(logger).Log("msg", "tracing is disabled, no url given")
	}

	//
	// Stores
	//
	var (
		repositories repository.Store
		sessions     session.Store
		users        user.Store
	)

	switch apiConfig.DatabaseDriver {
	default:
		db, err := sql.Open("postgres", apiConfig.DatabaseDSN)
		if err != nil {
			return err
		}
		defer db.Close()

		users = user.NewPostgresStore(db)
		sessions = session.NewPostgresStore(db)
		repositories = repository.NewPostgresStore(db)
	}

	//
	// Storage
	//
	storageClient, err := storage.NewClient(apiConfig.StorageGRPCURL)
	if err != nil {
		return err
	}

	//
	// Services
	//
	var ss session.Service
	ss = session.NewService(sessions)
	ss = session.NewMetricsService(ss, apiMetrics.SessionsCreated, apiMetrics.SessionsCleared)
	ss = session.NewTracingService(ss)

	var as authorization.Service
	as = authorization.NewService(users.(authorization.Store), ss)
	as = authorization.NewLoggingService(log.WithPrefix(logger, "service", "authorization"), as)
	as = authorization.NewMetricsService(apiMetrics.LoginAttempts, as)
	as = authorization.NewTracingService(as)

	var us user.Service
	us = user.NewService(users)
	us = user.NewLoggingService(log.WithPrefix(logger, "service", "user"), us)
	us = user.NewTracingService(us)

	var rs repository.Service
	rs = repository.NewService(repositories, storageClient)
	rs = repository.NewLoggingService(log.WithPrefix(logger, "service", "repository"), rs)
	rs = repository.NewTracingService(rs)

	//
	// Resolvers
	//
	res := &resolver.Resolver{
		RepositoryResolver: resolver.NewRepository(rs, us),
		TreeResolver:       resolver.NewTree(rs),
		UserResolver:       resolver.NewUser(rs, us),
	}
	schema := graphql.MustParseSchema(resolver.Schema, res)

	//
	// Router
	//
	router := chi.NewRouter()
	{
		router.Use(cmd.NewRequestLogger(logger))

		// Wrap the router inside a Router handler to make it possible to listen on / or on /api.
		// Change via APIPrefix.
		router.Route(apiConfig.APIPrefix, func(router chi.Router) {
			router.Get("/", func(w http.ResponseWriter, r *http.Request) {
				w.Write(page)
			})

			router.Mount("/authorize", authorization.NewHandler(as))

			router.Group(func(router chi.Router) {
				router.Use(session.Authorized(ss))
				router.Mount("/query", middleware.NoCache(&relay.Handler{Schema: schema}))
			})
		})

		if apiConfig.APIPrefix != "/" {
			router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
				fmt.Fprintln(w, "API is available at ", apiConfig.APIPrefix)
			})
		}

		router.NotFound(func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte(`{"error":"Not Found"}`))
		})
	}

	server := &http.Server{
		Addr:    apiConfig.HTTPAddr,
		Handler: router,
	}

	privateRouter := chi.NewRouter()
	privateRouter.Get("/healthz", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, http.StatusText(http.StatusOK))
	})
	privateRouter.Mount("/metrics", prom.UninstrumentedHandler())
	privateRouter.Get("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "0.0.0") // TODO: Return json
	})

	privateServer := &http.Server{
		Addr:    apiConfig.HTTPPrivateAddr,
		Handler: privateRouter,
	}

	var gr group.Group
	{
		gr.Add(func() error {
			dur := time.Minute
			level.Info(logger).Log("msg", "starting session cleaner", "interval", dur)
			for {
				if _, err := ss.ClearSessions(context.TODO()); err != nil {
					return err
				}
				time.Sleep(dur)
			}
		}, func(err error) {
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting gitpods api",
				"addr", apiConfig.HTTPAddr,
			)
			return server.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := server.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "http server shutdown gracefully")
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting internal gitpods api",
				"addr", apiConfig.HTTPPrivateAddr,
			)
			return privateServer.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := privateServer.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown internal http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "internal http server shutdown gracefully")
		})
	}

	return gr.Run()
}

type APIMetrics struct {
	LoginAttempts   metrics.Counter
	SessionsCreated metrics.Counter
	SessionsCleared metrics.Counter
}

func apiMetrics() *APIMetrics {
	namespace := "gitpods"

	return &APIMetrics{
		LoginAttempts: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "authentication",
			Name:      "login_attempts_total",
			Help:      "Number of login attempts that succeeded and failed",
		}, []string{"status"}),
		SessionsCreated: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "created_total",
			Help:      "Number of created sessions",
		}, []string{}),
		SessionsCleared: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "cleared_total",
			Help:      "Number of cleared sessions",
		}, []string{}),
	}
}

var page = []byte(`
<!DOCTYPE html>

		<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.7.8/graphiql.css" />

Loading...

`)
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	api "github.com/gogits/go-gogs-client"

	"github.com/gogits/gogs/modules/git"
	"github.com/gogits/gogs/modules/setting"
)

const (
	SLACK_COLOR string = "#dd4b39"
)

type SlackMeta struct {
	Channel string `json:"channel"`
}

type SlackPayload struct {
	Channel     string            `json:"channel"`
	Text        string            `json:"text"`
	Username    string            `json:"username"`
	IconUrl     string            `json:"icon_url"`
	UnfurlLinks int               `json:"unfurl_links"`
	LinkNames   int               `json:"link_names"`
	Attachments []SlackAttachment `json:"attachments"`
}

type SlackAttachment struct {
	Color string `json:"color"`
	Text  string `json:"text"`
}

func (p *SlackPayload) SetSecret(_ string) {}

func (p *SlackPayload) JSONPayload() ([]byte, error) {
	data, err := json.Marshal(p)
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

// see: https://api.slack.com/docs/formatting
func SlackTextFormatter(s string) string {
	// take only first line of commit
	first := strings.Split(s, "\n")[0]
	// replace & < >
	first = strings.Replace(first, "&", "&", -1)
	first = strings.Replace(first, "<", "<", -1)
	first = strings.Replace(first, ">", ">", -1)
	return first
}

func SlackLinkFormatter(url string, text string) string {
	return fmt.Sprintf("<%s|%s>", url, SlackTextFormatter(text))
}

func getSlackCreatePayload(p *api.CreatePayload, slack *SlackMeta) (*SlackPayload, error) {
	// created tag/branch
	refName := git.RefEndName(p.Ref)

	repoLink := SlackLinkFormatter(p.Repo.URL, p.Repo.Name)
	refLink := SlackLinkFormatter(p.Repo.URL+"/src/"+refName, refName)
	text := fmt.Sprintf("[%s:%s] %s created by %s", repoLink, refLink, p.RefType, p.Sender.UserName)

	return &SlackPayload{
		Channel:  slack.Channel,
		Text:     text,
		Username: setting.AppName,
		IconUrl:  setting.AppUrl + "/img/favicon.png",
	}, nil
}

func getSlackPushPayload(p *api.PushPayload, slack *SlackMeta) (*SlackPayload, error) {
	// n new commits
	var (
		branchName   = git.RefEndName(p.Ref)
		commitString string
	)

	if len(p.Commits) == 1 {
		commitString = "1 new commit"
		if len(p.CompareUrl) > 0 {
			commitString = SlackLinkFormatter(p.CompareUrl, commitString)
		}
	} else {
		commitString = fmt.Sprintf("%d new commits", len(p.Commits))
		if p.CompareUrl != "" {
			commitString = SlackLinkFormatter(p.CompareUrl, commitString)
		}
	}

	repoLink := SlackLinkFormatter(p.Repo.URL, p.Repo.Name)
	branchLink := SlackLinkFormatter(p.Repo.URL+"/src/"+branchName, branchName)
	text := fmt.Sprintf("[%s:%s] %s pushed by %s", repoLink, branchLink, commitString, p.Pusher.Name)

	var attachmentText string
	// for each commit, generate attachment text
	for i, commit := range p.Commits {
		attachmentText += fmt.Sprintf("%s: %s - %s", SlackLinkFormatter(commit.URL, commit.ID[:7]), SlackTextFormatter(commit.Message), SlackTextFormatter(commit.Author.Name))
		// add linebreak to each commit but the last
		if i < len(p.Commits)-1 {
			attachmentText += "\n"
		}
	}

	slackAttachments := []SlackAttachment{{Color: SLACK_COLOR, Text: attachmentText}}

	return &SlackPayload{
		Channel:     slack.Channel,
		Text:        text,
		Username:    setting.AppName,
		IconUrl:     setting.AppUrl + "/img/favicon.png",
		Attachments: slackAttachments,
	}, nil
}

func GetSlackPayload(p api.Payloader, event HookEventType, meta string) (*SlackPayload, error) {
	s := new(SlackPayload)

	slack := &SlackMeta{}
	if err := json.Unmarshal([]byte(meta), &slack); err != nil {
		return s, errors.New("GetSlackPayload meta json:" + err.Error())
	}

	switch event {
	case HOOK_EVENT_CREATE:
		return getSlackCreatePayload(p.(*api.CreatePayload), slack)
	case HOOK_EVENT_PUSH:
		return getSlackPushPayload(p.(*api.PushPayload), slack)
	}

	return s, nil
}
<commit_msg>slack JSON indent<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	api "github.com/gogits/go-gogs-client"

	"github.com/gogits/gogs/modules/git"
	"github.com/gogits/gogs/modules/setting"
)

const (
	SLACK_COLOR string = "#dd4b39"
)

type SlackMeta struct {
	Channel string `json:"channel"`
}

type SlackPayload struct {
	Channel     string            `json:"channel"`
	Text        string            `json:"text"`
	Username    string            `json:"username"`
	IconUrl     string            `json:"icon_url"`
	UnfurlLinks int               `json:"unfurl_links"`
	LinkNames   int               `json:"link_names"`
	Attachments []SlackAttachment `json:"attachments"`
}

type SlackAttachment struct {
	Color string `json:"color"`
	Text  string `json:"text"`
}

func (p *SlackPayload) SetSecret(_ string) {}

func (p *SlackPayload) JSONPayload() ([]byte, error) {
	data, err := json.MarshalIndent(p, "", "  ")
	if err != nil {
		return []byte{}, err
	}
	return data, nil
}

// see: https://api.slack.com/docs/formatting
func SlackTextFormatter(s string) string {
	// take only first line of commit
	first := strings.Split(s, "\n")[0]
	// replace & < >
	first = strings.Replace(first, "&", "&", -1)
	first = strings.Replace(first, "<", "<", -1)
	first = strings.Replace(first, ">", ">", -1)
	return first
}

func SlackLinkFormatter(url string, text string) string {
	return fmt.Sprintf("<%s|%s>", url, SlackTextFormatter(text))
}

func getSlackCreatePayload(p *api.CreatePayload, slack *SlackMeta) (*SlackPayload, error) {
	// created tag/branch
	refName := git.RefEndName(p.Ref)

	repoLink := SlackLinkFormatter(p.Repo.URL, p.Repo.Name)
	refLink := SlackLinkFormatter(p.Repo.URL+"/src/"+refName, refName)
	text := fmt.Sprintf("[%s:%s] %s created by %s", repoLink, refLink, p.RefType, p.Sender.UserName)

	return &SlackPayload{
		Channel:  slack.Channel,
		Text:     text,
		Username: setting.AppName,
		IconUrl:  setting.AppUrl + "/img/favicon.png",
	}, nil
}

func getSlackPushPayload(p *api.PushPayload, slack *SlackMeta) (*SlackPayload, error) {
	// n new commits
	var (
		branchName   = git.RefEndName(p.Ref)
		commitString string
	)

	if len(p.Commits) == 1 {
		commitString = "1 new commit"
		if len(p.CompareUrl) > 0 {
			commitString = SlackLinkFormatter(p.CompareUrl, commitString)
		}
	} else {
		commitString = fmt.Sprintf("%d new commits", len(p.Commits))
		if p.CompareUrl != "" {
			commitString = SlackLinkFormatter(p.CompareUrl, commitString)
		}
	}

	repoLink := SlackLinkFormatter(p.Repo.URL, p.Repo.Name)
	branchLink := SlackLinkFormatter(p.Repo.URL+"/src/"+branchName, branchName)
	text := fmt.Sprintf("[%s:%s] %s pushed by %s", repoLink, branchLink, commitString, p.Pusher.Name)

	var attachmentText string
	// for each commit, generate attachment text
	for i, commit := range p.Commits {
		attachmentText += fmt.Sprintf("%s: %s - %s", SlackLinkFormatter(commit.URL, commit.ID[:7]), SlackTextFormatter(commit.Message), SlackTextFormatter(commit.Author.Name))
		// add linebreak to each commit but the last
		if i < len(p.Commits)-1 {
			attachmentText += "\n"
		}
	}

	slackAttachments := []SlackAttachment{{Color: SLACK_COLOR, Text: attachmentText}}

	return &SlackPayload{
		Channel:     slack.Channel,
		Text:        text,
		Username:    setting.AppName,
		IconUrl:     setting.AppUrl + "/img/favicon.png",
		Attachments: slackAttachments,
	}, nil
}

func GetSlackPayload(p api.Payloader, event HookEventType, meta string) (*SlackPayload, error) {
	s := new(SlackPayload)

	slack := &SlackMeta{}
	if err := json.Unmarshal([]byte(meta), &slack); err != nil {
		return s, errors.New("GetSlackPayload meta json:" + err.Error())
	}

	switch event {
	case HOOK_EVENT_CREATE:
		return getSlackCreatePayload(p.(*api.CreatePayload), slack)
	case HOOK_EVENT_PUSH:
		return getSlackPushPayload(p.(*api.PushPayload), slack)
	}

	return s, nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/gitpods/gitpods/authorization"
	"github.com/gitpods/gitpods/cmd"
	"github.com/gitpods/gitpods/repository"
	"github.com/gitpods/gitpods/resolver"
	"github.com/gitpods/gitpods/session"
	"github.com/gitpods/gitpods/storage"
	"github.com/gitpods/gitpods/user"
	"github.com/go-chi/chi"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics"
	"github.com/go-kit/kit/metrics/prometheus"
	_ "github.com/lib/pq"
	graphql "github.com/neelance/graphql-go"
	"github.com/neelance/graphql-go/relay"
	"github.com/oklog/oklog/pkg/group"
	prom "github.com/prometheus/client_golang/prometheus"
	"github.com/urfave/cli"
	"google.golang.org/grpc"
)

type apiConf struct {
	HTTPAddr        string
	HTTPPrivateAddr string
	APIPrefix       string
	DatabaseDriver  string
	DatabaseDSN     string
	LogJSON         bool
	LogLevel        string
	Secret          string
	StorageGRPCURL  string
}

var (
	apiConfig = apiConf{}

	apiFlags = []cli.Flag{
		cli.StringFlag{
			Name:        cmd.FlagAPIPrefix,
			EnvVar:      cmd.EnvAPIPrefix,
			Usage:       "The prefix the api is serving from, default: /",
			Value:       "/",
			Destination: &apiConfig.APIPrefix,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDriver,
			EnvVar:      cmd.EnvDatabaseDriver,
			Usage:       "The database driver to use: memory & postgres",
			Value:       "postgres",
			Destination: &apiConfig.DatabaseDriver,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDSN,
			EnvVar:      cmd.EnvDatabaseDSN,
			Usage:       "The database connection data",
			Destination: &apiConfig.DatabaseDSN,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPAddr,
			EnvVar:      cmd.EnvHTTPAddr,
			Usage:       "The address gitpods API runs on",
			Value:       ":3020",
			Destination: &apiConfig.HTTPAddr,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPPrivateAddr,
			EnvVar:      cmd.EnvHTTPPrivateAddr,
			Usage:       "The address gitpods runs a http server only for internal access",
			Value:       ":3021",
			Destination: &apiConfig.HTTPPrivateAddr,
		},
		cli.BoolFlag{
			Name:        cmd.FlagLogJSON,
			EnvVar:      cmd.EnvLogJSON,
			Usage:       "The logger will log json lines",
			Destination: &apiConfig.LogJSON,
		},
		cli.StringFlag{
			Name:        cmd.FlagLogLevel,
			EnvVar:      cmd.EnvLogLevel,
			Usage:       "The log level to filter logs with before printing",
			Value:       "info",
			Destination: &apiConfig.LogLevel,
		},
		cli.StringFlag{
			Name:        cmd.FlagSecret,
			EnvVar:      cmd.EnvSecret,
			Usage:       "This secret is going to be used to generate cookies",
			Destination: &apiConfig.Secret,
		},
		cli.StringFlag{
			Name:        cmd.FlagStorageGRPCURL,
			EnvVar:      cmd.EnvStorageGRPCURL,
			Usage:       "The storage's gprc url to connect with",
			Destination: &apiConfig.StorageGRPCURL,
		},
	}
)

func apiAction(c *cli.Context) error {
	if apiConfig.Secret == "" {
		return errors.New("the secret for the api can't be empty")
	}

	logger := cmd.NewLogger(apiConfig.LogJSON, apiConfig.LogLevel)
	logger = log.WithPrefix(logger, "app", "api")

	apiMetrics := apiMetrics()

	//
	// Stores
	//
	var (
		repositories repository.Store
		sessions     session.Store
		users        user.Store
	)

	switch apiConfig.DatabaseDriver {
	default:
		db, err := sql.Open("postgres", apiConfig.DatabaseDSN)
		if err != nil {
			return err
		}
		defer db.Close()

		users = user.NewPostgresStore(db)
		sessions = session.NewPostgresStore(db)
		repositories = repository.NewPostgresStore(db)
	}

	//
	// Storage
	//
	var opts []grpc.DialOption
	opts = append(opts, grpc.WithInsecure())

	storageConn, err := grpc.DialContext(context.Background(), apiConfig.StorageGRPCURL, opts...)
	if err != nil {
		return err
	}
	storageClient := storage.NewClient(storageConn)

	//
	// Services
	//
	var ss session.Service
	ss = session.NewService(sessions)
	ss = session.NewMetricsService(ss, apiMetrics.SessionsCreated, apiMetrics.SessionsCleared)

	var as authorization.Service
	as = authorization.NewService(users.(authorization.Store), ss)
	as = authorization.NewLoggingService(log.WithPrefix(logger, "service", "authorization"), as)
	as = authorization.NewMetricsService(apiMetrics.LoginAttempts, as)

	var us user.Service
	us = user.NewService(users)
	us = user.NewLoggingService(log.WithPrefix(logger, "service", "user"), us)

	var rs repository.Service
	rs = repository.NewService(repositories, storageClient)
	rs = repository.NewLoggingService(log.WithPrefix(logger, "service", "repository"), rs)

	//
	// Resolvers
	//
	res := &resolver.Resolver{
		UserResolver:       resolver.NewUser(rs, us),
		RepositoryResolver: resolver.NewRepository(rs, us),
	}
	schema := graphql.MustParseSchema(resolver.Schema, res)

	//
	// Router
	//
	router := chi.NewRouter()
	router.Use(cmd.NewRequestLogger(logger))

	// Wrap the router inside a Router handler to make it possible to listen on / or on /api.
	// Change via APIPrefix.
	router.Route(apiConfig.APIPrefix, func(router chi.Router) {
		router.Get("/", func(w http.ResponseWriter, r *http.Request) {
			w.Write(page)
		})

		router.Mount("/authorize", authorization.NewHandler(as))

		router.Group(func(router chi.Router) {
			router.Use(session.Authorized(ss))

			router.Mount("/query", &relay.Handler{Schema: schema})
			router.Mount("/user", user.NewUserHandler(us))
			router.Mount("/users", user.NewUsersHandler(us))
			router.Mount("/users/{username}/repositories", repository.NewUsersHandler(rs))

			router.Mount("/repositories/{owner}/{name}", repository.NewHandler(rs))
		})
	})

	if apiConfig.APIPrefix != "/" {
		router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintln(w, "API is available at ", apiConfig.APIPrefix)
		})
	}

	router.NotFound(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte(`{"error":"Not Found"}`))
	})

	server := &http.Server{
		Addr:    apiConfig.HTTPAddr,
		Handler: router,
	}

	privateRouter := chi.NewRouter()
	privateRouter.Get("/healthz", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, http.StatusText(http.StatusOK))
	})
	privateRouter.Mount("/metrics", prom.UninstrumentedHandler())
	privateRouter.Get("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "0.0.0") // TODO: Return json
	})

	privateServer := &http.Server{
		Addr:    apiConfig.HTTPPrivateAddr,
		Handler: privateRouter,
	}

	var gr group.Group
	{
		gr.Add(func() error {
			dur := time.Minute
			level.Info(logger).Log("msg", "starting session cleaner", "interval", dur)
			for {
				if _, err := ss.ClearSessions(); err != nil {
					return err
				}
				time.Sleep(dur)
			}
		}, func(err error) {
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting gitpods api",
				"addr", apiConfig.HTTPAddr,
			)
			return server.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := server.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "http server shutdown gracefully")
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting internal gitpods api",
				"addr", apiConfig.HTTPPrivateAddr,
			)
			return privateServer.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := privateServer.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown internal http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "internal http server shutdown gracefully")
		})
	}

	return gr.Run()
}

type APIMetrics struct {
	LoginAttempts   metrics.Counter
	SessionsCreated metrics.Counter
	SessionsCleared metrics.Counter
}

func apiMetrics() *APIMetrics {
	namespace := "gitpods"

	return &APIMetrics{
		LoginAttempts: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "authentication",
			Name:      "login_attempts_total",
			Help:      "Number of login attempts that succeeded and failed",
		}, []string{"status"}),
		SessionsCreated: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "created_total",
			Help:      "Number of created sessions",
		}, []string{}),
		SessionsCleared: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "cleared_total",
			Help:      "Number of cleared sessions",
		}, []string{}),
	}
}

var page = []byte(`
<!DOCTYPE html>

		<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.7.8/graphiql.css" />

Loading...

`)
<commit_msg>Delete old http handlers in api.go<commit_after>package main

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/gitpods/gitpods/authorization"
	"github.com/gitpods/gitpods/cmd"
	"github.com/gitpods/gitpods/repository"
	"github.com/gitpods/gitpods/resolver"
	"github.com/gitpods/gitpods/session"
	"github.com/gitpods/gitpods/storage"
	"github.com/gitpods/gitpods/user"
	"github.com/go-chi/chi"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics"
	"github.com/go-kit/kit/metrics/prometheus"
	_ "github.com/lib/pq"
	graphql "github.com/neelance/graphql-go"
	"github.com/neelance/graphql-go/relay"
	"github.com/oklog/oklog/pkg/group"
	prom "github.com/prometheus/client_golang/prometheus"
	"github.com/urfave/cli"
	"google.golang.org/grpc"
)

type apiConf struct {
	HTTPAddr        string
	HTTPPrivateAddr string
	APIPrefix       string
	DatabaseDriver  string
	DatabaseDSN     string
	LogJSON         bool
	LogLevel        string
	Secret          string
	StorageGRPCURL  string
}

var (
	apiConfig = apiConf{}

	apiFlags = []cli.Flag{
		cli.StringFlag{
			Name:        cmd.FlagAPIPrefix,
			EnvVar:      cmd.EnvAPIPrefix,
			Usage:       "The prefix the api is serving from, default: /",
			Value:       "/",
			Destination: &apiConfig.APIPrefix,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDriver,
			EnvVar:      cmd.EnvDatabaseDriver,
			Usage:       "The database driver to use: memory & postgres",
			Value:       "postgres",
			Destination: &apiConfig.DatabaseDriver,
		},
		cli.StringFlag{
			Name:        cmd.FlagDatabaseDSN,
			EnvVar:      cmd.EnvDatabaseDSN,
			Usage:       "The database connection data",
			Destination: &apiConfig.DatabaseDSN,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPAddr,
			EnvVar:      cmd.EnvHTTPAddr,
			Usage:       "The address gitpods API runs on",
			Value:       ":3020",
			Destination: &apiConfig.HTTPAddr,
		},
		cli.StringFlag{
			Name:        cmd.FlagHTTPPrivateAddr,
			EnvVar:      cmd.EnvHTTPPrivateAddr,
			Usage:       "The address gitpods runs a http server only for internal access",
			Value:       ":3021",
			Destination: &apiConfig.HTTPPrivateAddr,
		},
		cli.BoolFlag{
			Name:        cmd.FlagLogJSON,
			EnvVar:      cmd.EnvLogJSON,
			Usage:       "The logger will log json lines",
			Destination: &apiConfig.LogJSON,
		},
		cli.StringFlag{
			Name:        cmd.FlagLogLevel,
			EnvVar:      cmd.EnvLogLevel,
			Usage:       "The log level to filter logs with before printing",
			Value:       "info",
			Destination: &apiConfig.LogLevel,
		},
		cli.StringFlag{
			Name:        cmd.FlagSecret,
			EnvVar:      cmd.EnvSecret,
			Usage:       "This secret is going to be used to generate cookies",
			Destination: &apiConfig.Secret,
		},
		cli.StringFlag{
			Name:        cmd.FlagStorageGRPCURL,
			EnvVar:      cmd.EnvStorageGRPCURL,
			Usage:       "The storage's gprc url to connect with",
			Destination: &apiConfig.StorageGRPCURL,
		},
	}
)

func apiAction(c *cli.Context) error {
	if apiConfig.Secret == "" {
		return errors.New("the secret for the api can't be empty")
	}

	logger := cmd.NewLogger(apiConfig.LogJSON, apiConfig.LogLevel)
	logger = log.WithPrefix(logger, "app", "api")

	apiMetrics := apiMetrics()

	//
	// Stores
	//
	var (
		repositories repository.Store
		sessions     session.Store
		users        user.Store
	)

	switch apiConfig.DatabaseDriver {
	default:
		db, err := sql.Open("postgres", apiConfig.DatabaseDSN)
		if err != nil {
			return err
		}
		defer db.Close()

		users = user.NewPostgresStore(db)
		sessions = session.NewPostgresStore(db)
		repositories = repository.NewPostgresStore(db)
	}

	//
	// Storage
	//
	var opts []grpc.DialOption
	opts = append(opts, grpc.WithInsecure())

	storageConn, err := grpc.DialContext(context.Background(), apiConfig.StorageGRPCURL, opts...)
	if err != nil {
		return err
	}
	storageClient := storage.NewClient(storageConn)

	//
	// Services
	//
	var ss session.Service
	ss = session.NewService(sessions)
	ss = session.NewMetricsService(ss, apiMetrics.SessionsCreated, apiMetrics.SessionsCleared)

	var as authorization.Service
	as = authorization.NewService(users.(authorization.Store), ss)
	as = authorization.NewLoggingService(log.WithPrefix(logger, "service", "authorization"), as)
	as = authorization.NewMetricsService(apiMetrics.LoginAttempts, as)

	var us user.Service
	us = user.NewService(users)
	us = user.NewLoggingService(log.WithPrefix(logger, "service", "user"), us)

	var rs repository.Service
	rs = repository.NewService(repositories, storageClient)
	rs = repository.NewLoggingService(log.WithPrefix(logger, "service", "repository"), rs)

	//
	// Resolvers
	//
	res := &resolver.Resolver{
		UserResolver:       resolver.NewUser(rs, us),
		RepositoryResolver: resolver.NewRepository(rs, us),
	}
	schema := graphql.MustParseSchema(resolver.Schema, res)

	//
	// Router
	//
	router := chi.NewRouter()
	router.Use(cmd.NewRequestLogger(logger))

	// Wrap the router inside a Router handler to make it possible to listen on / or on /api.
	// Change via APIPrefix.
	router.Route(apiConfig.APIPrefix, func(router chi.Router) {
		router.Get("/", func(w http.ResponseWriter, r *http.Request) {
			w.Write(page)
		})

		router.Mount("/authorize", authorization.NewHandler(as))

		router.Group(func(router chi.Router) {
			router.Use(session.Authorized(ss))
			router.Mount("/query", &relay.Handler{Schema: schema})
		})
	})

	if apiConfig.APIPrefix != "/" {
		router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintln(w, "API is available at ", apiConfig.APIPrefix)
		})
	}

	router.NotFound(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte(`{"error":"Not Found"}`))
	})

	server := &http.Server{
		Addr:    apiConfig.HTTPAddr,
		Handler: router,
	}

	privateRouter := chi.NewRouter()
	privateRouter.Get("/healthz", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, http.StatusText(http.StatusOK))
	})
	privateRouter.Mount("/metrics", prom.UninstrumentedHandler())
	privateRouter.Get("/version", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "0.0.0") // TODO: Return json
	})

	privateServer := &http.Server{
		Addr:    apiConfig.HTTPPrivateAddr,
		Handler: privateRouter,
	}

	var gr group.Group
	{
		gr.Add(func() error {
			dur := time.Minute
			level.Info(logger).Log("msg", "starting session cleaner", "interval", dur)
			for {
				if _, err := ss.ClearSessions(); err != nil {
					return err
				}
				time.Sleep(dur)
			}
		}, func(err error) {
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting gitpods api",
				"addr", apiConfig.HTTPAddr,
			)
			return server.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := server.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "http server shutdown gracefully")
		})
	}
	{
		gr.Add(func() error {
			level.Info(logger).Log(
				"msg", "starting internal gitpods api",
				"addr", apiConfig.HTTPPrivateAddr,
			)
			return privateServer.ListenAndServe()
		}, func(err error) {
			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
			defer cancel()

			if err := privateServer.Shutdown(ctx); err != nil {
				level.Error(logger).Log(
					"msg", "failed to shutdown internal http server gracefully",
					"err", err,
				)
				return
			}
			level.Info(logger).Log("msg", "internal http server shutdown gracefully")
		})
	}

	return gr.Run()
}

type APIMetrics struct {
	LoginAttempts   metrics.Counter
	SessionsCreated metrics.Counter
	SessionsCleared metrics.Counter
}

func apiMetrics() *APIMetrics {
	namespace := "gitpods"

	return &APIMetrics{
		LoginAttempts: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "authentication",
			Name:      "login_attempts_total",
			Help:      "Number of login attempts that succeeded and failed",
		}, []string{"status"}),
		SessionsCreated: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "created_total",
			Help:      "Number of created sessions",
		}, []string{}),
		SessionsCleared: prometheus.NewCounterFrom(prom.CounterOpts{
			Namespace: namespace,
			Subsystem: "sessions",
			Name:      "cleared_total",
			Help:      "Number of cleared sessions",
		}, []string{}),
	}
}

var page = []byte(`
<!DOCTYPE html>

		<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.7.8/graphiql.css" />

Loading...

`)
<|endoftext|>


<commit_before>package etcd

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/coreos/etcd/store"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
)

func TestAndSet(cluster string, key string, prevValue string, value string, ttl uint64) (*store.Response, bool, error) {

	httpPath := path.Join(cluster, "/", version, "/testAndSet/", key)

	//TODO: deal with https
	httpPath = "http://" + httpPath

	v := url.Values{}
	v.Set("value", value)
	v.Set("prevValue", prevValue)

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	var resp *http.Response
	var err error
	// if we connect to a follower, we will retry until we found a leader
	for {
		client := http.Client{}
		resp, err = client.PostForm(httpPath, v)

		if resp != nil {

			if resp.StatusCode == http.StatusTemporaryRedirect {
				httpPath = resp.Header.Get("Location")

				resp.Body.Close()

				if httpPath == "" {
					return nil, false, errors.New("Cannot get redirection location")
				}

				// try to connect the leader
				continue
			} else {
				break
			}

		}

		return nil, false, err
	}

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		resp.Body.Close()
		return nil, false, err
	}

	var result store.Response

	err = json.Unmarshal(b, &result)

	if err != nil {
		resp.Body.Close()
		return nil, false, err
	}
	resp.Body.Close()

	if result.PrevValue == prevValue && result.Value == value {

		return &result, true, nil
	}

	return &result, false, nil

}
<commit_msg>update testandset command<commit_after>package etcd

import (
	"encoding/json"
	"fmt"
	"github.com/coreos/etcd/store"
	"io/ioutil"
	"net/url"
)

func TestAndSet(cluster string, key string, prevValue string, value string, ttl uint64) (*store.Response, bool, error) {
	httpPath := getHttpPath("keys", key)

	v := url.Values{}
	v.Set("value", value)
	v.Set("prevValue", prevValue)

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	resp, err := sendRequest(httpPath, nil, &v)

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		resp.Body.Close()
		return nil, false, err
	}

	var result store.Response

	err = json.Unmarshal(b, &result)

	if err != nil {
		resp.Body.Close()
		return nil, false, err
	}
	resp.Body.Close()

	if result.PrevValue == prevValue && result.Value == value {

		return &result, true, nil
	}

	return &result, false, nil

}
<|endoftext|>