<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mutating delegates admission checks to dynamically configured
// mutating webhooks.
package mutating

import (
	"context"
	"fmt"
	"time"

	jsonpatch "github.com/evanphx/json-patch"
	"github.com/golang/glog"

	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	"k8s.io/api/admissionregistration/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
	"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
	webhookerrors "k8s.io/apiserver/pkg/admission/plugin/webhook/errors"
	"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
	"k8s.io/apiserver/pkg/admission/plugin/webhook/request"
)

type mutatingDispatcher struct {
	cm     *config.ClientManager
	plugin *Plugin
}

func newMutatingDispatcher(p *Plugin) func(cm *config.ClientManager) generic.Dispatcher {
	return func(cm *config.ClientManager) generic.Dispatcher {
		return &mutatingDispatcher{cm, p}
	}
}

var _ generic.Dispatcher = &mutatingDispatcher{}

func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr *generic.VersionedAttributes, relevantHooks []*v1beta1.Webhook) error {
	for _, hook := range relevantHooks {
		t := time.Now()
		err := a.callAttrMutatingHook(ctx, hook, attr)
		admissionmetrics.Metrics.ObserveWebhook(time.Since(t), err != nil, attr.Attributes, "admit", hook.Name)
		if err == nil {
			continue
		}

		ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == v1beta1.Ignore
		if callErr, ok := err.(*webhookerrors.ErrCallingWebhook); ok {
			if ignoreClientCallFailures {
				glog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
				utilruntime.HandleError(callErr)
				continue
			}
			glog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err)
		}
		return apierrors.NewInternalError(err)
	}

	// convert attr.VersionedObject to the internal version in the underlying admission.Attributes
	return a.plugin.scheme.Convert(attr.VersionedObject, attr.Attributes.GetObject(), nil)
}

// note that callAttrMutatingHook updates attr
func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *v1beta1.Webhook, attr *generic.VersionedAttributes) error {
	// Make the webhook request
	request := request.CreateAdmissionReview(attr)
	client, err := a.cm.HookClient(h)
	if err != nil {
		return &webhookerrors.ErrCallingWebhook{WebhookName: h.Name, Reason: err}
	}
	response := &admissionv1beta1.AdmissionReview{}
	if err := client.Post().Context(ctx).Body(&request).Do().Into(response); err != nil {
		return &webhookerrors.ErrCallingWebhook{WebhookName: h.Name, Reason: err}
	}

	if response.Response == nil {
		return &webhookerrors.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Webhook response was absent")}
	}

	if !response.Response.Allowed {
		return webhookerrors.ToStatusErr(h.Name, response.Response.Result)
	}

	patchJS := response.Response.Patch
	if len(patchJS) == 0 {
		return nil
	}
	patchObj, err := jsonpatch.DecodePatch(patchJS)
	if err != nil {
		return apierrors.NewInternalError(err)
	}
	objJS, err := runtime.Encode(a.plugin.jsonSerializer, attr.VersionedObject)
	if err != nil {
		return apierrors.NewInternalError(err)
	}
	patchedJS, err := patchObj.Apply(objJS)
	if err != nil {
		return apierrors.NewInternalError(err)
	}

	var newVersionedObject runtime.Object
	if _, ok := attr.VersionedObject.(*unstructured.Unstructured); ok {
		// Custom Resources don't have corresponding Go struct's.
		// They are represented as Unstructured.
		newVersionedObject = &unstructured.Unstructured{}
	} else {
		newVersionedObject, err = a.plugin.scheme.New(attr.GetKind())
		if err != nil {
			return apierrors.NewInternalError(err)
		}
	}
	// TODO: if we have multiple mutating webhooks, we can remember the json
	// instead of encoding and decoding for each one.
	if _, _, err := a.plugin.jsonSerializer.Decode(patchedJS, nil, newVersionedObject); err != nil {
		return apierrors.NewInternalError(err)
	}
	attr.VersionedObject = newVersionedObject
	a.plugin.scheme.Default(attr.VersionedObject)
	return nil
}
<commit_msg>Do not attempt to convert nil object during DELETE webhook admission<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package mutating delegates admission checks to dynamically configured
// mutating webhooks.
package mutating

import (
	"context"
	"fmt"
	"time"

	jsonpatch "github.com/evanphx/json-patch"
	"github.com/golang/glog"

	admissionv1beta1 "k8s.io/api/admission/v1beta1"
	"k8s.io/api/admissionregistration/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
	"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
	webhookerrors "k8s.io/apiserver/pkg/admission/plugin/webhook/errors"
	"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
	"k8s.io/apiserver/pkg/admission/plugin/webhook/request"
)

type mutatingDispatcher struct {
	cm     *config.ClientManager
	plugin *Plugin
}

func newMutatingDispatcher(p *Plugin) func(cm *config.ClientManager) generic.Dispatcher {
	return func(cm *config.ClientManager) generic.Dispatcher {
		return &mutatingDispatcher{cm, p}
	}
}

var _ generic.Dispatcher = &mutatingDispatcher{}

func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr *generic.VersionedAttributes, relevantHooks []*v1beta1.Webhook) error {
	for _, hook := range relevantHooks {
		t := time.Now()
		err := a.callAttrMutatingHook(ctx, hook, attr)
		admissionmetrics.Metrics.ObserveWebhook(time.Since(t), err != nil, attr.Attributes, "admit", hook.Name)
		if err == nil {
			continue
		}

		ignoreClientCallFailures := hook.FailurePolicy != nil && *hook.FailurePolicy == v1beta1.Ignore
		if callErr, ok := err.(*webhookerrors.ErrCallingWebhook); ok {
			if ignoreClientCallFailures {
				glog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
				utilruntime.HandleError(callErr)
				continue
			}
			glog.Warningf("Failed calling webhook, failing closed %v: %v", hook.Name, err)
		}
		return apierrors.NewInternalError(err)
	}

	// convert attr.VersionedObject to the internal version in the underlying admission.Attributes
	if attr.VersionedObject != nil {
		return a.plugin.scheme.Convert(attr.VersionedObject, attr.Attributes.GetObject(), nil)
	}
	return nil
}

// note that callAttrMutatingHook updates attr
func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *v1beta1.Webhook, attr *generic.VersionedAttributes) error {
	// Make the webhook request
	request := request.CreateAdmissionReview(attr)
	client, err := a.cm.HookClient(h)
	if err != nil {
		return &webhookerrors.ErrCallingWebhook{WebhookName: h.Name, Reason: err}
	}
	response := &admissionv1beta1.AdmissionReview{}
	if err := client.Post().Context(ctx).Body(&request).Do().Into(response); err != nil {
		return &webhookerrors.ErrCallingWebhook{WebhookName: h.Name, Reason: err}
	}

	if response.Response == nil {
		return &webhookerrors.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Webhook response was absent")}
	}

	if !response.Response.Allowed {
		return webhookerrors.ToStatusErr(h.Name, response.Response.Result)
	}

	patchJS := response.Response.Patch
	if len(patchJS) == 0 {
		return nil
	}
	patchObj, err := jsonpatch.DecodePatch(patchJS)
	if err != nil {
		return apierrors.NewInternalError(err)
	}
	if len(patchObj) == 0 {
		return nil
	}

	// if a non-empty patch was provided, and we have no object we can apply it to (e.g. a DELETE admission operation), error
	if attr.VersionedObject == nil {
		return apierrors.NewInternalError(fmt.Errorf("admission webhook %q attempted to modify the object, which is not supported for this operation", h.Name))
	}

	objJS, err := runtime.Encode(a.plugin.jsonSerializer, attr.VersionedObject)
	if err != nil {
		return apierrors.NewInternalError(err)
	}
	patchedJS, err := patchObj.Apply(objJS)
	if err != nil {
		return apierrors.NewInternalError(err)
	}

	var newVersionedObject runtime.Object
	if _, ok := attr.VersionedObject.(*unstructured.Unstructured); ok {
		// Custom Resources don't have corresponding Go struct's.
		// They are represented as Unstructured.
		newVersionedObject = &unstructured.Unstructured{}
	} else {
		newVersionedObject, err = a.plugin.scheme.New(attr.GetKind())
		if err != nil {
			return apierrors.NewInternalError(err)
		}
	}
	// TODO: if we have multiple mutating webhooks, we can remember the json
	// instead of encoding and decoding for each one.
	if _, _, err := a.plugin.jsonSerializer.Decode(patchedJS, nil, newVersionedObject); err != nil {
		return apierrors.NewInternalError(err)
	}
	attr.VersionedObject = newVersionedObject
	a.plugin.scheme.Default(attr.VersionedObject)
	return nil
}
<|endoftext|>


<commit_before><commit_msg>fix applySecurityGroupRule<commit_after><|endoftext|>


<commit_before>// Copyright 2019 CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package codec converts Go to and from CUE and validates Go values based on
// CUE constraints.
//
// CUE constraints can be used to validate Go types as well as fill out
// missing struct fields that are implied from the constraints and the values
// already defined by the struct value.
package gocodec

import (
	"sync"

	"cuelang.org/go/cue"
	"cuelang.org/go/internal"
)

// Config has no options yet, but is defined for future extensibility.
type Config struct {
}

// A Codec decodes and encodes CUE from and to Go values and validates and
// completes Go values based on CUE templates.
type Codec struct {
	runtime *cue.Runtime
	mutex   sync.RWMutex
}

// New creates a new Codec for the given instance.
//
// It is safe to use the methods of Codec concurrently as long as the given
// Runtime is not used elsewhere while using Codec. However, only the concurrent
// use of Decode, Validate, and Complete is efficient.
func New(r *cue.Runtime, c *Config) *Codec {
	return &Codec{runtime: r}
}

// ExtractType extracts a CUE value from a Go type.
//
// The type represented by x is converted as the underlying type. Specific
// values, such as map or slice elements or field values of structs are ignored.
// If x is of type reflect.Type, the type represented by x is extracted.
//
// Fields of structs can be annoted using additional constrains using the 'cue'
// field tag. The value of the tag is a CUE expression, which may contain
// references to the JSON name of other fields in a struct.
//
//     type Sum struct {
//         A int `cue:"c-b" json:"a,omitempty"`
//         B int `cue:"c-a" json:"b,omitempty"`
//         C int `cue:"a+b" json:"c,omitempty"`
//     }
//
func (c *Codec) ExtractType(x interface{}) (cue.Value, error) {
	// ExtractType cannot introduce new fields on repeated calls. We could
	// consider optimizing the lock usage based on this property.
	c.mutex.Lock()
	defer c.mutex.Unlock()

	return fromGoType(c.runtime, x)
}

// TODO: allow extracting constraints and type info separately?

// Decode converts x to a CUE value.
//
// If x is of type reflect.Value it will convert the value represented by x.
func (c *Codec) Decode(x interface{}) (cue.Value, error) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	// Depending on the type, can introduce new labels on repeated calls.
	return fromGoValue(c.runtime, x, false)
}

// Encode converts v to a Go value.
func (c *Codec) Encode(v cue.Value, x interface{}) error {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	return v.Decode(x)
}

// Validate checks whether x satisfies the constraints defined by v.
//
// The given value must be created using the same Runtime with which c was
// initialized.
func (c *Codec) Validate(v cue.Value, x interface{}) error {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	r := checkAndForkRuntime(c.runtime, v)
	w, err := fromGoValue(r, x, false)
	if err != nil {
		return err
	}
	return w.Unify(v).Err()
}

// Complete sets previously undefined values in x that can be uniquely
// determined form the constraints defined by v if validation passes, or returns
// an error, without modifying anything, otherwise.
//
// Only undefined values are modified. A value is considered undefined if it is
// pointer type and is nil or if it is a field with a zero value that has a json
// tag with the omitempty flag.
//
// The given value must be created using the same Runtime with which c was
// initialized.
//
// Complete does a JSON round trip. This means that data not preserved in such a
// round trip, such as the location name of a time.Time, is lost after a
// successful update.
func (c *Codec) Complete(v cue.Value, x interface{}) error {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	r := checkAndForkRuntime(c.runtime, v)
	w, err := fromGoValue(r, x, true)
	if err != nil {
		return err
	}

	return w.Unify(v).Decode(x)
}

func fromGoValue(r *cue.Runtime, x interface{}, allowDefault bool) (cue.Value, error) {
	v := internal.FromGoValue(r, x, allowDefault).(cue.Value)
	if err := v.Err(); err != nil {
		return v, err
	}
	return v, nil
}

func fromGoType(r *cue.Runtime, x interface{}) (cue.Value, error) {
	v := internal.FromGoType(r, x).(cue.Value)
	if err := v.Err(); err != nil {
		return v, err
	}
	return v, nil
}

func checkAndForkRuntime(r *cue.Runtime, v cue.Value) *cue.Runtime {
	return internal.CheckAndForkRuntime(r, v).(*cue.Runtime)
}
<commit_msg>encoding/gocodec: Validate helper function<commit_after>// Copyright 2019 CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package codec converts Go to and from CUE and validates Go values based on
// CUE constraints.
//
// CUE constraints can be used to validate Go types as well as fill out
// missing struct fields that are implied from the constraints and the values
// already defined by the struct value.
package gocodec

import (
	"sync"

	"cuelang.org/go/cue"
	"cuelang.org/go/internal"
)

// Config has no options yet, but is defined for future extensibility.
type Config struct {
}

// A Codec decodes and encodes CUE from and to Go values and validates and
// completes Go values based on CUE templates.
type Codec struct {
	runtime *cue.Runtime
	mutex   sync.RWMutex
}

// New creates a new Codec for the given instance.
//
// It is safe to use the methods of Codec concurrently as long as the given
// Runtime is not used elsewhere while using Codec. However, only the concurrent
// use of Decode, Validate, and Complete is efficient.
func New(r *cue.Runtime, c *Config) *Codec {
	return &Codec{runtime: r}
}

// ExtractType extracts a CUE value from a Go type.
//
// The type represented by x is converted as the underlying type. Specific
// values, such as map or slice elements or field values of structs are ignored.
// If x is of type reflect.Type, the type represented by x is extracted.
//
// Fields of structs can be annoted using additional constrains using the 'cue'
// field tag. The value of the tag is a CUE expression, which may contain
// references to the JSON name of other fields in a struct.
//
//     type Sum struct {
//         A int `cue:"c-b" json:"a,omitempty"`
//         B int `cue:"c-a" json:"b,omitempty"`
//         C int `cue:"a+b" json:"c,omitempty"`
//     }
//
func (c *Codec) ExtractType(x interface{}) (cue.Value, error) {
	// ExtractType cannot introduce new fields on repeated calls. We could
	// consider optimizing the lock usage based on this property.
	c.mutex.Lock()
	defer c.mutex.Unlock()

	return fromGoType(c.runtime, x)
}

// TODO: allow extracting constraints and type info separately?

// Decode converts x to a CUE value.
//
// If x is of type reflect.Value it will convert the value represented by x.
func (c *Codec) Decode(x interface{}) (cue.Value, error) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	// Depending on the type, can introduce new labels on repeated calls.
	return fromGoValue(c.runtime, x, false)
}

// Encode converts v to a Go value.
func (c *Codec) Encode(v cue.Value, x interface{}) error {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	return v.Decode(x)
}

var defaultCodec = New(&cue.Runtime{}, nil)

// Validate calls Validate on a default Codec for the type of x.
func Validate(x interface{}) error {
	c := defaultCodec
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	r := defaultCodec.runtime
	v, err := fromGoType(r, x)
	if err != nil {
		return err
	}
	w, err := fromGoValue(r, x, false)
	if err != nil {
		return err
	}
	v = v.Unify(w)
	if err := v.Validate(); err != nil {
		return err
	}
	return nil
}

// Validate checks whether x satisfies the constraints defined by v.
//
// The given value must be created using the same Runtime with which c was
// initialized.
func (c *Codec) Validate(v cue.Value, x interface{}) error {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	r := checkAndForkRuntime(c.runtime, v)
	w, err := fromGoValue(r, x, false)
	if err != nil {
		return err
	}
	return w.Unify(v).Err()
}

// Complete sets previously undefined values in x that can be uniquely
// determined form the constraints defined by v if validation passes, or returns
// an error, without modifying anything, otherwise.
//
// Only undefined values are modified. A value is considered undefined if it is
// pointer type and is nil or if it is a field with a zero value that has a json
// tag with the omitempty flag.
//
// The given value must be created using the same Runtime with which c was
// initialized.
//
// Complete does a JSON round trip. This means that data not preserved in such a
// round trip, such as the location name of a time.Time, is lost after a
// successful update.
func (c *Codec) Complete(v cue.Value, x interface{}) error {
	c.mutex.RLock()
	defer c.mutex.RUnlock()

	r := checkAndForkRuntime(c.runtime, v)
	w, err := fromGoValue(r, x, true)
	if err != nil {
		return err
	}

	return w.Unify(v).Decode(x)
}

func fromGoValue(r *cue.Runtime, x interface{}, allowDefault bool) (cue.Value, error) {
	v := internal.FromGoValue(r, x, allowDefault).(cue.Value)
	if err := v.Err(); err != nil {
		return v, err
	}
	return v, nil
}

func fromGoType(r *cue.Runtime, x interface{}) (cue.Value, error) {
	v := internal.FromGoType(r, x).(cue.Value)
	if err := v.Err(); err != nil {
		return v, err
	}
	return v, nil
}

func checkAndForkRuntime(r *cue.Runtime, v cue.Value) *cue.Runtime {
	return internal.CheckAndForkRuntime(r, v).(*cue.Runtime)
}
<|endoftext|>


<commit_before>package kallax

import (
	"bytes"
	"database/sql"
	"database/sql/driver"
	"encoding/hex"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/oklog/ulid"
	uuid "github.com/satori/go.uuid"
)

// Model contains all the basic fields that make something a model, that is,
// the ID and some internal data used by kallax.
// To make a struct a model, it only needs to have Model embedded.
//
//	type MyModel struct {
//		kallax.Model
//		Foo string
//	}
//
// Custom name for the table can be specified using the struct tag `table` when
// embedding the Model.
//
//	type MyModel struct {
//		kallax.Model `table:"custom_name"`
//	}
//
// Otherwise, the default name of the table is the name of the model converted
// to lower snake case. E.g: MyModel => my_model.
// No pluralization is done right now, but might be done in the future, so
// please, set the name of the tables yourself.
type Model struct {
	virtualColumns map[string]Identifier
	persisted      bool
	writable       bool
}

// NewModel creates a new Model that is writable and not persisted.
func NewModel() Model {
	return Model{
		persisted:      false,
		writable:       true,
		virtualColumns: make(map[string]Identifier),
	}
}

// IsPersisted returns whether the Model has already been persisted to the
// database or not.
func (m *Model) IsPersisted() bool {
	return m.persisted
}

func (m *Model) setPersisted() {
	m.persisted = true
}

// IsWritable returns whether this Model can be saved into the database.
// For example, a model with partially retrieved data is not writable, so
// it is not saved by accident and the data is corrupted. For example, if
// you select only 2 columns out of all the ones the table has, it will not
// be writable.
func (m *Model) IsWritable() bool {
	return m.writable
}

func (m *Model) setWritable(w bool) {
	m.writable = w
}

// ClearVirtualColumns clears all the previous virtual columns.
// This method is only intended for internal use. It is only exposed for
// technical reasons.
func (m *Model) ClearVirtualColumns() {
	m.virtualColumns = make(map[string]Identifier)
}

// AddVirtualColumn adds a new virtual column with the given name and value.
// This method is only intended for internal use. It is only exposed for
// technical reasons.
func (m *Model) AddVirtualColumn(name string, v Identifier) {
	if m.virtualColumns == nil {
		m.ClearVirtualColumns()
	}
	m.virtualColumns[name] = v
}

// VirtualColumn returns the value of the virtual column with the given column name.
// This method is only intended for internal use. It is only exposed for
// technical reasons.
func (m *Model) VirtualColumn(name string) Identifier {
	if m.virtualColumns == nil {
		m.ClearVirtualColumns()
	}
	return m.virtualColumns[name]
}

// Identifier is a type used to identify a model.
type Identifier interface {
	sql.Scanner
	driver.Valuer
	// Equals reports whether the identifier and the given one are equal.
	Equals(Identifier) bool
	// IsEmpty returns whether the ID is empty or not.
	IsEmpty() bool
	// Raw returns the internal value of the identifier.
	Raw() interface{}
}

// Identifiable must be implemented by those values that can be identified by an ID.
type Identifiable interface {
	// GetID returns the ID.
	GetID() Identifier
}

// Persistable must be implemented by those values that can be persisted.
type Persistable interface {
	// IsPersisted returns whether this Model is new in the store or not.
	IsPersisted() bool
	setPersisted()
}

// Writable must be implemented by those values that defines internally
// if they can be sent back to the database to be stored with its changes.
type Writable interface {
	// IsWritable returns whether this Model can be saved into the database.
	IsWritable() bool
	setWritable(bool)
}

// ColumnAddresser provides the pointer addresses of columns.
type ColumnAddresser interface {
	// ColumnAddress returns the pointer to the column value of the given
	// column name, or an error if it does not exist in the model.
	ColumnAddress(string) (interface{}, error)
}

// Relationable can perform operations related to relationships of a record.
type Relationable interface {
	// NewRelationshipRecord returns a new Record for the relationship at the
	// given field.
	NewRelationshipRecord(string) (Record, error)
	// SetRelationship sets the relationship value at the given field.
	SetRelationship(string, interface{}) error
}

// Valuer provides the values for columns.
type Valuer interface {
	// Value returns the value of the given column, or an error if it does not
	// exist in the model.
	Value(string) (interface{}, error)
}

// VirtualColumnContainer contains a collection of virtual columns and
// manages them.
type VirtualColumnContainer interface {
	// ClearVirtualColumns removes all virtual columns.
	ClearVirtualColumns()
	// AddVirtualColumn adds a new virtual column with the given name and value
	AddVirtualColumn(string, Identifier)
	// VirtualColumn returns the virtual column with the given column name.
	VirtualColumn(string) Identifier
}

// RecordValues returns the values of a record at the given columns in the same
// order as the columns.
func RecordValues(record Valuer, columns ...string) ([]interface{}, error) {
	var values = make([]interface{}, len(columns))
	for i, col := range columns {
		v, err := record.Value(col)
		if err != nil {
			return nil, err
		}
		values[i] = v
	}
	return values, nil
}

// Record is something that can be stored as a row in the database.
type Record interface {
	Identifiable
	Persistable
	Writable
	Relationable
	ColumnAddresser
	Valuer
	VirtualColumnContainer
}

var randPool = &sync.Pool{
	New: func() interface{} {
		return rand.NewSource(time.Now().UnixNano())
	},
}

// ULID is an ID type provided by kallax that is a lexically sortable UUID.
// The internal representation is an ULID (https://github.com/oklog/ulid).
// It already implements sql.Scanner and driver.Valuer, so it's perfectly
// safe for database usage.
type ULID uuid.UUID

// NewULID returns a new ULID, which is a lexically sortable UUID.
func NewULID() ULID {
	entropy := randPool.Get().(rand.Source)
	id := ULID(ulid.MustNew(ulid.Timestamp(time.Now()), rand.New(entropy)))
	randPool.Put(entropy)

	return id
}

// Scan implements the Scanner interface.
func (id *ULID) Scan(src interface{}) error {
	switch src := src.(type) {
	case []byte:
		if len(src) != 16 {
			return id.UnmarshalText(src)
		}

		var ulid ulid.ULID
		if err := ulid.UnmarshalBinary(src); err != nil {
			return err
		}
		*id = ULID(ulid)
		return nil
	case string:
		return id.Scan([]byte(src))
	default:
		return fmt.Errorf("kallax: cannot scan %T into ULID", src)
	}
}

var (
	urnPrefix  = []byte("urn:uuid:")
	byteGroups = []int{8, 4, 4, 4, 12}
)

// UnmarshalText implements the encoding.TextUnmarshaler interface.
// Following formats are supported:
// "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}",
// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8"
// Implements the exact same code as the UUID UnmarshalText removing the
// version check.
func (u *ULID) UnmarshalText(text []byte) (err error) {
	if len(text) < 32 {
		err = fmt.Errorf("uuid: UUID string too short: %s", text)
		return
	}

	t := text[:]
	braced := false

	if bytes.Equal(t[:9], urnPrefix) {
		t = t[9:]
	} else if t[0] == '{' {
		braced = true
		t = t[1:]
	}

	b := u[:]

	for i, byteGroup := range byteGroups {
		if i > 0 && t[0] == '-' {
			t = t[1:]
		} else if i > 0 && t[0] != '-' {
			err = fmt.Errorf("kallax: invalid ulid string format")
			return
		}

		if len(t) < byteGroup {
			err = fmt.Errorf("kallax: ulid string too short: %s", text)
			return
		}

		if i == 4 && len(t) > byteGroup &&
			((braced && t[byteGroup] != '}') || len(t[byteGroup:]) > 1 || !braced) {
			err = fmt.Errorf("kallax: ulid string too long: %s", t)
			return
		}

		_, err = hex.Decode(b[:byteGroup/2], t[:byteGroup])

		if err != nil {
			return
		}

		t = t[byteGroup:]
		b = b[byteGroup/2:]
	}

	return
}

func (id ULID) MarshalText() ([]byte, error) {
	return []byte(id.String()), nil
}

// Value implements the Valuer interface.
func (id ULID) Value() (driver.Value, error) {
	return uuid.UUID(id).Value()
}

// IsEmpty returns whether the ID is empty or not. An empty ID means it has not
// been set yet.
func (id ULID) IsEmpty() bool {
	return uuid.Equal(uuid.UUID(id), uuid.Nil)
}

// String returns the string representation of the ID.
func (id ULID) String() string {
	return uuid.UUID(id).String()
}

// Equals reports whether the ID and the given one are equals.
func (id ULID) Equals(other Identifier) bool {
	v, ok := other.(*ULID)
	if !ok {
		return false
	}

	return uuid.Equal(uuid.UUID(id), uuid.UUID(*v))
}

// Raw returns the underlying raw value.
func (id ULID) Raw() interface{} {
	return id
}

// NumericID is a wrapper for int64 that implements the Identifier interface.
// You don't need to actually use this as a type in your model. They will be
// automatically converted to and from in the generated code.
type NumericID int64

// Scan implements the Scanner interface.
func (id *NumericID) Scan(src interface{}) error {
	switch src := src.(type) {
	case int64:
		*(*int64)(id) = src
	default:
		return fmt.Errorf("kallax: cannot scan value of type %T into a numeric ID", src)
	}

	return nil
}

// Value implements the Valuer interface.
func (id NumericID) Value() (driver.Value, error) {
	return int64(id), nil
}

// IsEmpty returns whether the ID is empty or not. An empty ID means it has not
// been set yet.
func (id NumericID) IsEmpty() bool {
	return int64(id) == 0
}

// String returns the string representation of the ID.
func (id NumericID) String() string {
	return fmt.Sprint(int64(id))
}

// Equals reports whether the ID and the given one are equals.
func (id NumericID) Equals(other Identifier) bool {
	v, ok := other.(*NumericID)
	if !ok {
		return false
	}

	return int64(id) == int64(*v)
}

// Raw returns the underlying raw value.
func (id NumericID) Raw() interface{} {
	return id
}

// UUID is a wrapper type for uuid.UUID that implements the Identifier
// interface.
// You don't need to actually use this as a type in your model. They will be
// automatically converted to and from in the generated code.
type UUID uuid.UUID

// Scan implements the Scanner interface.
func (id *UUID) Scan(src interface{}) error {
	return (*uuid.UUID)(id).Scan(src)
}

// Value implements the Valuer interface.
func (id UUID) Value() (driver.Value, error) {
	return uuid.UUID(id).Value()
}

// IsEmpty returns whether the ID is empty or not. An empty ID means it has not
// been set yet.
func (id UUID) IsEmpty() bool {
	return uuid.Equal(uuid.UUID(id), uuid.Nil)
}

// String returns the string representation of the ID.
func (id UUID) String() string {
	return uuid.UUID(id).String()
}

// Equals reports whether the ID and the given one are equals.
func (id UUID) Equals(other Identifier) bool {
	v, ok := other.(*UUID)
	if !ok {
		return false
	}

	return uuid.Equal(uuid.UUID(id), uuid.UUID(*v))
}

// Raw returns the underlying raw value.
func (id UUID) Raw() interface{} {
	return id
}

type virtualColumn struct {
	r   Record
	col string
	id  Identifier
}

// VirtualColumn returns a sql.Scanner that will scan the given column as a
// virtual column in the given record.
func VirtualColumn(col string, r Record, id Identifier) sql.Scanner {
	return &virtualColumn{r, col, id}
}

// Scan implements the scanner interface.
func (c *virtualColumn) Scan(src interface{}) error {
	if err := c.id.Scan(src); err != nil {
		return err
	}

	c.r.AddVirtualColumn(c.col, c.id)
	return nil
}
<commit_msg>add NewULIDFromText function<commit_after>package kallax

import (
	"bytes"
	"database/sql"
	"database/sql/driver"
	"encoding/hex"
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/oklog/ulid"
	uuid "github.com/satori/go.uuid"
)

// Model contains all the basic fields that make something a model, that is,
// the ID and some internal data used by kallax.
// To make a struct a model, it only needs to have Model embedded.
//
//	type MyModel struct {
//		kallax.Model
//		Foo string
//	}
//
// Custom name for the table can be specified using the struct tag `table` when
// embedding the Model.
//
//	type MyModel struct {
//		kallax.Model `table:"custom_name"`
//	}
//
// Otherwise, the default name of the table is the name of the model converted
// to lower snake case. E.g: MyModel => my_model.
// No pluralization is done right now, but might be done in the future, so
// please, set the name of the tables yourself.
type Model struct {
	virtualColumns map[string]Identifier
	persisted      bool
	writable       bool
}

// NewModel creates a new Model that is writable and not persisted.
func NewModel() Model {
	return Model{
		persisted:      false,
		writable:       true,
		virtualColumns: make(map[string]Identifier),
	}
}

// IsPersisted returns whether the Model has already been persisted to the
// database or not.
func (m *Model) IsPersisted() bool {
	return m.persisted
}

func (m *Model) setPersisted() {
	m.persisted = true
}

// IsWritable returns whether this Model can be saved into the database.
// For example, a model with partially retrieved data is not writable, so
// it is not saved by accident and the data is corrupted. For example, if
// you select only 2 columns out of all the ones the table has, it will not
// be writable.
func (m *Model) IsWritable() bool {
	return m.writable
}

func (m *Model) setWritable(w bool) {
	m.writable = w
}

// ClearVirtualColumns clears all the previous virtual columns.
// This method is only intended for internal use. It is only exposed for
// technical reasons.
func (m *Model) ClearVirtualColumns() {
	m.virtualColumns = make(map[string]Identifier)
}

// AddVirtualColumn adds a new virtual column with the given name and value.
// This method is only intended for internal use. It is only exposed for
// technical reasons.
func (m *Model) AddVirtualColumn(name string, v Identifier) {
	if m.virtualColumns == nil {
		m.ClearVirtualColumns()
	}
	m.virtualColumns[name] = v
}

// VirtualColumn returns the value of the virtual column with the given column name.
// This method is only intended for internal use. It is only exposed for
// technical reasons.
func (m *Model) VirtualColumn(name string) Identifier {
	if m.virtualColumns == nil {
		m.ClearVirtualColumns()
	}
	return m.virtualColumns[name]
}

// Identifier is a type used to identify a model.
type Identifier interface {
	sql.Scanner
	driver.Valuer
	// Equals reports whether the identifier and the given one are equal.
	Equals(Identifier) bool
	// IsEmpty returns whether the ID is empty or not.
	IsEmpty() bool
	// Raw returns the internal value of the identifier.
	Raw() interface{}
}

// Identifiable must be implemented by those values that can be identified by an ID.
type Identifiable interface {
	// GetID returns the ID.
	GetID() Identifier
}

// Persistable must be implemented by those values that can be persisted.
type Persistable interface {
	// IsPersisted returns whether this Model is new in the store or not.
	IsPersisted() bool
	setPersisted()
}

// Writable must be implemented by those values that defines internally
// if they can be sent back to the database to be stored with its changes.
type Writable interface {
	// IsWritable returns whether this Model can be saved into the database.
	IsWritable() bool
	setWritable(bool)
}

// ColumnAddresser provides the pointer addresses of columns.
type ColumnAddresser interface {
	// ColumnAddress returns the pointer to the column value of the given
	// column name, or an error if it does not exist in the model.
	ColumnAddress(string) (interface{}, error)
}

// Relationable can perform operations related to relationships of a record.
type Relationable interface {
	// NewRelationshipRecord returns a new Record for the relationship at the
	// given field.
	NewRelationshipRecord(string) (Record, error)
	// SetRelationship sets the relationship value at the given field.
	SetRelationship(string, interface{}) error
}

// Valuer provides the values for columns.
type Valuer interface {
	// Value returns the value of the given column, or an error if it does not
	// exist in the model.
	Value(string) (interface{}, error)
}

// VirtualColumnContainer contains a collection of virtual columns and
// manages them.
type VirtualColumnContainer interface {
	// ClearVirtualColumns removes all virtual columns.
	ClearVirtualColumns()
	// AddVirtualColumn adds a new virtual column with the given name and value
	AddVirtualColumn(string, Identifier)
	// VirtualColumn returns the virtual column with the given column name.
	VirtualColumn(string) Identifier
}

// RecordValues returns the values of a record at the given columns in the same
// order as the columns.
func RecordValues(record Valuer, columns ...string) ([]interface{}, error) {
	var values = make([]interface{}, len(columns))
	for i, col := range columns {
		v, err := record.Value(col)
		if err != nil {
			return nil, err
		}
		values[i] = v
	}
	return values, nil
}

// Record is something that can be stored as a row in the database.
type Record interface {
	Identifiable
	Persistable
	Writable
	Relationable
	ColumnAddresser
	Valuer
	VirtualColumnContainer
}

var randPool = &sync.Pool{
	New: func() interface{} {
		return rand.NewSource(time.Now().UnixNano())
	},
}

// ULID is an ID type provided by kallax that is a lexically sortable UUID.
// The internal representation is an ULID (https://github.com/oklog/ulid).
// It already implements sql.Scanner and driver.Valuer, so it's perfectly
// safe for database usage.
type ULID uuid.UUID

// NewULID returns a new ULID, which is a lexically sortable UUID.
func NewULID() ULID {
	entropy := randPool.Get().(rand.Source)
	id := ULID(ulid.MustNew(ulid.Timestamp(time.Now()), rand.New(entropy)))
	randPool.Put(entropy)

	return id
}

// NewULIDFromText creates a new ULID from its string representation. Will
// return an error if the text is not a valid ULID.
func NewULIDFromText(text string) (ULID, error) {
	var id ULID
	err := id.UnmarshalText([]byte(text))
	return id, err
}

// Scan implements the Scanner interface.
func (id *ULID) Scan(src interface{}) error {
	switch src := src.(type) {
	case []byte:
		if len(src) != 16 {
			return id.UnmarshalText(src)
		}

		var ulid ulid.ULID
		if err := ulid.UnmarshalBinary(src); err != nil {
			return err
		}
		*id = ULID(ulid)
		return nil
	case string:
		return id.Scan([]byte(src))
	default:
		return fmt.Errorf("kallax: cannot scan %T into ULID", src)
	}
}

var (
	urnPrefix  = []byte("urn:uuid:")
	byteGroups = []int{8, 4, 4, 4, 12}
)

// UnmarshalText implements the encoding.TextUnmarshaler interface.
// Following formats are supported:
// "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}",
// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8"
// Implements the exact same code as the UUID UnmarshalText removing the
// version check.
func (u *ULID) UnmarshalText(text []byte) (err error) {
	if len(text) < 32 {
		err = fmt.Errorf("uuid: UUID string too short: %s", text)
		return
	}

	t := text[:]
	braced := false

	if bytes.Equal(t[:9], urnPrefix) {
		t = t[9:]
	} else if t[0] == '{' {
		braced = true
		t = t[1:]
	}

	b := u[:]

	for i, byteGroup := range byteGroups {
		if i > 0 && t[0] == '-' {
			t = t[1:]
		} else if i > 0 && t[0] != '-' {
			err = fmt.Errorf("kallax: invalid ulid string format")
			return
		}

		if len(t) < byteGroup {
			err = fmt.Errorf("kallax: ulid string too short: %s", text)
			return
		}

		if i == 4 && len(t) > byteGroup &&
			((braced && t[byteGroup] != '}') || len(t[byteGroup:]) > 1 || !braced) {
			err = fmt.Errorf("kallax: ulid string too long: %s", t)
			return
		}

		_, err = hex.Decode(b[:byteGroup/2], t[:byteGroup])

		if err != nil {
			return
		}

		t = t[byteGroup:]
		b = b[byteGroup/2:]
	}

	return
}

func (id ULID) MarshalText() ([]byte, error) {
	return []byte(id.String()), nil
}

// Value implements the Valuer interface.
func (id ULID) Value() (driver.Value, error) {
	return uuid.UUID(id).Value()
}

// IsEmpty returns whether the ID is empty or not. An empty ID means it has not
// been set yet.
func (id ULID) IsEmpty() bool {
	return uuid.Equal(uuid.UUID(id), uuid.Nil)
}

// String returns the string representation of the ID.
func (id ULID) String() string {
	return uuid.UUID(id).String()
}

// Equals reports whether the ID and the given one are equals.
func (id ULID) Equals(other Identifier) bool {
	v, ok := other.(*ULID)
	if !ok {
		return false
	}

	return uuid.Equal(uuid.UUID(id), uuid.UUID(*v))
}

// Raw returns the underlying raw value.
func (id ULID) Raw() interface{} {
	return id
}

// NumericID is a wrapper for int64 that implements the Identifier interface.
// You don't need to actually use this as a type in your model. They will be
// automatically converted to and from in the generated code.
type NumericID int64

// Scan implements the Scanner interface.
func (id *NumericID) Scan(src interface{}) error {
	switch src := src.(type) {
	case int64:
		*(*int64)(id) = src
	default:
		return fmt.Errorf("kallax: cannot scan value of type %T into a numeric ID", src)
	}

	return nil
}

// Value implements the Valuer interface.
func (id NumericID) Value() (driver.Value, error) {
	return int64(id), nil
}

// IsEmpty returns whether the ID is empty or not. An empty ID means it has not
// been set yet.
func (id NumericID) IsEmpty() bool {
	return int64(id) == 0
}

// String returns the string representation of the ID.
func (id NumericID) String() string {
	return fmt.Sprint(int64(id))
}

// Equals reports whether the ID and the given one are equals.
func (id NumericID) Equals(other Identifier) bool {
	v, ok := other.(*NumericID)
	if !ok {
		return false
	}

	return int64(id) == int64(*v)
}

// Raw returns the underlying raw value.
func (id NumericID) Raw() interface{} {
	return id
}

// UUID is a wrapper type for uuid.UUID that implements the Identifier
// interface.
// You don't need to actually use this as a type in your model. They will be
// automatically converted to and from in the generated code.
type UUID uuid.UUID

// Scan implements the Scanner interface.
func (id *UUID) Scan(src interface{}) error {
	return (*uuid.UUID)(id).Scan(src)
}

// Value implements the Valuer interface.
func (id UUID) Value() (driver.Value, error) {
	return uuid.UUID(id).Value()
}

// IsEmpty returns whether the ID is empty or not. An empty ID means it has not
// been set yet.
func (id UUID) IsEmpty() bool {
	return uuid.Equal(uuid.UUID(id), uuid.Nil)
}

// String returns the string representation of the ID.
func (id UUID) String() string {
	return uuid.UUID(id).String()
}

// Equals reports whether the ID and the given one are equals.
func (id UUID) Equals(other Identifier) bool {
	v, ok := other.(*UUID)
	if !ok {
		return false
	}

	return uuid.Equal(uuid.UUID(id), uuid.UUID(*v))
}

// Raw returns the underlying raw value.
func (id UUID) Raw() interface{} {
	return id
}

type virtualColumn struct {
	r   Record
	col string
	id  Identifier
}

// VirtualColumn returns a sql.Scanner that will scan the given column as a
// virtual column in the given record.
func VirtualColumn(col string, r Record, id Identifier) sql.Scanner {
	return &virtualColumn{r, col, id}
}

// Scan implements the scanner interface.
func (c *virtualColumn) Scan(src interface{}) error {
	if err := c.id.Scan(src); err != nil {
		return err
	}

	c.r.AddVirtualColumn(c.col, c.id)
	return nil
}
<|endoftext|>


<commit_before>package mysql

import (
	"database/sql"
	"fmt"
	"math"

	_ "github.com/go-sql-driver/mysql"
	"gorm.io/gorm"
	"gorm.io/gorm/callbacks"
	"gorm.io/gorm/clause"
	"gorm.io/gorm/logger"
	"gorm.io/gorm/migrator"
	"gorm.io/gorm/schema"
)

type Dialector struct {
	DSN string
}

func Open(dsn string) gorm.Dialector {
	return &Dialector{DSN: dsn}
}

func (dialector Dialector) Name() string {
	return "mysql"
}

func (dialector Dialector) Initialize(db *gorm.DB) (err error) {
	// register callbacks
	callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{})
	db.ConnPool, err = sql.Open("mysql", dialector.DSN)

	for k, v := range dialector.ClauseBuilders() {
		db.ClauseBuilders[k] = v
	}
	return
}

func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder {
	return map[string]clause.ClauseBuilder{
		"ON CONFLICT": func(c clause.Clause, builder clause.Builder) {
			if onConflict, ok := c.Expression.(clause.OnConflict); ok {
				builder.WriteString("ON DUPLICATE KEY UPDATE ")
				if len(onConflict.DoUpdates) == 0 {
					if s := builder.(*gorm.Statement).Schema; s != nil {
						var column clause.Column
						onConflict.DoNothing = false

						if s.PrioritizedPrimaryField != nil {
							column = clause.Column{Name: s.PrioritizedPrimaryField.DBName}
						} else {
							for _, field := range s.FieldsByDBName {
								column = clause.Column{Name: field.DBName}
								break
							}
						}
						onConflict.DoUpdates = []clause.Assignment{{Column: column, Value: column}}
					}
				}

				onConflict.DoUpdates.Build(builder)
			} else {
				c.Build(builder)
			}
		},
		"VALUES": func(c clause.Clause, builder clause.Builder) {
			if values, ok := c.Expression.(clause.Values); ok && len(values.Columns) == 0 {
				builder.WriteString("VALUES()")
				return
			}
			c.Build(builder)
		},
	}
}

func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator {
	return Migrator{migrator.Migrator{Config: migrator.Config{
		DB:        db,
		Dialector: dialector,
	}}}
}

func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) {
	writer.WriteByte('?')
}

func (dialector Dialector) QuoteTo(writer clause.Writer, str string) {
	writer.WriteByte('`')
	writer.WriteString(str)
	writer.WriteByte('`')
}

func (dialector Dialector) Explain(sql string, vars ...interface{}) string {
	return logger.ExplainSQL(sql, nil, `"`, vars...)
}

func (dialector Dialector) DataTypeOf(field *schema.Field) string {
	switch field.DataType {
	case schema.Bool:
		return "boolean"
	case schema.Int, schema.Uint:
		sqlType := "int"
		switch {
		case field.Size <= 8:
			sqlType = "tinyint"
		case field.Size <= 16:
			sqlType = "smallint"
		case field.Size <= 32:
			sqlType = "int"
		default:
			sqlType = "bigint"
		}

		if field.DataType == schema.Uint {
			sqlType += " unsigned"
		}

		if field.AutoIncrement || field == field.Schema.PrioritizedPrimaryField {
			sqlType += " AUTO_INCREMENT"
		}
		return sqlType
	case schema.Float:
		if field.Size <= 32 {
			return "float"
		}
		return "double"
	case schema.String:
		size := field.Size
		if size == 0 {
			if field.PrimaryKey || field.HasDefaultValue {
				size = 256
			}
		}

		if size >= 65536 && size <= int(math.Pow(2, 24)) {
			return "mediumtext"
		} else if size > int(math.Pow(2, 24)) || size <= 0 {
			return "longtext"
		}
		return fmt.Sprintf("varchar(%d)", size)
	case schema.Time:
		precision := ""
		if field.Precision == 0 {
			field.Precision = 3
		}

		if field.Precision > 0 {
			precision = fmt.Sprintf("(%d)", field.Precision)
		}

		if field.NotNull || field.PrimaryKey {
			return "datetime" + precision
		}
		return "datetime" + precision + " NULL"
	case schema.Bytes:
		if field.Size > 0 && field.Size < 65536 {
			return fmt.Sprintf("varbinary(%d)", field.Size)
		}

		if field.Size >= 65536 && field.Size <= int(math.Pow(2, 24)) {
			return "mediumblob"
		}

		return "longblob"
	}

	return ""
}
<commit_msg>Support batch upsert<commit_after>package mysql

import (
	"database/sql"
	"fmt"
	"math"

	_ "github.com/go-sql-driver/mysql"
	"gorm.io/gorm"
	"gorm.io/gorm/callbacks"
	"gorm.io/gorm/clause"
	"gorm.io/gorm/logger"
	"gorm.io/gorm/migrator"
	"gorm.io/gorm/schema"
)

type Dialector struct {
	DSN string
}

func Open(dsn string) gorm.Dialector {
	return &Dialector{DSN: dsn}
}

func (dialector Dialector) Name() string {
	return "mysql"
}

func (dialector Dialector) Initialize(db *gorm.DB) (err error) {
	// register callbacks
	callbacks.RegisterDefaultCallbacks(db, &callbacks.Config{})
	db.ConnPool, err = sql.Open("mysql", dialector.DSN)

	for k, v := range dialector.ClauseBuilders() {
		db.ClauseBuilders[k] = v
	}
	return
}

func (dialector Dialector) ClauseBuilders() map[string]clause.ClauseBuilder {
	return map[string]clause.ClauseBuilder{
		"ON CONFLICT": func(c clause.Clause, builder clause.Builder) {
			if onConflict, ok := c.Expression.(clause.OnConflict); ok {
				builder.WriteString("ON DUPLICATE KEY UPDATE ")
				if len(onConflict.DoUpdates) == 0 {
					if s := builder.(*gorm.Statement).Schema; s != nil {
						var column clause.Column
						onConflict.DoNothing = false

						if s.PrioritizedPrimaryField != nil {
							column = clause.Column{Name: s.PrioritizedPrimaryField.DBName}
						} else {
							for _, field := range s.FieldsByDBName {
								column = clause.Column{Name: field.DBName}
								break
							}
						}
						onConflict.DoUpdates = []clause.Assignment{{Column: column, Value: column}}
					}
				}

				for idx, assignment := range onConflict.DoUpdates {
					if idx > 0 {
						builder.WriteByte(',')
					}

					builder.WriteQuoted(assignment.Column)
					builder.WriteByte('=')
					if column, ok := assignment.Value.(clause.Column); ok && column.Table == "excluded" {
						column.Table = ""
						builder.WriteString("VALUES(")
						builder.WriteQuoted(column)
						builder.WriteByte(')')
					} else {
						builder.AddVar(builder, assignment.Value)
					}
				}
			} else {
				c.Build(builder)
			}
		},
		"VALUES": func(c clause.Clause, builder clause.Builder) {
			if values, ok := c.Expression.(clause.Values); ok && len(values.Columns) == 0 {
				builder.WriteString("VALUES()")
				return
			}
			c.Build(builder)
		},
	}
}

func (dialector Dialector) Migrator(db *gorm.DB) gorm.Migrator {
	return Migrator{migrator.Migrator{Config: migrator.Config{
		DB:        db,
		Dialector: dialector,
	}}}
}

func (dialector Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{}) {
	writer.WriteByte('?')
}

func (dialector Dialector) QuoteTo(writer clause.Writer, str string) {
	writer.WriteByte('`')
	writer.WriteString(str)
	writer.WriteByte('`')
}

func (dialector Dialector) Explain(sql string, vars ...interface{}) string {
	return logger.ExplainSQL(sql, nil, `"`, vars...)
}

func (dialector Dialector) DataTypeOf(field *schema.Field) string {
	switch field.DataType {
	case schema.Bool:
		return "boolean"
	case schema.Int, schema.Uint:
		sqlType := "int"
		switch {
		case field.Size <= 8:
			sqlType = "tinyint"
		case field.Size <= 16:
			sqlType = "smallint"
		case field.Size <= 32:
			sqlType = "int"
		default:
			sqlType = "bigint"
		}

		if field.DataType == schema.Uint {
			sqlType += " unsigned"
		}

		if field.AutoIncrement || field == field.Schema.PrioritizedPrimaryField {
			sqlType += " AUTO_INCREMENT"
		}
		return sqlType
	case schema.Float:
		if field.Size <= 32 {
			return "float"
		}
		return "double"
	case schema.String:
		size := field.Size
		if size == 0 {
			if field.PrimaryKey || field.HasDefaultValue {
				size = 256
			}
		}

		if size >= 65536 && size <= int(math.Pow(2, 24)) {
			return "mediumtext"
		} else if size > int(math.Pow(2, 24)) || size <= 0 {
			return "longtext"
		}
		return fmt.Sprintf("varchar(%d)", size)
	case schema.Time:
		precision := ""
		if field.Precision == 0 {
			field.Precision = 3
		}

		if field.Precision > 0 {
			precision = fmt.Sprintf("(%d)", field.Precision)
		}

		if field.NotNull || field.PrimaryKey {
			return "datetime" + precision
		}
		return "datetime" + precision + " NULL"
	case schema.Bytes:
		if field.Size > 0 && field.Size < 65536 {
			return fmt.Sprintf("varbinary(%d)", field.Size)
		}

		if field.Size >= 65536 && field.Size <= int(math.Pow(2, 24)) {
			return "mediumblob"
		}

		return "longblob"
	}

	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	//"github.com/crunchydata/postgres-operator/util"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"os"
)

var Clientset *kubernetes.Clientset
var COMMAND, PODNAME, Namespace string

const backrestCommand = "pgbackrest"

//pgbackrest --stanza=db backup
const backrestStanza = "--stanza=db"
const backrestBackupCommand = "backup"
const backrestInfoCommand = "info"
const containername = "database"

func main() {
	log.Info("pgo-backrest starts")
	kubeconfig := flag.String("kubeconfig", "", "Path to a kube config. Only required if out-of-cluster.")
	flag.Parse()

	debugFlag := os.Getenv("CRUNCHY_DEBUG")
	if debugFlag == "true" {
		log.SetLevel(log.DebugLevel)
		log.Debug("debug flag set to true")
	} else {
		log.Info("debug flag set to false")
	}

	Namespace = os.Getenv("NAMESPACE")
	log.Debug("setting NAMESPACE to " + Namespace)
	if Namespace == "" {
		log.Error("NAMESPACE env var not set")
		os.Exit(2)
	}

	COMMAND = os.Getenv("COMMAND")
	log.Debug("setting COMMAND to " + COMMAND)
	if COMMAND == "" {
		log.Error("COMMAND env var not set")
		os.Exit(2)
	}
	PODNAME = os.Getenv("PODNAME")
	log.Debug("setting PODNAME to " + PODNAME)
	if PODNAME == "" {
		log.Error("PODNAME env var not set")
		os.Exit(2)
	}

	config, err := buildConfig(*kubeconfig)
	if err != nil {
		panic(err)
	}

	Clientset, err = kubernetes.NewForConfig(config)
	if err != nil {
		log.Info("error creating Clientset")
		panic(err.Error())
	}

	cmd := make([]string, 0)

	switch COMMAND {
	case crv1.PgtaskBackrestInfo:
		log.Info("backrest info command requested")
		cmd = append(cmd, "ls")
		cmd = append(cmd, "/pgdata")
		//pgbackrest --stanza=db info
		//cmd = append(cmd, backrestCommand)
		//cmd = append(cmd, backrestStanza)
		//cmd = append(cmd, backrestBackupCommand)
	case crv1.PgtaskBackrestBackup:
		log.Info("backrest backup command requested")
		cmd = append(cmd, "ls")
		cmd = append(cmd, "/pgdata")
	//pgbackrest --stanza=db backup
	//cmd = append(cmd, backrestCommand)
	//cmd = append(cmd, backrestStanza)
	//cmd = append(cmd, backrestBackupCommand)
	default:
		log.Error("unsupported backup command specified " + COMMAND)
		os.Exit(2)
	}

	log.Infof("command is %v ", cmd)
	output, stderr, err := kubeapi.ExecToPodThroughAPI(config, Clientset, cmd, containername, PODNAME, Namespace, nil)
	/**
	err = util.Exec(config, Namespace, PODNAME, containername, cmd)
	if err != nil {
		log.Error(err)
	}
	*/
	log.Info("output=[" + output + "]")
	log.Info("stderr=[" + stderr + "]")

	log.Info("pgo-backrest ends")

}

func buildConfig(kubeconfig string) (*rest.Config, error) {
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfig)
	}
	return rest.InClusterConfig()
}
<commit_msg>fix pgo backrest for backup command<commit_after>package main

import (
	"flag"
	log "github.com/Sirupsen/logrus"
	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	//"github.com/crunchydata/postgres-operator/util"
	"github.com/crunchydata/postgres-operator/kubeapi"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"os"
)

var Clientset *kubernetes.Clientset
var COMMAND, PODNAME, Namespace string

const backrestCommand = "pgbackrest"

//pgbackrest --stanza=db backup
const backrestStanza = "--stanza=db"
const backrestBackupCommand = "backup"
const backrestInfoCommand = "info"
const containername = "database"

func main() {
	log.Info("pgo-backrest starts")
	kubeconfig := flag.String("kubeconfig", "", "Path to a kube config. Only required if out-of-cluster.")
	flag.Parse()

	debugFlag := os.Getenv("CRUNCHY_DEBUG")
	if debugFlag == "true" {
		log.SetLevel(log.DebugLevel)
		log.Debug("debug flag set to true")
	} else {
		log.Info("debug flag set to false")
	}

	Namespace = os.Getenv("NAMESPACE")
	log.Debug("setting NAMESPACE to " + Namespace)
	if Namespace == "" {
		log.Error("NAMESPACE env var not set")
		os.Exit(2)
	}

	COMMAND = os.Getenv("COMMAND")
	log.Debug("setting COMMAND to " + COMMAND)
	if COMMAND == "" {
		log.Error("COMMAND env var not set")
		os.Exit(2)
	}
	PODNAME = os.Getenv("PODNAME")
	log.Debug("setting PODNAME to " + PODNAME)
	if PODNAME == "" {
		log.Error("PODNAME env var not set")
		os.Exit(2)
	}

	config, err := buildConfig(*kubeconfig)
	if err != nil {
		panic(err)
	}

	Clientset, err = kubernetes.NewForConfig(config)
	if err != nil {
		log.Info("error creating Clientset")
		panic(err.Error())
	}

	cmd := make([]string, 0)

	switch COMMAND {
	case crv1.PgtaskBackrestInfo:
		log.Info("backrest info command requested")
		//cmd = append(cmd, "ls")
		//cmd = append(cmd, "/pgdata")
		//pgbackrest --stanza=db info
		cmd = append(cmd, backrestCommand)
		cmd = append(cmd, backrestStanza)
		cmd = append(cmd, backrestBackupCommand)
	case crv1.PgtaskBackrestBackup:
		log.Info("backrest backup command requested")
		//cmd = append(cmd, "ls")
		//cmd = append(cmd, "/pgdata")
		//pgbackrest --stanza=db backup
		cmd = append(cmd, backrestCommand)
		cmd = append(cmd, backrestStanza)
		cmd = append(cmd, backrestBackupCommand)
	default:
		log.Error("unsupported backup command specified " + COMMAND)
		os.Exit(2)
	}

	log.Infof("command is %v ", cmd)
	output, stderr, err := kubeapi.ExecToPodThroughAPI(config, Clientset, cmd, containername, PODNAME, Namespace, nil)
	/**
	err = util.Exec(config, Namespace, PODNAME, containername, cmd)
	if err != nil {
		log.Error(err)
	}
	*/
	log.Info("output=[" + output + "]")
	log.Info("stderr=[" + stderr + "]")

	log.Info("pgo-backrest ends")

}

func buildConfig(kubeconfig string) (*rest.Config, error) {
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfig)
	}
	return rest.InClusterConfig()
}
<|endoftext|>


<commit_before>package pigae

import (
	"fmt"
	"net/http"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"google.golang.org/appengine"
)

//
// 2016-09 strategy: no file upload.
// Admin only.
// Admin types URL, preferably to dedicated GCS bucket programming-idioms-pictures.
//

// IdiomAddPictureFacade is the Facade for the Add Idiom Picture page.
type IdiomAddPictureFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Idiom       *Idiom
}

func idiomAddPicture(w http.ResponseWriter, r *http.Request) error {
	if !IsAdmin(r) {
		return fmt.Errorf("For now, only the Admin may add an idiom picture.")
	}

	vars := mux.Vars(r)
	c := appengine.NewContext(r)

	idiomIDStr := vars["idiomId"]
	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	_, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	myToggles := copyToggles(toggles)
	myToggles["editing"] = true

	data := &IdiomAddPictureFacade{
		PageMeta: PageMeta{
			PageTitle: fmt.Sprintf("Adding picture to Idiom %d : %s", idiom.Id, idiom.Title),
			Toggles:   myToggles,
		},
		UserProfile: readUserProfile(r),
		Idiom:       idiom,
	}

	return templates.ExecuteTemplate(w, "page-idiom-add-picture", data)
}

func idiomSavePicture(w http.ResponseWriter, r *http.Request) error {
	if !IsAdmin(r) {
		return fmt.Errorf("For now, only the Admin may add an idiom picture.")
	}

	c := appengine.NewContext(r)

	idiomIDStr := r.FormValue("idiom_id")
	pictureURL := r.FormValue("picture_url")

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	idiom.ImageURL = pictureURL

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceIdiomURL(idiom), http.StatusFound)
	return nil
}
<commit_msg>Edit summary for picture URL update.<commit_after>package pigae

import (
	"fmt"
	"net/http"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"google.golang.org/appengine"
)

//
// 2016-09 strategy: no file upload.
// Admin only.
// Admin types URL, preferably to dedicated GCS bucket programming-idioms-pictures.
//

// IdiomAddPictureFacade is the Facade for the Add Idiom Picture page.
type IdiomAddPictureFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Idiom       *Idiom
}

func idiomAddPicture(w http.ResponseWriter, r *http.Request) error {
	if !IsAdmin(r) {
		return fmt.Errorf("For now, only the Admin may add an idiom picture.")
	}

	vars := mux.Vars(r)
	c := appengine.NewContext(r)

	idiomIDStr := vars["idiomId"]
	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	_, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	myToggles := copyToggles(toggles)
	myToggles["editing"] = true

	data := &IdiomAddPictureFacade{
		PageMeta: PageMeta{
			PageTitle: fmt.Sprintf("Adding picture to Idiom %d : %s", idiom.Id, idiom.Title),
			Toggles:   myToggles,
		},
		UserProfile: readUserProfile(r),
		Idiom:       idiom,
	}

	return templates.ExecuteTemplate(w, "page-idiom-add-picture", data)
}

func idiomSavePicture(w http.ResponseWriter, r *http.Request) error {
	if !IsAdmin(r) {
		return fmt.Errorf("For now, only the Admin may add an idiom picture.")
	}

	c := appengine.NewContext(r)
	userProfile := readUserProfile(r)

	idiomIDStr := r.FormValue("idiom_id")
	pictureURL := r.FormValue("picture_url")

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	idiom.ImageURL = pictureURL
	idiom.EditSummary = "Updated picture URL by user [" + userProfile.Nickname + "]"

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceIdiomURL(idiom), http.StatusFound)
	return nil
}
<|endoftext|>


<commit_before>package acsengine

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/Azure/acs-engine/pkg/api"
	"github.com/Azure/acs-engine/pkg/api/v20160930"
)

const (
	TestDataDir = "./testdata"
)

func TestExpected(t *testing.T) {
	// iterate the test data directory
	apiModelTestFiles := &[]APIModelTestFile{}
	if e := IterateTestFilesDirectory(TestDataDir, apiModelTestFiles); e != nil {
		t.Error(e.Error())
		return
	}

	for _, tuple := range *apiModelTestFiles {
		containerService, version, err := api.LoadContainerServiceFromFile(tuple.APIModelFilename)
		if err != nil {
			t.Errorf("Loading file %s got error: %s", tuple.APIModelFilename, err.Error())
			continue
		}

		if version == v20160930.APIVersion {
			// v20160930 need certificate profile to match expected template
			containerService.Properties.CertificateProfile = &api.CertificateProfile{}
			addV20160930CertificateProfile(containerService.Properties.CertificateProfile)
		}

		expectedJson, e1 := ioutil.ReadFile(tuple.GetExpectedArmTemplateFilename())
		if e1 != nil {
			t.Error(e1.Error())
			continue
		}

		expectedParams, e2 := ioutil.ReadFile(tuple.GetExpectedArmTemplateParamsFilename())
		if e2 != nil {
			t.Error(e2.Error())
			continue
		}
		expectedJsonStr := strings.Replace(string(expectedJson), "\r", "", -1)
		expectedParamsStr := strings.Replace(string(expectedParams), "\r", "", -1)

		isClassicMode := false
		if strings.Contains(tuple.GetExpectedArmTemplateFilename(), "_classicmode_expected") {
			isClassicMode = true
		}

		// test the output container service 3 times:
		// 1. first time tests loaded containerService
		// 2. second time tests generated containerService
		// 3. third time tests the generated containerService from the generated containerService
		templateGenerator, e3 := InitializeTemplateGenerator(isClassicMode)
		if e3 != nil {
			t.Error(e3.Error())
			continue
		}
		for i := 0; i < 3; i++ {
			armTemplate, params, certsGenerated, err := templateGenerator.GenerateTemplate(containerService)
			if err != nil {
				t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, err.Error()))
				continue
			}
			ppArmTemplate, e1 := PrettyPrintArmTemplate(armTemplate)
			if e1 != nil {
				t.Error(armTemplate)
				t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, e1.Error()))
				break
			}

			ppParams, e2 := PrettyPrintJSON(params)
			if e2 != nil {
				t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, e2.Error()))
				continue
			}

			if certsGenerated == true {
				t.Errorf("cert generation unexpected for %s", containerService.Properties.OrchestratorProfile.OrchestratorType)
			}

			if !bytes.Equal([]byte(expectedJsonStr), []byte(ppArmTemplate)) {
				diffstr, differr := tuple.WriteArmTemplateErrFilename([]byte(ppArmTemplate))
				if differr != nil {
					diffstr += differr.Error()
				}
				t.Errorf("generated output different from expected for model %s: '%s'", tuple.GetExpectedArmTemplateFilename(), diffstr)
			}

			if !bytes.Equal([]byte(expectedParamsStr), []byte(ppParams)) {
				diffstr, differr := tuple.WriteArmTemplateParamsErrFilename([]byte(ppParams))
				if differr != nil {
					diffstr += differr.Error()
				}
				t.Errorf("generated parameters different from expected for model %s: '%s'", tuple.GetExpectedArmTemplateParamsFilename(), diffstr)
			}

			b, err := api.SerializeContainerService(containerService, version)
			if err != nil {
				t.Error(err)
			}
			containerService, version, err = api.DeserializeContainerService(b)
			if err != nil {
				t.Error(err)
			}
			if version == v20160930.APIVersion {
				// v20160930 need certificate profile to match expected template
				containerService.Properties.CertificateProfile = &api.CertificateProfile{}
				addV20160930CertificateProfile(containerService.Properties.CertificateProfile)
			}
		}
	}
}

// APIModelTestFile holds the test file name and knows how to find the expected files
type APIModelTestFile struct {
	APIModelFilename string
}

// GetExpectedArmTemplateFilename returns the expected ARM template output for the model file
func (a *APIModelTestFile) GetExpectedArmTemplateFilename() string {
	j := strings.LastIndex(a.APIModelFilename, filepath.Ext(a.APIModelFilename))
	basename := a.APIModelFilename[:j]
	return fmt.Sprintf("%s_expected.json", basename)
}

// WriteArmTemplateErrFilename writes out an error file to sit parallel for comparison
func (a *APIModelTestFile) WriteArmTemplateErrFilename(contents []byte) (string, error) {
	filename := fmt.Sprintf("%s.err", a.GetExpectedArmTemplateFilename())
	if err := ioutil.WriteFile(filename, contents, 0600); err != nil {
		return "", err
	}
	return fmt.Sprintf("%s written for diff", filename), nil
}

// GetExpectedArmTemplateParamsFilename returns the expected ARM parameters output for the model file
func (a *APIModelTestFile) GetExpectedArmTemplateParamsFilename() string {
	j := strings.LastIndex(a.APIModelFilename, filepath.Ext(a.APIModelFilename))
	basename := a.APIModelFilename[:j]
	return fmt.Sprintf("%s_expected_params.json", basename)
}

// WriteArmTemplateParamsErrFilename writes out an error file to sit parallel for comparison
func (a *APIModelTestFile) WriteArmTemplateParamsErrFilename(contents []byte) (string, error) {
	filename := fmt.Sprintf("%s.err", a.GetExpectedArmTemplateParamsFilename())
	if err := ioutil.WriteFile(filename, contents, 0600); err != nil {
		return "", err
	}
	return fmt.Sprintf("%s written for diff", filename), nil
}

// IterateTestFilesDirectory iterates the test data directory adding api model files to the test file slice.
func IterateTestFilesDirectory(directory string, APIModelTestFiles *[]APIModelTestFile) error {
	files, err := ioutil.ReadDir(directory)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			if e := IterateTestFilesDirectory(filepath.Join(directory, file.Name()), APIModelTestFiles); e != nil {
				return e
			}
		} else {
			if !strings.Contains(file.Name(), "_expected") && strings.HasSuffix(file.Name(), ".json") {
				tuple := &APIModelTestFile{}
				tuple.APIModelFilename = filepath.Join(directory, file.Name())
				if _, ferr := os.Stat(tuple.GetExpectedArmTemplateFilename()); os.IsNotExist(ferr) {
					return fmt.Errorf("expected file '%s' is missing", tuple.GetExpectedArmTemplateFilename())
				}
				if _, ferr := os.Stat(tuple.GetExpectedArmTemplateParamsFilename()); os.IsNotExist(ferr) {
					return fmt.Errorf("expected file '%s' is missing", tuple.GetExpectedArmTemplateParamsFilename())
				}
				*APIModelTestFiles = append(*APIModelTestFiles, *tuple)
			}
		}
	}
	return nil
}

// addV20160930CertificateProfile add certificate artifacts for test purpose
func addV20160930CertificateProfile(api *api.CertificateProfile) {
	api.CaCertificate = "caCertificate"
	api.APIServerCertificate = "apiServerCertificate"
	api.APIServerPrivateKey = "apiServerPrivateKey"
	api.ClientCertificate = "clientCertificate"
	api.ClientPrivateKey = "clientPrivateKey"
	api.KubeConfigCertificate = "kubeConfigCertificate"
	api.KubeConfigPrivateKey = "kubeConfigPrivateKey"
	api.SetCAPrivateKey("")
}
<commit_msg>removed generation/validation expected templates/params<commit_after>package acsengine

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"testing"

	"github.com/Azure/acs-engine/pkg/api"
	"github.com/Azure/acs-engine/pkg/api/v20160930"
)

const (
	TestDataDir = "./testdata"
)

func TestExpected(t *testing.T) {
	// iterate the test data directory
	apiModelTestFiles := &[]APIModelTestFile{}
	if e := IterateTestFilesDirectory(TestDataDir, apiModelTestFiles); e != nil {
		t.Error(e.Error())
		return
	}

	for _, tuple := range *apiModelTestFiles {
		containerService, version, err := api.LoadContainerServiceFromFile(tuple.APIModelFilename)
		if err != nil {
			t.Errorf("Loading file %s got error: %s", tuple.APIModelFilename, err.Error())
			continue
		}

		if version == v20160930.APIVersion {
			// v20160930 need certificate profile to match expected template
			containerService.Properties.CertificateProfile = &api.CertificateProfile{}
			addV20160930CertificateProfile(containerService.Properties.CertificateProfile)
		}

		isClassicMode := false
		if strings.Contains(tuple.APIModelFilename, "_classicmode") {
			isClassicMode = true
		}

		// test the output container service 3 times:
		// 1. first time tests loaded containerService
		// 2. second time tests generated containerService
		// 3. third time tests the generated containerService from the generated containerService
		templateGenerator, e3 := InitializeTemplateGenerator(isClassicMode)
		if e3 != nil {
			t.Error(e3.Error())
			continue
		}

		armTemplate, params, certsGenerated, err := templateGenerator.GenerateTemplate(containerService)
		if err != nil {
			t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, err.Error()))
			continue
		}

		ppArmTemplate, e1 := PrettyPrintArmTemplate(armTemplate)
		if e1 != nil {
			t.Error(armTemplate)
			t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, e1.Error()))
			break
		}

		ppParams, e2 := PrettyPrintJSON(params)
		if e2 != nil {
			t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, e2.Error()))
			continue
		}

		if certsGenerated == true {
			t.Errorf("cert generation unexpected for %s", containerService.Properties.OrchestratorProfile.OrchestratorType)
		}

		for i := 0; i < 3; i++ {
			armTemplate, params, certsGenerated, err := templateGenerator.GenerateTemplate(containerService)
			if err != nil {
				t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, err.Error()))
				continue
			}
			ppArmTemplateNew, e1 := PrettyPrintArmTemplate(armTemplate)
			if e1 != nil {
				t.Error(armTemplate)
				t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, e1.Error()))
				break
			}

			ppParamsNew, e2 := PrettyPrintJSON(params)
			if e2 != nil {
				t.Error(fmt.Errorf("error in file %s: %s", tuple.APIModelFilename, e2.Error()))
				continue
			}

			if certsGenerated == true {
				t.Errorf("cert generation unexpected for %s", containerService.Properties.OrchestratorProfile.OrchestratorType)
			}

			if !bytes.Equal([]byte(ppArmTemplateNew), []byte(ppArmTemplate)) {
				diffstr, differr := tuple.WriteArmTemplateErrFilename([]byte(ppArmTemplateNew))
				if differr != nil {
					diffstr += differr.Error()
				}
				t.Errorf("generated output different from expected for model %s: '%s'", tuple.APIModelFilename, diffstr)
			}

			if !bytes.Equal([]byte(ppParamsNew), []byte(ppParams)) {
				diffstr, differr := tuple.WriteArmTemplateParamsErrFilename([]byte(ppParamsNew))
				if differr != nil {
					diffstr += differr.Error()
				}
				t.Errorf("generated parameters different from expected for model %s: '%s'", tuple.APIModelFilename, diffstr)
			}

			b, err := api.SerializeContainerService(containerService, version)
			if err != nil {
				t.Error(err)
			}
			containerService, version, err = api.DeserializeContainerService(b)
			if err != nil {
				t.Error(err)
			}
			if version == v20160930.APIVersion {
				// v20160930 need certificate profile to match expected template
				containerService.Properties.CertificateProfile = &api.CertificateProfile{}
				addV20160930CertificateProfile(containerService.Properties.CertificateProfile)
			}
		}
	}
}

// APIModelTestFile holds the test file name and knows how to find the expected files
type APIModelTestFile struct {
	APIModelFilename string
}

// WriteArmTemplateErrFilename writes out an error file to sit parallel for comparison
func (a *APIModelTestFile) WriteArmTemplateErrFilename(contents []byte) (string, error) {
	filename := fmt.Sprintf("%s_expected.err", a.APIModelFilename)
	if err := ioutil.WriteFile(filename, contents, 0600); err != nil {
		return "", err
	}
	return fmt.Sprintf("%s written for diff", filename), nil
}

// WriteArmTemplateParamsErrFilename writes out an error file to sit parallel for comparison
func (a *APIModelTestFile) WriteArmTemplateParamsErrFilename(contents []byte) (string, error) {
	filename := fmt.Sprintf("%s_expected_params.err", a.APIModelFilename)
	if err := ioutil.WriteFile(filename, contents, 0600); err != nil {
		return "", err
	}
	return fmt.Sprintf("%s written for diff", filename), nil
}

// IterateTestFilesDirectory iterates the test data directory adding api model files to the test file slice.
func IterateTestFilesDirectory(directory string, APIModelTestFiles *[]APIModelTestFile) error {
	files, err := ioutil.ReadDir(directory)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			if e := IterateTestFilesDirectory(filepath.Join(directory, file.Name()), APIModelTestFiles); e != nil {
				return e
			}
		} else {
			if !strings.Contains(file.Name(), "_expected") && strings.HasSuffix(file.Name(), ".json") {
				tuple := &APIModelTestFile{}
				tuple.APIModelFilename = filepath.Join(directory, file.Name())
				*APIModelTestFiles = append(*APIModelTestFiles, *tuple)
			}
		}
	}
	return nil
}

// addV20160930CertificateProfile add certificate artifacts for test purpose
func addV20160930CertificateProfile(api *api.CertificateProfile) {
	api.CaCertificate = "caCertificate"
	api.APIServerCertificate = "apiServerCertificate"
	api.APIServerPrivateKey = "apiServerPrivateKey"
	api.ClientCertificate = "clientCertificate"
	api.ClientPrivateKey = "clientPrivateKey"
	api.KubeConfigCertificate = "kubeConfigCertificate"
	api.KubeConfigPrivateKey = "kubeConfigPrivateKey"
	api.SetCAPrivateKey("")
}
<|endoftext|>


<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"context"
	"fmt"
	"net/http"
	"sync"
	"time"

	"cloud.google.com/go/pubsub"
	ceclient "github.com/cloudevents/sdk-go/v2/client"
	"github.com/google/knative-gcp/pkg/logging"
	"go.uber.org/zap"

	"github.com/google/knative-gcp/pkg/broker/config"
	handlerctx "github.com/google/knative-gcp/pkg/broker/handler/context"
	"github.com/google/knative-gcp/pkg/broker/handler/processors"
	"github.com/google/knative-gcp/pkg/broker/handler/processors/deliver"
	"github.com/google/knative-gcp/pkg/broker/handler/processors/fanout"
	"github.com/google/knative-gcp/pkg/broker/handler/processors/filter"
	"github.com/google/knative-gcp/pkg/metrics"
)

// FanoutPool is the sync pool for fanout handlers.
// For each broker in the config, it will attempt to create a handler.
// It will also stop/delete the handler if the corresponding broker is deleted
// in the config.
type FanoutPool struct {
	options *Options
	targets config.ReadonlyTargets
	pool    *syncMapBrokerKey

	// Pubsub client used to pull events from decoupling topics.
	pubsubClient *pubsub.Client
	// For sending retry events. We only need a shared client.
	// And we can set retry topic dynamically.
	deliverRetryClient ceclient.Client
	// For initial events delivery. We only need a shared client.
	// And we can set target address dynamically.
	deliverClient *http.Client
	statsReporter *metrics.DeliveryReporter
}

type fanoutHandlerCache struct {
	Handler
	b *config.CellTenant
}

// If somehow the existing handler's setting has deviated from the current broker config,
// we need to renew the handler.
func (hc *fanoutHandlerCache) shouldRenew(b *config.CellTenant) bool {
	if !hc.IsAlive() {
		return true
	}
	// If this really happens, it means a data corruption.
	// The handler creation will fail (which is expected).
	if b == nil || b.DecoupleQueue == nil {
		return true
	}
	if b.DecoupleQueue.Topic != hc.b.DecoupleQueue.Topic ||
		b.DecoupleQueue.Subscription != hc.b.DecoupleQueue.Subscription {
		return true
	}
	return false
}

// NewFanoutPool creates a new fanout handler pool.
func NewFanoutPool(
	targets config.ReadonlyTargets,
	pubsubClient *pubsub.Client,
	deliverClient *http.Client,
	retryClient RetryClient,
	statsReporter *metrics.DeliveryReporter,
	opts ...Option,
) (*FanoutPool, error) {
	options, err := NewOptions(opts...)
	if err != nil {
		return nil, err
	}

	if options.TimeoutPerEvent < 5*time.Second {
		return nil, fmt.Errorf("timeout per event cannot be lower than %v", 5*time.Second)
	}

	// For fanout delivery, we need a slightly shorter timeout
	// than the handler timeout per event.
	// It allows the delivery processor to timeout the delivery
	// before the handler nacks the pubsub message, which will
	// cause event re-delivery for all targets.
	if options.DeliveryTimeout == 0 || options.DeliveryTimeout > options.TimeoutPerEvent-(5*time.Second) {
		options.DeliveryTimeout = options.TimeoutPerEvent - (5 * time.Second)
	}
	p := &FanoutPool{
		targets:            targets,
		options:            options,
		pool:               &syncMapBrokerKey{},
		pubsubClient:       pubsubClient,
		deliverClient:      deliverClient,
		deliverRetryClient: retryClient,
		statsReporter:      statsReporter,
	}
	return p, nil
}

// SyncOnce syncs once the handler pool based on the targets config.
func (p *FanoutPool) SyncOnce(ctx context.Context) error {
	ctx, err := p.statsReporter.AddTags(ctx)
	if err != nil {
		logging.FromContext(ctx).Error("failed to add tags to context", zap.Error(err))
	}

	p.pool.Range(func(key config.CellTenantKey, value *fanoutHandlerCache) bool {
		if _, ok := p.targets.GetCellTenantByKey(&key); !ok {
			value.Stop()
			p.pool.Delete(key)
		}
		return true
	})

	p.targets.RangeCellTenants(func(b *config.CellTenant) bool {
		if value, ok := p.pool.Load(*b.Key()); ok {
			// Skip if we don't need to renew the handler.
			if !value.shouldRenew(b) {
				return true
			}
			// Stop and clean up the old handler before we start a new one.
			value.Stop()
			p.pool.Delete(*b.Key())
		}

		// Don't start the handler if broker is not ready.
		// The decouple topic/sub might not be ready at this point.
		if b.State != config.State_READY {
			return true
		}

		sub := p.pubsubClient.Subscription(b.DecoupleQueue.Subscription)
		sub.ReceiveSettings = p.options.PubsubReceiveSettings

		h := NewHandler(
			sub,
			processors.ChainProcessors(
				&fanout.Processor{MaxConcurrency: p.options.MaxConcurrencyPerEvent, Targets: p.targets},
				&filter.Processor{Targets: p.targets},
				&deliver.Processor{
					DeliverClient:      p.deliverClient,
					Targets:            p.targets,
					RetryOnFailure:     true,
					DeliverRetryClient: p.deliverRetryClient,
					DeliverTimeout:     p.options.DeliveryTimeout,
					StatsReporter:      p.statsReporter,
				},
			),
			p.options.TimeoutPerEvent,
		)
		hc := &fanoutHandlerCache{
			Handler: *h,
			b:       b,
		}

		// Start the handler with broker key in context.
		hc.Start(handlerctx.WithBrokerKey(ctx, b.Key()), func(err error) {
			if err != nil {
				logging.FromContext(ctx).Error("handler for broker has stopped with error", zap.Stringer("broker", b.Key()), zap.Error(err))
			} else {
				logging.FromContext(ctx).Info("handler for broker has stopped", zap.Stringer("broker", b.Key()))
			}
		})

		p.pool.Store(*b.Key(), hc)
		return true
	})

	return nil
}

// syncMapBrokerKey is a typed version of sync.Map.
type syncMapBrokerKey struct {
	m sync.Map
}

func (m *syncMapBrokerKey) Store(k config.CellTenantKey, v *fanoutHandlerCache) {
	m.m.Store(k, v)
}

func (m *syncMapBrokerKey) Load(k config.CellTenantKey) (*fanoutHandlerCache, bool) {
	v, ok := m.m.Load(k)
	if v == nil {
		return nil, ok
	}
	return v.(*fanoutHandlerCache), ok
}

func (m *syncMapBrokerKey) Delete(k config.CellTenantKey) {
	m.m.Delete(k)
}

func (m *syncMapBrokerKey) Range(f func(key config.CellTenantKey, value *fanoutHandlerCache) bool) {
	wrapped := func(key interface{}, value interface{}) bool {
		var wrappedValue *fanoutHandlerCache
		if value != nil {
			wrappedValue = value.(*fanoutHandlerCache)
		}
		return f(key.(config.CellTenantKey), wrappedValue)
	}
	m.m.Range(wrapped)
}
<commit_msg>Remove magic 5 seconds in fanout (#2098)<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"context"
	"fmt"
	"net/http"
	"sync"
	"time"

	"cloud.google.com/go/pubsub"
	ceclient "github.com/cloudevents/sdk-go/v2/client"
	"github.com/google/knative-gcp/pkg/logging"
	"go.uber.org/zap"

	"github.com/google/knative-gcp/pkg/broker/config"
	handlerctx "github.com/google/knative-gcp/pkg/broker/handler/context"
	"github.com/google/knative-gcp/pkg/broker/handler/processors"
	"github.com/google/knative-gcp/pkg/broker/handler/processors/deliver"
	"github.com/google/knative-gcp/pkg/broker/handler/processors/fanout"
	"github.com/google/knative-gcp/pkg/broker/handler/processors/filter"
	"github.com/google/knative-gcp/pkg/metrics"
)

const (
	// timeoutCushion is the difference between event timeout and delivery
	// timeout so that delivery can timeout first to trigger retry
	timeoutCushion = 5 * time.Second
)

// FanoutPool is the sync pool for fanout handlers.
// For each broker in the config, it will attempt to create a handler.
// It will also stop/delete the handler if the corresponding broker is deleted
// in the config.
type FanoutPool struct {
	options *Options
	targets config.ReadonlyTargets
	pool    *syncMapBrokerKey

	// Pubsub client used to pull events from decoupling topics.
	pubsubClient *pubsub.Client
	// For sending retry events. We only need a shared client.
	// And we can set retry topic dynamically.
	deliverRetryClient ceclient.Client
	// For initial events delivery. We only need a shared client.
	// And we can set target address dynamically.
	deliverClient *http.Client
	statsReporter *metrics.DeliveryReporter
}

type fanoutHandlerCache struct {
	Handler
	b *config.CellTenant
}

// If somehow the existing handler's setting has deviated from the current broker config,
// we need to renew the handler.
func (hc *fanoutHandlerCache) shouldRenew(b *config.CellTenant) bool {
	if !hc.IsAlive() {
		return true
	}
	// If this really happens, it means a data corruption.
	// The handler creation will fail (which is expected).
	if b == nil || b.DecoupleQueue == nil {
		return true
	}
	if b.DecoupleQueue.Topic != hc.b.DecoupleQueue.Topic ||
		b.DecoupleQueue.Subscription != hc.b.DecoupleQueue.Subscription {
		return true
	}
	return false
}

// NewFanoutPool creates a new fanout handler pool.
func NewFanoutPool(
	targets config.ReadonlyTargets,
	pubsubClient *pubsub.Client,
	deliverClient *http.Client,
	retryClient RetryClient,
	statsReporter *metrics.DeliveryReporter,
	opts ...Option,
) (*FanoutPool, error) {
	options, err := NewOptions(opts...)
	if err != nil {
		return nil, err
	}

	if options.TimeoutPerEvent < timeoutCushion {
		return nil, fmt.Errorf("timeout per event cannot be lower than %v", 5*time.Second)
	}

	// For fanout delivery, we need a slightly shorter timeout
	// than the handler timeout per event.
	// It allows the delivery processor to timeout the delivery
	// before the handler nacks the pubsub message, which will
	// cause event re-delivery for all targets.
	if options.DeliveryTimeout == 0 || options.DeliveryTimeout > options.TimeoutPerEvent-timeoutCushion {
		options.DeliveryTimeout = options.TimeoutPerEvent - timeoutCushion
	}
	p := &FanoutPool{
		targets:            targets,
		options:            options,
		pool:               &syncMapBrokerKey{},
		pubsubClient:       pubsubClient,
		deliverClient:      deliverClient,
		deliverRetryClient: retryClient,
		statsReporter:      statsReporter,
	}
	return p, nil
}

// SyncOnce syncs once the handler pool based on the targets config.
func (p *FanoutPool) SyncOnce(ctx context.Context) error {
	ctx, err := p.statsReporter.AddTags(ctx)
	if err != nil {
		logging.FromContext(ctx).Error("failed to add tags to context", zap.Error(err))
	}

	p.pool.Range(func(key config.CellTenantKey, value *fanoutHandlerCache) bool {
		if _, ok := p.targets.GetCellTenantByKey(&key); !ok {
			value.Stop()
			p.pool.Delete(key)
		}
		return true
	})

	p.targets.RangeCellTenants(func(b *config.CellTenant) bool {
		if value, ok := p.pool.Load(*b.Key()); ok {
			// Skip if we don't need to renew the handler.
			if !value.shouldRenew(b) {
				return true
			}
			// Stop and clean up the old handler before we start a new one.
			value.Stop()
			p.pool.Delete(*b.Key())
		}

		// Don't start the handler if broker is not ready.
		// The decouple topic/sub might not be ready at this point.
		if b.State != config.State_READY {
			return true
		}

		sub := p.pubsubClient.Subscription(b.DecoupleQueue.Subscription)
		sub.ReceiveSettings = p.options.PubsubReceiveSettings

		h := NewHandler(
			sub,
			processors.ChainProcessors(
				&fanout.Processor{MaxConcurrency: p.options.MaxConcurrencyPerEvent, Targets: p.targets},
				&filter.Processor{Targets: p.targets},
				&deliver.Processor{
					DeliverClient:      p.deliverClient,
					Targets:            p.targets,
					RetryOnFailure:     true,
					DeliverRetryClient: p.deliverRetryClient,
					DeliverTimeout:     p.options.DeliveryTimeout,
					StatsReporter:      p.statsReporter,
				},
			),
			p.options.TimeoutPerEvent,
		)
		hc := &fanoutHandlerCache{
			Handler: *h,
			b:       b,
		}

		// Start the handler with broker key in context.
		hc.Start(handlerctx.WithBrokerKey(ctx, b.Key()), func(err error) {
			if err != nil {
				logging.FromContext(ctx).Error("handler for broker has stopped with error", zap.Stringer("broker", b.Key()), zap.Error(err))
			} else {
				logging.FromContext(ctx).Info("handler for broker has stopped", zap.Stringer("broker", b.Key()))
			}
		})

		p.pool.Store(*b.Key(), hc)
		return true
	})

	return nil
}

// syncMapBrokerKey is a typed version of sync.Map.
type syncMapBrokerKey struct {
	m sync.Map
}

func (m *syncMapBrokerKey) Store(k config.CellTenantKey, v *fanoutHandlerCache) {
	m.m.Store(k, v)
}

func (m *syncMapBrokerKey) Load(k config.CellTenantKey) (*fanoutHandlerCache, bool) {
	v, ok := m.m.Load(k)
	if v == nil {
		return nil, ok
	}
	return v.(*fanoutHandlerCache), ok
}

func (m *syncMapBrokerKey) Delete(k config.CellTenantKey) {
	m.m.Delete(k)
}

func (m *syncMapBrokerKey) Range(f func(key config.CellTenantKey, value *fanoutHandlerCache) bool) {
	wrapped := func(key interface{}, value interface{}) bool {
		var wrappedValue *fanoutHandlerCache
		if value != nil {
			wrappedValue = value.(*fanoutHandlerCache)
		}
		return f(key.(config.CellTenantKey), wrappedValue)
	}
	m.m.Range(wrapped)
}
<|endoftext|>