<commit_before>package store

import (
	"github.com/balzaczyy/golucene/core/codec"
	"github.com/balzaczyy/golucene/core/util"
	"sync"
)

// store/CompoundFileWriter.java

type AtomicBool struct {
	*sync.RWMutex
	v bool
}

func NewAtomicBool() *AtomicBool {
	return &AtomicBool{&sync.RWMutex{}, false}
}

func (b *AtomicBool) Get() bool {
	b.RLock()
	defer b.RUnlock()
	return b.v
}

func (b *AtomicBool) CompareAndSet(from, to bool) bool {
	b.Lock()
	defer b.Unlock()
	if b.v == from {
		b.v = to
	}
	return b.v
}

type FileEntry struct {
	file           string    // source file
	length, offset int64     // temporary holder for the start of this file's data section
	dir            Directory // which contains the file.
}

// Combines multiple files into a single compound file
type CompoundFileWriter struct {
	sync.Locker
	directory      Directory
	entries        map[string]FileEntry
	seenIDs        map[string]bool
	closed         bool
	dataOut        IndexOutput
	outputTaken    *AtomicBool
	entryTableName string
	dataFileName   string
}

/*
Create the compound stream in the specified file. The filename is the
entire name (no extensions are added).
*/
func newCompoundFileWriter(dir Directory, name string) *CompoundFileWriter {
	assert2(dir != nil, "directory cannot be nil")
	assert2(name != "", "name cannot be empty")
	return &CompoundFileWriter{
		Locker:      &sync.Mutex{},
		directory:   dir,
		entries:     make(map[string]FileEntry),
		seenIDs:     make(map[string]bool),
		outputTaken: NewAtomicBool(),
		entryTableName: util.SegmentFileName(
			util.StripExtension(name),
			"",
			COMPOUND_FILE_ENTRIES_EXTENSION,
		),
		dataFileName: name,
	}
}

func (w *CompoundFileWriter) output() (IndexOutput, error) {
	w.Lock()
	defer w.Unlock()
	if w.dataOut == nil {
		var success = false
		defer func() {
			if !success {
				util.CloseWhileSuppressingError(w.dataOut)
			}
		}()

		var err error
		w.dataOut, err = w.directory.CreateOutput(w.dataFileName, IO_CONTEXT_DEFAULT)
		if err != nil {
			return nil, err
		}
		err = codec.WriteHeader(w.dataOut, CFD_DATA_CODEC, CFD_VERSION_CURRENT)
		if err != nil {
			return nil, err
		}
		success = true
	}
	return w.dataOut, nil
}

/* Closes all resouces and writes the entry table */
func (w *CompoundFileWriter) Close() error {
	panic("not implemented yet")
}

func (w *CompoundFileWriter) ensureOpen() {
	assert2(!w.closed, "CFS Directory is already closed")
}

func (w *CompoundFileWriter) createOutput(name string, context IOContext) (IndexOutput, error) {
	w.ensureOpen()
	var success = false
	var outputLocked = false
	defer func() {
		if !success {
			delete(w.entries, name)
			if outputLocked { // release the output lock if not successful
				assert(w.outputTaken.Get())
				w.releaseOutputLock()
			}
		}
	}()

	assert2(name != "", "name must not be empty")
	_, ok := w.entries[name]
	assert2(!ok, "File %v already exists", name)
	entry := FileEntry{}
	entry.file = name
	w.entries[name] = entry
	id := util.StripSegmentName(name)
	_, ok = w.seenIDs[id]
	assert2(!ok, "file='%v' maps to id='%v', which was already written", name, id)
	w.seenIDs[id] = true

	var out *DirectCFSIndexOutput
	if outputLocked := w.outputTaken.CompareAndSet(false, true); outputLocked {
		o, err := w.output()
		if err != nil {
			return nil, err
		}
		out = newDirectCFSIndexOutput(w, o, entry, false)
	} else {
		entry.dir = w.directory
		assert2(!w.directory.FileExists(name), "File %v already exists", name)
		o, err := w.directory.CreateOutput(name, context)
		if err != nil {
			return nil, err
		}
		out = newDirectCFSIndexOutput(w, o, entry, true)
	}
	success = true
	return out, nil
}

func (w *CompoundFileWriter) releaseOutputLock() {
	w.outputTaken.CompareAndSet(true, false)
}

type DirectCFSIndexOutput struct {
	*IndexOutputImpl
}

func newDirectCFSIndexOutput(owner *CompoundFileWriter,
	delegate IndexOutput, entry FileEntry, isSeparate bool) *DirectCFSIndexOutput {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) Flush() error {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) Close() error {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) FilePointer() int64 {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) Length() (int64, error) {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) WriteByte(b byte) error {
	panic("not implemented yet")
}
func (out *DirectCFSIndexOutput) WriteBytes(b []byte) error {
	panic("not implemented yet")
}
<commit_msg>implement DirectCFSIndexOutput ctor<commit_after>package store

import (
	"github.com/balzaczyy/golucene/core/codec"
	"github.com/balzaczyy/golucene/core/util"
	"sync"
)

// store/CompoundFileWriter.java

type AtomicBool struct {
	*sync.RWMutex
	v bool
}

func NewAtomicBool() *AtomicBool {
	return &AtomicBool{&sync.RWMutex{}, false}
}

func (b *AtomicBool) Get() bool {
	b.RLock()
	defer b.RUnlock()
	return b.v
}

func (b *AtomicBool) CompareAndSet(from, to bool) bool {
	b.Lock()
	defer b.Unlock()
	if b.v == from {
		b.v = to
	}
	return b.v
}

type FileEntry struct {
	file           string    // source file
	length, offset int64     // temporary holder for the start of this file's data section
	dir            Directory // which contains the file.
}

// Combines multiple files into a single compound file
type CompoundFileWriter struct {
	sync.Locker
	directory      Directory
	entries        map[string]FileEntry
	seenIDs        map[string]bool
	closed         bool
	dataOut        IndexOutput
	outputTaken    *AtomicBool
	entryTableName string
	dataFileName   string
}

/*
Create the compound stream in the specified file. The filename is the
entire name (no extensions are added).
*/
func newCompoundFileWriter(dir Directory, name string) *CompoundFileWriter {
	assert2(dir != nil, "directory cannot be nil")
	assert2(name != "", "name cannot be empty")
	return &CompoundFileWriter{
		Locker:      &sync.Mutex{},
		directory:   dir,
		entries:     make(map[string]FileEntry),
		seenIDs:     make(map[string]bool),
		outputTaken: NewAtomicBool(),
		entryTableName: util.SegmentFileName(
			util.StripExtension(name),
			"",
			COMPOUND_FILE_ENTRIES_EXTENSION,
		),
		dataFileName: name,
	}
}

func (w *CompoundFileWriter) output() (IndexOutput, error) {
	w.Lock()
	defer w.Unlock()
	if w.dataOut == nil {
		var success = false
		defer func() {
			if !success {
				util.CloseWhileSuppressingError(w.dataOut)
			}
		}()

		var err error
		w.dataOut, err = w.directory.CreateOutput(w.dataFileName, IO_CONTEXT_DEFAULT)
		if err != nil {
			return nil, err
		}
		err = codec.WriteHeader(w.dataOut, CFD_DATA_CODEC, CFD_VERSION_CURRENT)
		if err != nil {
			return nil, err
		}
		success = true
	}
	return w.dataOut, nil
}

/* Closes all resouces and writes the entry table */
func (w *CompoundFileWriter) Close() error {
	panic("not implemented yet")
}

func (w *CompoundFileWriter) ensureOpen() {
	assert2(!w.closed, "CFS Directory is already closed")
}

func (w *CompoundFileWriter) createOutput(name string, context IOContext) (IndexOutput, error) {
	w.ensureOpen()
	var success = false
	var outputLocked = false
	defer func() {
		if !success {
			delete(w.entries, name)
			if outputLocked { // release the output lock if not successful
				assert(w.outputTaken.Get())
				w.releaseOutputLock()
			}
		}
	}()

	assert2(name != "", "name must not be empty")
	_, ok := w.entries[name]
	assert2(!ok, "File %v already exists", name)
	entry := FileEntry{}
	entry.file = name
	w.entries[name] = entry
	id := util.StripSegmentName(name)
	_, ok = w.seenIDs[id]
	assert2(!ok, "file='%v' maps to id='%v', which was already written", name, id)
	w.seenIDs[id] = true

	var out *DirectCFSIndexOutput
	if outputLocked := w.outputTaken.CompareAndSet(false, true); outputLocked {
		o, err := w.output()
		if err != nil {
			return nil, err
		}
		out = newDirectCFSIndexOutput(w, o, entry, false)
	} else {
		entry.dir = w.directory
		assert2(!w.directory.FileExists(name), "File %v already exists", name)
		o, err := w.directory.CreateOutput(name, context)
		if err != nil {
			return nil, err
		}
		out = newDirectCFSIndexOutput(w, o, entry, true)
	}
	success = true
	return out, nil
}

func (w *CompoundFileWriter) releaseOutputLock() {
	w.outputTaken.CompareAndSet(true, false)
}

type DirectCFSIndexOutput struct {
	*IndexOutputImpl
	owner      *CompoundFileWriter
	delegate   IndexOutput
	offset     int64
	entry      FileEntry
	isSeparate bool
}

func newDirectCFSIndexOutput(owner *CompoundFileWriter,
	delegate IndexOutput, entry FileEntry, isSeparate bool) *DirectCFSIndexOutput {
	ans := &DirectCFSIndexOutput{
		owner:      owner,
		delegate:   delegate,
		entry:      entry,
		offset:     delegate.FilePointer(),
		isSeparate: isSeparate,
	}
	ans.entry.offset = ans.offset
	ans.IndexOutputImpl = NewIndexOutput(ans)
	return ans
}

func (out *DirectCFSIndexOutput) Flush() error {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) Close() error {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) FilePointer() int64 {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) Length() (int64, error) {
	panic("not implemented yet")
}

func (out *DirectCFSIndexOutput) WriteByte(b byte) error {
	panic("not implemented yet")
}
func (out *DirectCFSIndexOutput) WriteBytes(b []byte) error {
	panic("not implemented yet")
}
<|endoftext|>


<commit_before>package automaton

import (
	"github.com/balzaczyy/golucene/core/util"
)

// util/automaton/Automaton.java

/*
Represents an automaton and all its states and transitions. States
are integers and must be created using {@link #createState}.  Mark a
state as an accept state using {@link #setAccept}.  Add transitions
using {@link #addTransition}.  Each state must have all of its
transitions added at once; if this is too restrictive then use
{@link Automaton.Builder} instead.  State 0 is always the
initial state.  Once a state is finished, either
because you've starting adding transitions to another state or you
call {@link #finishState}, then that states transitions are sorted
(first by min, then max, then dest) and reduced (transitions with
adjacent labels going to the same dest are combined).
*/
type Automaton struct {
	curState      int
	states        []int // 2x
	transitions   []int // 3x
	isAccept      *util.OpenBitSet
	deterministic bool
}

func newEmptyAutomaton() *Automaton {
	return &Automaton{
		deterministic: true,
		curState:      -1,
		isAccept:      util.NewOpenBitSet(),
	}
}

/* Create a new state. */
func (a *Automaton) createState() int {
	state := len(a.states) / 2
	a.states = append(a.states, -1, 0)
	return state
}

/* Set or clear this state as an accept state. */
func (a *Automaton) setAccept(state int, accept bool) {
	assert2(state < a.numStates(), "state=%v is out of bounds (numStates=%v)", state, a.numStates())
	if accept {
		a.isAccept.Set(int64(state))
	} else {
		a.isAccept.Clear(int64(state))
	}
}

/* Returns true if this state is an accept state. */
func (a *Automaton) IsAccept(state int) bool {
	return a.isAccept.Get(int64(state))
}

/* Add a new transition with min = max = label. */
func (a *Automaton) addTransition(source, dest, label int) {
	a.addTransitionRange(source, dest, label, label)
}

/* Add a new transition with the specified source, dest, min, max. */
func (a *Automaton) addTransitionRange(source, dest, min, max int) {
	assert(len(a.transitions)%3 == 0)
	assert2(source < a.numStates(), "source=%v is out of bounds (maxState is %v)", source, a.numStates()-1)
	assert2(dest < a.numStates(), "dest=%v is out of bounds (maxState is %v)", dest, a.numStates()-1)

	if a.curState != source {
		if a.curState != -1 {
			a.finishCurrentState()
		}

		// move to next source:
		a.curState = source
		assert2(a.states[2*a.curState] == -1, "from state (%v) already had transitions added", source)
		assert(a.states[2*a.curState+1] == 0)
		a.states[2*a.curState] = len(a.transitions)
	}

	a.transitions = append(a.transitions, dest, min, max)

	// increment transition count for this state
	a.states[2*a.curState+1]++
}

/*
Add a [virtual] epsilon transition between source and dest. Dest
state must already have all transitions added because this method
simply copies those same transitions over to source.
*/
func (a *Automaton) addEpsilon(source, dest int) {
	t := newTransition()
	count := a.initTransition(dest, t)
	for i := 0; i < count; i++ {
		a.nextTransition(t)
		a.addTransitionRange(source, t.dest, t.min, t.max)
	}
	if a.IsAccept(dest) {
		a.setAccept(source, true)
	}
}

/*
Copies over all state/transition from other. The state numbers are
sequentially assigned (appended).
*/
func (a *Automaton) copy(other *Automaton) {
	// bulk copy and then fixup the state pointers
	stateOffset := a.numStates()
	a.states = append(a.states, other.states...)
	for i := 0; i < len(other.states); i += 2 {
		if a.states[stateOffset*2+i] != -1 {
			a.states[stateOffset*2+i] += len(a.transitions)
		}
	}
	otherNumStates := other.numStates()
	for state := 0; state < otherNumStates; state++ {
		if !other.IsAccept(state) {
			continue
		}
		a.setAccept(stateOffset+state, true)
	}

	// bulk copy and then fixup dest for each transition
	transOffset := len(a.transitions)
	a.transitions = append(a.transitions, other.transitions...)
	for i := 0; i < len(other.transitions); i += 3 {
		a.transitions[transOffset+i] += stateOffset
	}

	if !other.deterministic {
		a.deterministic = false
	}
}

/* Freezes the last state, sorting and reducing the transitions. */
func (a *Automaton) finishCurrentState() {
	numTransitions := a.states[2*a.curState+1]
	assert(numTransitions > 0)

	offset := a.states[2*a.curState]
	start := offset / 3
	util.NewInPlaceMergeSorter(destMinMaxTransitions(a.transitions)).Sort(start, start+numTransitions)

	// reduce any "adjacent" transitions:
	upto, min, max, dest := 0, -1, -1, -1

	for i := 0; i < numTransitions; i++ {
		tDest := a.transitions[offset+3*i]
		tMin := a.transitions[offset+3*i+1]
		tMax := a.transitions[offset+3*i+2]

		if dest == tDest {
			panic("niy")
		} else {
			if dest != -1 {
				a.transitions[offset+3*upto] = dest
				a.transitions[offset+3*upto+1] = min
				a.transitions[offset+3*upto+2] = max
			}
			dest, min, max = tDest, tMin, tMax
		}
	}

	if dest != -1 {
		// last transition
		a.transitions[offset+3*upto] = dest
		a.transitions[offset+3*upto+1] = min
		a.transitions[offset+3*upto+2] = max
		upto++
	}

	a.transitions = a.transitions[:len(a.transitions)-(numTransitions-upto)*3]
	a.states[2*a.curState+1] = upto

	// sort transitions by min/max/dest:
	util.NewInPlaceMergeSorter(minMaxDestTransitions(a.transitions)).Sort(start, start+upto)

	if a.deterministic && upto > 1 {
		panic("niy")
	}
}

/*
Finishes the current state; call this once you are done adding
transitions for a state. This is automatically called if you start
adding transitions to a new source state, but for the last state you
add, you need to call this method yourself.
*/
func (a *Automaton) finishState() {
	if a.curState != -1 {
		a.finishCurrentState()
		a.curState = -1
	}
}

/* How many states this automaton has. */
func (a *Automaton) numStates() int {
	return len(a.states) / 2
}

/* How many transitions this state has. */
func (a *Automaton) numTransitions(state int) int {
	if count := a.states[2*state+1]; count != -1 {
		return count
	}
	return 0
}

type destMinMaxTransitions []int

func (s destMinMaxTransitions) Len() int {
	panic("niy")
}

func (s destMinMaxTransitions) Swap(i, j int) {
	panic("niy")
}

func (s destMinMaxTransitions) Less(i, j int) bool {
	iStart := 3 * i
	jStart := 3 * j

	// first dest:
	iDest := s[iStart]
	jDest := s[jStart]
	if iDest < jDest {
		return true
	} else if iDest > jDest {
		return false
	}

	// then min:
	iMin := s[iStart+1]
	jMin := s[jStart+1]
	if iMin < jMin {
		return true
	} else if iMin > jMin {
		return false
	}

	// then max:
	iMax := s[iStart+2]
	jMax := s[jStart+2]
	return iMax < jMax
}

type minMaxDestTransitions []int

func (s minMaxDestTransitions) Len() int {
	panic("niy")
}

func (s minMaxDestTransitions) Swap(i, j int) {
	panic("niy")
}

func (s minMaxDestTransitions) Less(i, j int) bool {
	panic("niy")
}

/*
Initialize the provided Transition to iterate through all transitions
leaving the specified state. You must call nextTransition() to get
each transition. Returns the number of transitions leaving this tate.
*/
func (a *Automaton) initTransition(state int, t *Transition) int {
	assert2(state < a.numStates(), "state=%v nextState=%v", state, a.numStates())
	t.source = state
	t.transitionUpto = a.states[2*state]
	return a.numTransitions(state)
}

/* Iterate to the next transition after the provided one */
func (a *Automaton) nextTransition(t *Transition) {
	// make sure there is still a transition left
	assert((t.transitionUpto + 3 - a.states[2*t.source]) <= 3*a.states[2*t.source+1])
	t.dest = a.transitions[t.transitionUpto]
	t.min = a.transitions[t.transitionUpto+1]
	t.max = a.transitions[t.transitionUpto+2]
	t.transitionUpto += 3
}

// Go doesn't have unicode.MinRune which should be 0
const MIN_CODE_POINT = 0

type AutomatonBuilder struct {
	transitions []int
	a           *Automaton
}

func newAutomatonBuilder() *AutomatonBuilder {
	panic("niy")
}

func (b *AutomatonBuilder) addTransitionRange(source, dest, min, max int) {
	panic("niy")
}

func (b *AutomatonBuilder) finish() *Automaton {
	panic("niy")
}

func (b *AutomatonBuilder) createState() int {
	return b.a.createState()
}
<commit_msg>implement AutomatonBuilder<commit_after>package automaton

import (
	"fmt"
	"github.com/balzaczyy/golucene/core/util"
)

// util/automaton/Automaton.java

/*
Represents an automaton and all its states and transitions. States
are integers and must be created using {@link #createState}.  Mark a
state as an accept state using {@link #setAccept}.  Add transitions
using {@link #addTransition}.  Each state must have all of its
transitions added at once; if this is too restrictive then use
{@link Automaton.Builder} instead.  State 0 is always the
initial state.  Once a state is finished, either
because you've starting adding transitions to another state or you
call {@link #finishState}, then that states transitions are sorted
(first by min, then max, then dest) and reduced (transitions with
adjacent labels going to the same dest are combined).
*/
type Automaton struct {
	curState      int
	states        []int // 2x
	transitions   []int // 3x
	isAccept      *util.OpenBitSet
	deterministic bool
}

func newEmptyAutomaton() *Automaton {
	return &Automaton{
		deterministic: true,
		curState:      -1,
		isAccept:      util.NewOpenBitSet(),
	}
}

/* Create a new state. */
func (a *Automaton) createState() int {
	state := len(a.states) / 2
	a.states = append(a.states, -1, 0)
	return state
}

/* Set or clear this state as an accept state. */
func (a *Automaton) setAccept(state int, accept bool) {
	assert2(state < a.numStates(), "state=%v is out of bounds (numStates=%v)", state, a.numStates())
	if accept {
		a.isAccept.Set(int64(state))
	} else {
		a.isAccept.Clear(int64(state))
	}
}

/* Returns true if this state is an accept state. */
func (a *Automaton) IsAccept(state int) bool {
	return a.isAccept.Get(int64(state))
}

/* Add a new transition with min = max = label. */
func (a *Automaton) addTransition(source, dest, label int) {
	a.addTransitionRange(source, dest, label, label)
}

/* Add a new transition with the specified source, dest, min, max. */
func (a *Automaton) addTransitionRange(source, dest, min, max int) {
	assert(len(a.transitions)%3 == 0)
	assert2(source < a.numStates(), "source=%v is out of bounds (maxState is %v)", source, a.numStates()-1)
	assert2(dest < a.numStates(), "dest=%v is out of bounds (maxState is %v)", dest, a.numStates()-1)

	if a.curState != source {
		if a.curState != -1 {
			a.finishCurrentState()
		}

		// move to next source:
		a.curState = source
		assert2(a.states[2*a.curState] == -1, "from state (%v) already had transitions added", source)
		assert(a.states[2*a.curState+1] == 0)
		a.states[2*a.curState] = len(a.transitions)
	}

	a.transitions = append(a.transitions, dest, min, max)

	// increment transition count for this state
	a.states[2*a.curState+1]++
}

/*
Add a [virtual] epsilon transition between source and dest. Dest
state must already have all transitions added because this method
simply copies those same transitions over to source.
*/
func (a *Automaton) addEpsilon(source, dest int) {
	t := newTransition()
	count := a.initTransition(dest, t)
	for i := 0; i < count; i++ {
		a.nextTransition(t)
		a.addTransitionRange(source, t.dest, t.min, t.max)
	}
	if a.IsAccept(dest) {
		a.setAccept(source, true)
	}
}

/*
Copies over all state/transition from other. The state numbers are
sequentially assigned (appended).
*/
func (a *Automaton) copy(other *Automaton) {
	// bulk copy and then fixup the state pointers
	stateOffset := a.numStates()
	a.states = append(a.states, other.states...)
	for i := 0; i < len(other.states); i += 2 {
		if a.states[stateOffset*2+i] != -1 {
			a.states[stateOffset*2+i] += len(a.transitions)
		}
	}
	otherNumStates := other.numStates()
	for state := 0; state < otherNumStates; state++ {
		if !other.IsAccept(state) {
			continue
		}
		a.setAccept(stateOffset+state, true)
	}

	// bulk copy and then fixup dest for each transition
	transOffset := len(a.transitions)
	a.transitions = append(a.transitions, other.transitions...)
	for i := 0; i < len(other.transitions); i += 3 {
		a.transitions[transOffset+i] += stateOffset
	}

	if !other.deterministic {
		a.deterministic = false
	}
}

/* Freezes the last state, sorting and reducing the transitions. */
func (a *Automaton) finishCurrentState() {
	numTransitions := a.states[2*a.curState+1]
	assert(numTransitions > 0)

	offset := a.states[2*a.curState]
	start := offset / 3
	util.NewInPlaceMergeSorter(destMinMaxTransitions(a.transitions)).Sort(start, start+numTransitions)

	// reduce any "adjacent" transitions:
	upto, min, max, dest := 0, -1, -1, -1

	for i := 0; i < numTransitions; i++ {
		tDest := a.transitions[offset+3*i]
		tMin := a.transitions[offset+3*i+1]
		tMax := a.transitions[offset+3*i+2]

		if dest == tDest {
			panic("niy")
		} else {
			if dest != -1 {
				a.transitions[offset+3*upto] = dest
				a.transitions[offset+3*upto+1] = min
				a.transitions[offset+3*upto+2] = max
			}
			dest, min, max = tDest, tMin, tMax
		}
	}

	if dest != -1 {
		// last transition
		a.transitions[offset+3*upto] = dest
		a.transitions[offset+3*upto+1] = min
		a.transitions[offset+3*upto+2] = max
		upto++
	}

	a.transitions = a.transitions[:len(a.transitions)-(numTransitions-upto)*3]
	a.states[2*a.curState+1] = upto

	// sort transitions by min/max/dest:
	util.NewInPlaceMergeSorter(minMaxDestTransitions(a.transitions)).Sort(start, start+upto)

	if a.deterministic && upto > 1 {
		panic("niy")
	}
}

/*
Finishes the current state; call this once you are done adding
transitions for a state. This is automatically called if you start
adding transitions to a new source state, but for the last state you
add, you need to call this method yourself.
*/
func (a *Automaton) finishState() {
	if a.curState != -1 {
		a.finishCurrentState()
		a.curState = -1
	}
}

/* How many states this automaton has. */
func (a *Automaton) numStates() int {
	return len(a.states) / 2
}

/* How many transitions this state has. */
func (a *Automaton) numTransitions(state int) int {
	if count := a.states[2*state+1]; count != -1 {
		return count
	}
	return 0
}

type destMinMaxTransitions []int

func (s destMinMaxTransitions) Len() int {
	panic("niy")
}

func (s destMinMaxTransitions) Swap(i, j int) {
	panic("niy")
}

func (s destMinMaxTransitions) Less(i, j int) bool {
	iStart := 3 * i
	jStart := 3 * j

	// first dest:
	iDest := s[iStart]
	jDest := s[jStart]
	if iDest < jDest {
		return true
	} else if iDest > jDest {
		return false
	}

	// then min:
	iMin := s[iStart+1]
	jMin := s[jStart+1]
	if iMin < jMin {
		return true
	} else if iMin > jMin {
		return false
	}

	// then max:
	iMax := s[iStart+2]
	jMax := s[jStart+2]
	return iMax < jMax
}

type minMaxDestTransitions []int

func (s minMaxDestTransitions) Len() int {
	panic("niy")
}

func (s minMaxDestTransitions) Swap(i, j int) {
	panic("niy")
}

func (s minMaxDestTransitions) Less(i, j int) bool {
	panic("niy")
}

/*
Initialize the provided Transition to iterate through all transitions
leaving the specified state. You must call nextTransition() to get
each transition. Returns the number of transitions leaving this tate.
*/
func (a *Automaton) initTransition(state int, t *Transition) int {
	assert2(state < a.numStates(), "state=%v nextState=%v", state, a.numStates())
	t.source = state
	t.transitionUpto = a.states[2*state]
	return a.numTransitions(state)
}

/* Iterate to the next transition after the provided one */
func (a *Automaton) nextTransition(t *Transition) {
	// make sure there is still a transition left
	assert((t.transitionUpto + 3 - a.states[2*t.source]) <= 3*a.states[2*t.source+1])
	t.dest = a.transitions[t.transitionUpto]
	t.min = a.transitions[t.transitionUpto+1]
	t.max = a.transitions[t.transitionUpto+2]
	t.transitionUpto += 3
}

// Go doesn't have unicode.MinRune which should be 0
const MIN_CODE_POINT = 0

type AutomatonBuilder struct {
	transitions []int
	a           *Automaton
}

func newAutomatonBuilder() *AutomatonBuilder {
	return &AutomatonBuilder{
		a: newEmptyAutomaton(),
	}
}

func (b *AutomatonBuilder) addTransitionRange(source, dest, min, max int) {
	b.transitions = append(b.transitions, source, dest, min, max)
}

type srcMinMaxDestSorter []int

func (s srcMinMaxDestSorter) Len() int {
	panic("niy")
}

func (s srcMinMaxDestSorter) Swap(i, j int) {
	panic("niy")
}

func (s srcMinMaxDestSorter) Less(i, j int) bool {
	panic("niy")
}

/* Compiles all added states and transitions into a new Automaton and returns it. */
func (b *AutomatonBuilder) finish() *Automaton {
	fmt.Printf("LA.Builder.finish: count=%v\n", len(b.transitions)/4)
	fmt.Println("finish pending")
	util.NewInPlaceMergeSorter(srcMinMaxDestSorter(b.transitions)).Sort(0, len(b.transitions)/4)
	for upto := 0; upto < len(b.transitions); upto += 4 {
		b.a.addTransitionRange(
			b.transitions[upto],
			b.transitions[upto+1],
			b.transitions[upto+2],
			b.transitions[upto]+3,
		)
	}

	b.a.finishState()
	return b.a
}

func (b *AutomatonBuilder) createState() int {
	return b.a.createState()
}
<|endoftext|>


<commit_before>package engine

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/ssddanbrown/haste/options"
)

// A Manager keeps control over builds and keeps track of what files are in build
// in addition to containing build-based configuration such as syntax patterns.
type Manager struct {
	options *options.Options

	buildFiles map[string]*BuildFile
	glob       string
	globDepth  int
}

// NewManager creates and initializes a new Manager with a set of defaults
func NewManager(options *options.Options) *Manager {
	m := &Manager{
		options:    options,
		buildFiles: make(map[string]*BuildFile),
		glob:       "*" + options.BuildFileExtension,
		globDepth:  5,
	}

	if options.InputPaths != nil {
		m.loadPaths(options.InputPaths)
	}

	return m
}

func (m *Manager) loadPaths(paths []string) error {
	for _, path := range paths {
		err := m.loadPath(path)
		if err != nil {
			return err
		}
	}
	return nil
}

func (m *Manager) loadPath(path string) error {
	absPath, err := filepath.Abs(path)
	fileStat, err := os.Stat(absPath)
	if err != nil {
		return err
	}

	if !fileStat.IsDir() {
		m.addBuildFile(absPath)
		return err
	}

	newBuildFiles, err := m.scanNewBuildFiles(absPath)
	for _, buildFilePath := range newBuildFiles {
		m.addBuildFile(buildFilePath)
	}

	return err
}

func (m *Manager) BuildAll() []string {

	var outPaths []string

	for _, bf := range m.buildFiles {
		outPath, err := m.BuildToFile(bf)
		outPaths = append(outPaths, outPath)
		if err != nil {
			fmt.Println(err)
		}
	}

	return outPaths
}

func (m *Manager) BuildToFile(b *BuildFile) (string, error) {
	relPath := b.path
	outPath := strings.TrimSuffix(relPath, ".haste.html") + ".html"
	outPath = filepath.Join(m.options.OutPath, outPath)
	outPathDir := filepath.Dir(outPath)
	err := os.MkdirAll(outPathDir, os.ModePerm)
	if err != nil {
		return outPath, err
	}

	reader, err := m.Build(b)
	file, err := os.Create(outPath)
	if err != nil {
		return outPath, err
	}

	io.Copy(file, reader)
	return outPath, err
}

func (m *Manager) NotifyChange(file string) []string {
	var outPaths []string

	// If a BuildFile rebuild and exit
	match, err := filepath.Match(m.glob, filepath.Base(file))

	if match && err == nil {
		bf := m.addBuildFile(file)
		outPath, err := m.BuildToFile(bf)
		outPaths = append(outPaths, outPath)
		if err != nil {
			fmt.Println(err)
		}
		return outPaths
	}

	// Rebuild any BuildFiles that depend on this file
	for _, bf := range m.buildFiles {

		if _, ok := bf.includes[file]; ok {
			outPath, err := m.BuildToFile(bf)
			outPaths = append(outPaths, outPath)
			if err != nil {
				fmt.Println(err)
			}
		}
	}
	return outPaths
}

func (m *Manager) Build(buildFile *BuildFile) (io.Reader, error) {
	fmt.Println("Building:", buildFile.path)
	file, err := os.Open(buildFile.path)
	builder := NewBuilder(file, m.options, nil)
	bReader := builder.Build()
	buildFile.includes = builder.FilesParsed
	return bReader, err
}

func (m *Manager) addBuildFile(path string) *BuildFile {
	if bf, ok := m.buildFiles[path]; ok {
		return bf
	}

	newBuild := NewBuildFile(path)
	m.buildFiles[newBuild.path] = newBuild
	return newBuild
}

func (m *Manager) scanNewBuildFiles(root string) ([]string, error) {
	var fileList []string
	err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		match, err := filepath.Match(m.glob, f.Name())
		if match && err == nil {
			fileList = append(fileList, path)
		}
		return nil
	})
	return fileList, err
}
<commit_msg>Fixed some paths and made buildAll parallel<commit_after>package engine

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ssddanbrown/haste/options"
)

// A Manager keeps control over builds and keeps track of what files are in build
// in addition to containing build-based configuration such as syntax patterns.
type Manager struct {
	options *options.Options

	buildFiles map[string]*BuildFile
	glob       string
	globDepth  int
}

// NewManager creates and initializes a new Manager with a set of defaults
func NewManager(options *options.Options) *Manager {
	m := &Manager{
		options:    options,
		buildFiles: make(map[string]*BuildFile),
		glob:       "*" + options.BuildFileExtension,
		globDepth:  5,
	}

	if options.InputPaths != nil {
		m.loadPaths(options.InputPaths)
	}

	return m
}

func (m *Manager) loadPaths(paths []string) error {
	for _, path := range paths {
		err := m.loadPath(path)
		if err != nil {
			return err
		}
	}
	return nil
}

func (m *Manager) loadPath(path string) error {
	absPath, err := filepath.Abs(path)
	relPath, err := filepath.Rel(m.options.RootPath, path)
	fileStat, err := os.Stat(absPath)
	if err != nil {
		return err
	}

	if !fileStat.IsDir() {
		m.addBuildFile(relPath)
		return err
	}

	newBuildFiles, err := m.scanNewBuildFiles(absPath)
	for _, buildFilePath := range newBuildFiles {
		buildRelPath, err := filepath.Rel(m.options.RootPath, buildFilePath)
		if err == nil {
			m.addBuildFile(buildRelPath)
		}
	}

	return err
}

func (m *Manager) BuildAll() []string {

	var outPaths []string
	var wg sync.WaitGroup

	for _, bf := range m.buildFiles {
		wg.Add(1)
		go func(bf *BuildFile) {
			defer wg.Done()
			outPath, err := m.BuildToFile(bf)
			outPaths = append(outPaths, outPath)
			if err != nil {
				fmt.Println(err)
			}
		}(bf)
	}

	wg.Wait()
	return outPaths
}

func (m *Manager) BuildToFile(b *BuildFile) (string, error) {
	relPath := b.path
	outPath := strings.TrimSuffix(relPath, m.options.BuildFileExtension) + ".html"
	outPath = filepath.Join(m.options.OutPath, outPath)
	outPathDir := filepath.Dir(outPath)
	err := os.MkdirAll(outPathDir, os.ModePerm)
	if err != nil {
		return outPath, err
	}

	reader, err := m.Build(b)
	file, err := os.Create(outPath)
	if err != nil {
		return outPath, err
	}

	io.Copy(file, reader)
	return outPath, err
}

func (m *Manager) NotifyChange(file string) []string {
	var outPaths []string

	// If a BuildFile rebuild and exit
	match, err := filepath.Match(m.glob, filepath.Base(file))

	if match && err == nil {
		bf := m.addBuildFile(file)
		outPath, err := m.BuildToFile(bf)
		outPaths = append(outPaths, outPath)
		if err != nil {
			fmt.Println(err)
		}
		return outPaths
	}

	// Rebuild any BuildFiles that depend on this file
	for _, bf := range m.buildFiles {

		if _, ok := bf.includes[file]; ok {
			outPath, err := m.BuildToFile(bf)
			outPaths = append(outPaths, outPath)
			if err != nil {
				fmt.Println(err)
			}
		}
	}
	return outPaths
}

func (m *Manager) Build(buildFile *BuildFile) (io.Reader, error) {
	fmt.Println("Building:", buildFile.path)
	file, err := os.Open(buildFile.path)
	builder := NewBuilder(file, m.options, nil)
	bReader := builder.Build()
	buildFile.includes = builder.FilesParsed
	return bReader, err
}

func (m *Manager) addBuildFile(path string) *BuildFile {
	if bf, ok := m.buildFiles[path]; ok {
		return bf
	}

	newBuild := NewBuildFile(path)
	m.buildFiles[newBuild.path] = newBuild
	return newBuild
}

func (m *Manager) scanNewBuildFiles(root string) ([]string, error) {
	var fileList []string
	err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		match, err := filepath.Match(m.glob, f.Name())
		if match && err == nil {
			fileList = append(fileList, path)
		}
		return nil
	})
	return fileList, err
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2017      Sergio Rubio
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Sergio Rubio <sergio@rubio.im>
 */

package s3bee

import (
	"mime"
	"os"
	"path/filepath"
	"strings"

	"github.com/minio/minio-go"
	"github.com/muesli/beehive/bees"
)

type S3Bee struct {
	bees.Bee
	client *minio.Client
}

// Action triggers the action passed to it.
func (bee *S3Bee) Action(action bees.Action) []bees.Placeholder {
	outs := []bees.Placeholder{}

	switch action.Name {
	case "upload":
		bucket := ""
		action.Options.Bind("bucket", &bucket)

		path := ""
		action.Options.Bind("path", &path)

		objectPath := ""
		action.Options.Bind("object_path", &objectPath)

		if objectPath == "" {
			objectPath = filepath.Base(path)
		}

		_, err := bee.client.FPutObject(bucket, objectPath, path, mime.TypeByExtension(filepath.Ext(path)))
		if err != nil {
			bee.LogFatal(err)
		}
	default:
		panic("Unknown action triggered in " + bee.Name() + ": " + action.Name)
	}

	return outs
}

func (bee *S3Bee) ReloadOptions(options bees.BeeOptions) {
	bee.SetOptions(options)

	endpoint := getConfigValue("endpoint", &options)

	var useSSL bool
	options.Bind("use_ssl", &useSSL)

	accessKeyID := getConfigValue("access_key_id", &options)
	secretAccessKey := getConfigValue("secret_access_key", &options)

	client, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		panic(err)
		return
	}

	bee.client = client
}

func getConfigValue(key string, options *bees.BeeOptions) string {
	var value string
	options.Bind(key, &value)

	if strings.HasPrefix(value, "env://") {
		buf := strings.TrimPrefix(value, "env://")
		value = os.Getenv(string(buf))
	}

	return strings.TrimSpace(value)
}
<commit_msg>Adapt S3Bee to upstream changes<commit_after>/*
 *    Copyright (C) 2017      Sergio Rubio
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Sergio Rubio <sergio@rubio.im>
 */

package s3bee

import (
	"mime"
	"os"
	"path/filepath"
	"strings"

	"github.com/minio/minio-go"
	"github.com/muesli/beehive/bees"
)

type S3Bee struct {
	bees.Bee
	client *minio.Client
}

// Action triggers the action passed to it.
func (bee *S3Bee) Action(action bees.Action) []bees.Placeholder {
	outs := []bees.Placeholder{}

	switch action.Name {
	case "upload":
		bucket := ""
		action.Options.Bind("bucket", &bucket)

		path := ""
		action.Options.Bind("path", &path)

		objectPath := ""
		action.Options.Bind("object_path", &objectPath)

		if objectPath == "" {
			objectPath = filepath.Base(path)
		}

		_, err := bee.client.FPutObject(bucket, objectPath, path, minio.PutObjectOptions{ContentType: mime.TypeByExtension(filepath.Ext(path))})
		if err != nil {
			bee.LogFatal(err)
		}
	default:
		panic("Unknown action triggered in " + bee.Name() + ": " + action.Name)
	}

	return outs
}

func (bee *S3Bee) ReloadOptions(options bees.BeeOptions) {
	bee.SetOptions(options)

	endpoint := getConfigValue("endpoint", &options)

	var useSSL bool
	options.Bind("use_ssl", &useSSL)

	accessKeyID := getConfigValue("access_key_id", &options)
	secretAccessKey := getConfigValue("secret_access_key", &options)

	client, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	if err != nil {
		panic(err)
		return
	}

	bee.client = client
}

func getConfigValue(key string, options *bees.BeeOptions) string {
	var value string
	options.Bind(key, &value)

	if strings.HasPrefix(value, "env://") {
		buf := strings.TrimPrefix(value, "env://")
		value = os.Getenv(string(buf))
	}

	return strings.TrimSpace(value)
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`
const healthcheckTimeout = 5 * time.Minute

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]*string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, &container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, healthcheckTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			log.Printf(`[%s] Container %s for %s is healthy`, user.Username, appName, message.From)

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]deployedService, oldContainers []types.Container) {
	addCounter(1)
	defer addCounter(-1)
	defer cancel()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
		log.Printf(`[%s] Succeeded to deploy %s`, user.Username, appName)
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)
	log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

	createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(service, user, string(appName)), getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
		return nil, err
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	log.Printf(`[%s] Deploying %s`, user.Username, appName)

	oldContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		forbidden(w)
	}

	newServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = *deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		deleteServices(appName, newServices, user)
		errorHandler(w, err)
		cancel()
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{newServices})
}
<commit_msg>Update compose.go<commit_after>package docker

import (
	"context"
	"fmt"
	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`
const healthcheckTimeout = 5 * time.Minute

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
	}

	for _, container := range containers {
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while stopping service %s for %s: %v`, user.Username, service, appName, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]*string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, &container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, healthcheckTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			log.Printf(`[%s] Container %s for %s is healthy`, user.Username, appName, message.From)

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]deployedService, oldContainers []types.Container) {
	addCounter(1)
	defer addCounter(-1)
	defer cancel()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
		log.Printf(`[%s] Succeeded to deploy %s`, user.Username, appName)
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)
	log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

	createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(service, user, string(appName)), getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
		return nil, err
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func createAppHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	log.Printf(`[%s] Deploying %s`, user.Username, appName)

	oldContainers, err := listContainers(user, &appNameStr)
	if err != nil {
		errorHandler(w, err)
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		forbidden(w)
	}

	newServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = *deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		deleteServices(appName, newServices, user)
		errorHandler(w, err)
		cancel()
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
		return
	}

	jsonHttp.ResponseJSON(w, results{newServices})
}
<|endoftext|>


<commit_before>package eval

import (
	"reflect"
	"strconv"
	"syscall"
	"testing"

	"github.com/xiaq/elvish/parse"
)

func TestNewEvaluator(t *testing.T) {
	ev := NewEvaluator()
	pid := strconv.Itoa(syscall.Getpid())
	if (*ev.scope["pid"]).String() != pid {
		t.Errorf(`ev.scope["pid"] = %v, want %v`, ev.scope["pid"], pid)
	}
}

func mustParse(name, text string) *parse.ChunkNode {
	n, e := parse.Parse(name, text)
	if e != nil {
		panic("parser error")
	}
	return n
}

func stringValues(ss ...string) []Value {
	vs := make([]Value, len(ss))
	for i, s := range ss {
		vs[i] = NewString(s)
	}
	return vs
}

var evalTests = []struct {
	text   string
	wanted []Value
}{
	// Empty chunk
	{"", []Value{}},

	// Trivial command
	{"put 233 lorem ipsum", stringValues("233", "lorem", "ipsum")},

	// Byte Pipeline
	{`echo "Albert\nAllan\nAlbraham\nBerlin" | sed s/l/1/g | grep e | feedchan`,
		stringValues("A1bert", "Ber1in")},

	// Arithmetics
	// TODO test more edge cases
	{"* 353 661", stringValues("233333")},
	{"+ 233100 233", stringValues("233333")},
	{"- 233333 233100", stringValues("233")},
	{"/ 233333 353", stringValues("661")},
	{"/ 1 0", stringValues("+Inf")},

	// String literal
	{"put `such \\\"``literal`", stringValues("such \\\"`literal")},
	{`put "much \n\033[31;1m$cool\033[m"`,
		stringValues("much \n\033[31;1m$cool\033[m")},

	// Compounding list primaries
	{"put {fi elvi}sh{1 2}", stringValues("fish1", "fish2", "elvish1", "elvish2")},

	// Table and subscript
	{"println [a b c &key value] | feedchan",
		stringValues("[a b c &key value]")},
	{"put [a b c &key value][2]", stringValues("c")},
	{"put [a b c &key value][key]", stringValues("value")},

	// Variable and compounding
	{"var $x string `SHELL`\nput `WOW, SUCH `$x`, MUCH COOL`\n",
		stringValues("WOW, SUCH SHELL, MUCH COOL")},

	// var and set
	{"var [$x $y string] [SUCH VAR]; put $x $y", stringValues("SUCH", "VAR")},
	{"var [$x $y string]; set [$x $y] [SUCH SET]; put $x $y",
		stringValues("SUCH", "SET")},

	// Status capture
	{"put ?(true|false|false)", stringValues("", "exited 1", "exited 1")},
}

func TestEval(t *testing.T) {
	name := "<eval test>"
	for _, tt := range evalTests {
		n := mustParse(name, tt.text)

		ev := NewEvaluator()
		out := make(chan Value, len(tt.wanted))
		outs := []Value{}
		exhausted := make(chan struct{})
		go func() {
			for v := range out {
				outs = append(outs, v)
			}
			exhausted <- struct{}{}
		}()

		ev.ports[1].ch = out

		e := ev.Eval(name, tt.text, n)
		close(out)
		if e != nil {
			t.Errorf("ev.Eval(*, %q, *) => %v, want <nil>", tt.text, e)
		}
		<-exhausted
		if !reflect.DeepEqual(outs, tt.wanted) {
			t.Errorf("Evalling %q outputs %v, want %v", tt.text, outs, tt.wanted)
		}
	}
}
<commit_msg>eval: Also Test for eval errors<commit_after>package eval

import (
	"reflect"
	"strconv"
	"syscall"
	"testing"

	"github.com/xiaq/elvish/parse"
)

func TestNewEvaluator(t *testing.T) {
	ev := NewEvaluator()
	pid := strconv.Itoa(syscall.Getpid())
	if (*ev.scope["pid"]).String() != pid {
		t.Errorf(`ev.scope["pid"] = %v, want %v`, ev.scope["pid"], pid)
	}
}

func mustParse(name, text string) *parse.ChunkNode {
	n, e := parse.Parse(name, text)
	if e != nil {
		panic("parser error")
	}
	return n
}

func stringValues(ss ...string) []Value {
	vs := make([]Value, len(ss))
	for i, s := range ss {
		vs[i] = NewString(s)
	}
	return vs
}

var evalTests = []struct {
	text      string
	wanted    []Value
	wantError bool
}{
	// Empty chunk
	{"", []Value{}, false},

	// Trivial command
	{"put 233 lorem ipsum", stringValues("233", "lorem", "ipsum"), false},

	// Byte Pipeline
	{`echo "Albert\nAllan\nAlbraham\nBerlin" | sed s/l/1/g | grep e | feedchan`,
		stringValues("A1bert", "Ber1in"), false},

	// Arithmetics
	// TODO test more edge cases
	{"* 353 661", stringValues("233333"), false},
	{"+ 233100 233", stringValues("233333"), false},
	{"- 233333 233100", stringValues("233"), false},
	{"/ 233333 353", stringValues("661"), false},
	{"/ 1 0", stringValues("+Inf"), false},

	// String literal
	{"put `such \\\"``literal`", stringValues("such \\\"`literal"), false},
	{`put "much \n\033[31;1m$cool\033[m"`,
		stringValues("much \n\033[31;1m$cool\033[m"), false},

	// Compounding list primaries
	{"put {fi elvi}sh{1 2}", stringValues("fish1", "fish2", "elvish1", "elvish2"), false},

	// Table and subscript
	{"println [a b c &key value] | feedchan",
		stringValues("[a b c &key value]"), false},
	{"put [a b c &key value][2]", stringValues("c"), false},
	{"put [a b c &key value][key]", stringValues("value"), false},

	// Variable and compounding
	{"var $x string `SHELL`\nput `WOW, SUCH `$x`, MUCH COOL`\n",
		stringValues("WOW, SUCH SHELL, MUCH COOL"), false},

	// var and set
	{"var [$x $y string] [SUCH VAR]; put $x $y", stringValues("SUCH", "VAR"), false},
	{"var [$x $y string]; set [$x $y] [SUCH SET]; put $x $y",
		stringValues("SUCH", "SET"), false},
	{"var $x", nil, true},
	{"var [$x string $y]", nil, true},

	// Status capture
	{"put ?(true|false|false)", stringValues("", "exited 1", "exited 1"), false},
}

func TestEval(t *testing.T) {
	name := "<eval test>"
	for _, tt := range evalTests {
		n := mustParse(name, tt.text)

		ev := NewEvaluator()
		out := make(chan Value, len(tt.wanted))
		outs := []Value{}
		exhausted := make(chan struct{})
		go func() {
			for v := range out {
				outs = append(outs, v)
			}
			exhausted <- struct{}{}
		}()

		ev.ports[1].ch = out

		e := ev.Eval(name, tt.text, n)
		close(out)
		<-exhausted
		if tt.wantError {
			// Test for error, ignore output
			if e == nil {
				t.Errorf("ev.Eval(*, %q, *) => <nil>, want non-nil", tt.text)
			}
		} else {
			// Test for output
			if e != nil {
				t.Errorf("ev.Eval(*, %q, *) => %v, want <nil>", tt.text, e)
			}
			if !reflect.DeepEqual(outs, tt.wanted) {
				t.Errorf("Evalling %q outputs %v, want %v", tt.text, outs, tt.wanted)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ra

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"time"

	cert "k8s.io/api/certificates/v1"
	clientset "k8s.io/client-go/kubernetes"

	meshconfig "istio.io/api/mesh/v1alpha1"
	"istio.io/istio/security/pkg/k8s/chiron"
	"istio.io/istio/security/pkg/pki/ca"
	raerror "istio.io/istio/security/pkg/pki/error"
	"istio.io/istio/security/pkg/pki/util"
	"istio.io/pkg/log"
)

// KubernetesRA integrated with an external CA using Kubernetes CSR API
type KubernetesRA struct {
	csrInterface                 clientset.Interface
	keyCertBundle                *util.KeyCertBundle
	raOpts                       *IstioRAOptions
	caCertificatesFromMeshConfig map[string]string
	certSignerDomain             string
	// mutex protects the R/W to caCertificatesFromMeshConfig.
	mutex sync.RWMutex
}

var pkiRaLog = log.RegisterScope("pkira", "Istiod RA log", 0)

// NewKubernetesRA : Create a RA that interfaces with K8S CSR CA
func NewKubernetesRA(raOpts *IstioRAOptions) (*KubernetesRA, error) {
	keyCertBundle, err := util.NewKeyCertBundleWithRootCertFromFile(raOpts.CaCertFile)
	if err != nil {
		return nil, raerror.NewError(raerror.CAInitFail, fmt.Errorf("error processing Certificate Bundle for Kubernetes RA"))
	}
	istioRA := &KubernetesRA{
		csrInterface:                 raOpts.K8sClient,
		raOpts:                       raOpts,
		keyCertBundle:                keyCertBundle,
		certSignerDomain:             raOpts.CertSignerDomain,
		caCertificatesFromMeshConfig: make(map[string]string),
	}
	return istioRA, nil
}

func (r *KubernetesRA) kubernetesSign(csrPEM []byte, caCertFile string, certSigner string,
	requestedLifetime time.Duration,
) ([]byte, error) {
	certSignerDomain := r.certSignerDomain
	if certSignerDomain == "" && certSigner != "" {
		return nil, raerror.NewError(raerror.CertGenError, fmt.Errorf("certSignerDomain is requiered for signer %s", certSigner))
	}
	if certSignerDomain != "" && certSigner != "" {
		certSigner = certSignerDomain + "/" + certSigner
	} else {
		certSigner = r.raOpts.CaSigner
	}
	usages := []cert.KeyUsage{
		cert.UsageDigitalSignature,
		cert.UsageKeyEncipherment,
		cert.UsageServerAuth,
		cert.UsageClientAuth,
	}
	certChain, _, err := chiron.SignCSRK8s(r.csrInterface, csrPEM, certSigner, usages, "", caCertFile, true, false, requestedLifetime)
	if err != nil {
		return nil, raerror.NewError(raerror.CertGenError, err)
	}
	return certChain, err
}

// Sign takes a PEM-encoded CSR and cert opts, and returns a certificate signed by k8s CA.
func (r *KubernetesRA) Sign(csrPEM []byte, certOpts ca.CertOpts) ([]byte, error) {
	_, err := preSign(r.raOpts, csrPEM, certOpts.SubjectIDs, certOpts.TTL, certOpts.ForCA)
	if err != nil {
		return nil, err
	}
	certSigner := certOpts.CertSigner

	return r.kubernetesSign(csrPEM, r.raOpts.CaCertFile, certSigner, certOpts.TTL)
}

// SignWithCertChain is similar to Sign but returns the leaf cert and the entire cert chain.
// root cert comes from two sources, order matters:
// 1. Specified in mesh config
// 2. Extract from the cert-chain signed by the CSR signer.
// If no root cert can be found from either of the two sources, error returned.
// There are several possible situations:
// 1. root cert is specified in mesh config and is empty in signed cert chain, in this case
// we verify the signed cert chain against the root cert from mesh config and append the
// root cert into the cert chain.
// 2. root cert is specified in mesh config and also can be extracted in signed cert chain, in this
// case we verify the signed cert chain against the root cert from mesh config and append it
// into the cert chain if the two root certs are different. This is typical when
// the returned cert chain only contains the intermediate CA.
// 3. root cert is not specified in mesh config but can be extracted in signed cert chain, in this case
// we verify the signed cert chain against the root cert and return the cert chain directly.
func (r *KubernetesRA) SignWithCertChain(csrPEM []byte, certOpts ca.CertOpts) ([]string, error) {
	cert, err := r.Sign(csrPEM, certOpts)
	if err != nil {
		return nil, err
	}
	chainPem := r.GetCAKeyCertBundle().GetCertChainPem()
	if len(chainPem) > 0 {
		cert = append(cert, chainPem...)
	}
	respCertChain := []string{string(cert)}
	var possibleRootCert, rootCertFromMeshConfig, rootCertFromCertChain []byte
	certSigner := r.certSignerDomain + "/" + certOpts.CertSigner
	if len(r.GetCAKeyCertBundle().GetRootCertPem()) == 0 {
		rootCertFromCertChain, err = util.FindRootCertFromCertificateChainBytes(cert)
		if err != nil {
			return nil, fmt.Errorf("failed to find root cert from signed cert-chain (%v)", err.Error())
		}
		rootCertFromMeshConfig, err = r.GetRootCertFromMeshConfig(certSigner)
		if err != nil {
			pkiRaLog.Infof("failed to find root cert from mesh config (%v)", err.Error())
		}
		if rootCertFromMeshConfig != nil {
			possibleRootCert = rootCertFromMeshConfig
		} else if rootCertFromCertChain != nil {
			possibleRootCert = rootCertFromCertChain
		}
		if possibleRootCert == nil {
			return nil, fmt.Errorf("failed to find root cert from either signed cert-chain or mesh config")
		}
		if verifyErr := util.VerifyCertificate(nil, cert, possibleRootCert, nil); verifyErr != nil {
			return nil, fmt.Errorf("root cert from signed cert-chain is invalid %v ", verifyErr)
		}
		if !bytes.Equal(possibleRootCert, rootCertFromCertChain) {
			respCertChain = append(respCertChain, string(possibleRootCert))
		}
	}
	return respCertChain, nil
}

// GetCAKeyCertBundle returns the KeyCertBundle for the CA.
func (r *KubernetesRA) GetCAKeyCertBundle() *util.KeyCertBundle {
	return r.keyCertBundle
}

func (r *KubernetesRA) SetCACertificatesFromMeshConfig(caCertificates []*meshconfig.MeshConfig_CertificateData) {
	r.mutex.Lock()
	for _, pemCert := range caCertificates {
		// TODO:  take care of spiffe bundle format as well
		cert := pemCert.GetPem()
		certSigners := pemCert.CertSigners
		if len(certSigners) != 0 {
			certSigner := strings.Join(certSigners, ",")
			if cert != "" {
				r.caCertificatesFromMeshConfig[certSigner] = cert
			}
		}
	}
	r.mutex.Unlock()
}

func (r *KubernetesRA) GetRootCertFromMeshConfig(signerName string) ([]byte, error) {
	r.mutex.RLock()
	defer r.mutex.RUnlock()
	caCertificates := r.caCertificatesFromMeshConfig
	if len(caCertificates) == 0 {
		return nil, fmt.Errorf("no caCertificates defined in mesh config")
	}
	for signers, caCertificate := range caCertificates {
		signerList := strings.Split(signers, ",")
		if len(signerList) == 0 {
			continue
		}
		for _, signer := range signerList {
			if signer == signerName {
				return []byte(caCertificate), nil
			}
		}
	}
	return nil, fmt.Errorf("failed to find root cert for signer: %v in mesh config", signerName)
}
<commit_msg>fix CSR signing error (#42121)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ra

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"time"

	cert "k8s.io/api/certificates/v1"
	clientset "k8s.io/client-go/kubernetes"

	meshconfig "istio.io/api/mesh/v1alpha1"
	"istio.io/istio/security/pkg/k8s/chiron"
	"istio.io/istio/security/pkg/pki/ca"
	raerror "istio.io/istio/security/pkg/pki/error"
	"istio.io/istio/security/pkg/pki/util"
	"istio.io/pkg/log"
)

// KubernetesRA integrated with an external CA using Kubernetes CSR API
type KubernetesRA struct {
	csrInterface                 clientset.Interface
	keyCertBundle                *util.KeyCertBundle
	raOpts                       *IstioRAOptions
	caCertificatesFromMeshConfig map[string]string
	certSignerDomain             string
	// mutex protects the R/W to caCertificatesFromMeshConfig.
	mutex sync.RWMutex
}

var pkiRaLog = log.RegisterScope("pkira", "Istiod RA log", 0)

// NewKubernetesRA : Create a RA that interfaces with K8S CSR CA
func NewKubernetesRA(raOpts *IstioRAOptions) (*KubernetesRA, error) {
	keyCertBundle, err := util.NewKeyCertBundleWithRootCertFromFile(raOpts.CaCertFile)
	if err != nil {
		return nil, raerror.NewError(raerror.CAInitFail, fmt.Errorf("error processing Certificate Bundle for Kubernetes RA"))
	}
	istioRA := &KubernetesRA{
		csrInterface:                 raOpts.K8sClient,
		raOpts:                       raOpts,
		keyCertBundle:                keyCertBundle,
		certSignerDomain:             raOpts.CertSignerDomain,
		caCertificatesFromMeshConfig: make(map[string]string),
	}
	return istioRA, nil
}

func (r *KubernetesRA) kubernetesSign(csrPEM []byte, caCertFile string, certSigner string,
	requestedLifetime time.Duration,
) ([]byte, error) {
	certSignerDomain := r.certSignerDomain
	if certSignerDomain == "" && certSigner != "" {
		return nil, raerror.NewError(raerror.CertGenError, fmt.Errorf("certSignerDomain is requiered for signer %s", certSigner))
	}
	if certSignerDomain != "" && certSigner != "" {
		certSigner = certSignerDomain + "/" + certSigner
	} else {
		certSigner = r.raOpts.CaSigner
	}
	usages := []cert.KeyUsage{
		cert.UsageDigitalSignature,
		cert.UsageKeyEncipherment,
		cert.UsageServerAuth,
		cert.UsageClientAuth,
	}
	certChain, _, err := chiron.SignCSRK8s(r.csrInterface, csrPEM, certSigner, usages, "", caCertFile, true, false, requestedLifetime)
	if err != nil {
		return nil, raerror.NewError(raerror.CertGenError, err)
	}
	return certChain, err
}

// Sign takes a PEM-encoded CSR and cert opts, and returns a certificate signed by k8s CA.
func (r *KubernetesRA) Sign(csrPEM []byte, certOpts ca.CertOpts) ([]byte, error) {
	_, err := preSign(r.raOpts, csrPEM, certOpts.SubjectIDs, certOpts.TTL, certOpts.ForCA)
	if err != nil {
		return nil, err
	}
	certSigner := certOpts.CertSigner

	return r.kubernetesSign(csrPEM, r.raOpts.CaCertFile, certSigner, certOpts.TTL)
}

// SignWithCertChain is similar to Sign but returns the leaf cert and the entire cert chain.
// root cert comes from two sources, order matters:
// 1. Specified in mesh config
// 2. Extract from the cert-chain signed by the CSR signer.
// If no root cert can be found from either of the two sources, error returned.
// There are several possible situations:
// 1. root cert is specified in mesh config and is empty in signed cert chain, in this case
// we verify the signed cert chain against the root cert from mesh config and append the
// root cert into the cert chain.
// 2. root cert is specified in mesh config and also can be extracted in signed cert chain, in this
// case we verify the signed cert chain against the root cert from mesh config and append it
// into the cert chain if the two root certs are different. This is typical when
// the returned cert chain only contains the intermediate CA.
// 3. root cert is not specified in mesh config but can be extracted in signed cert chain, in this case
// we verify the signed cert chain against the root cert and return the cert chain directly.
func (r *KubernetesRA) SignWithCertChain(csrPEM []byte, certOpts ca.CertOpts) ([]string, error) {
	cert, err := r.Sign(csrPEM, certOpts)
	if err != nil {
		return nil, err
	}
	chainPem := r.GetCAKeyCertBundle().GetCertChainPem()
	if len(chainPem) > 0 {
		cert = append(cert, chainPem...)
	}
	respCertChain := []string{string(cert)}
	var possibleRootCert, rootCertFromMeshConfig, rootCertFromCertChain []byte
	certSigner := r.certSignerDomain + "/" + certOpts.CertSigner
	if len(r.GetCAKeyCertBundle().GetRootCertPem()) == 0 {
		rootCertFromCertChain, err = util.FindRootCertFromCertificateChainBytes(cert)
		if err != nil {
			return nil, raerror.NewError(raerror.CSRError, fmt.Errorf("failed to find root cert from signed cert-chain (%v)", err.Error()))
		}
		rootCertFromMeshConfig, err = r.GetRootCertFromMeshConfig(certSigner)
		if err != nil {
			pkiRaLog.Infof("failed to find root cert from mesh config (%v)", err.Error())
		}
		if rootCertFromMeshConfig != nil {
			possibleRootCert = rootCertFromMeshConfig
		} else if rootCertFromCertChain != nil {
			possibleRootCert = rootCertFromCertChain
		}
		if possibleRootCert == nil {
			return nil, raerror.NewError(raerror.CSRError, fmt.Errorf("failed to find root cert from either signed cert-chain or mesh config"))
		}
		if verifyErr := util.VerifyCertificate(nil, cert, possibleRootCert, nil); verifyErr != nil {
			return nil, raerror.NewError(raerror.CSRError, fmt.Errorf("root cert from signed cert-chain is invalid %v ", verifyErr))
		}
		if !bytes.Equal(possibleRootCert, rootCertFromCertChain) {
			respCertChain = append(respCertChain, string(possibleRootCert))
		}
	}
	return respCertChain, nil
}

// GetCAKeyCertBundle returns the KeyCertBundle for the CA.
func (r *KubernetesRA) GetCAKeyCertBundle() *util.KeyCertBundle {
	return r.keyCertBundle
}

func (r *KubernetesRA) SetCACertificatesFromMeshConfig(caCertificates []*meshconfig.MeshConfig_CertificateData) {
	r.mutex.Lock()
	for _, pemCert := range caCertificates {
		// TODO:  take care of spiffe bundle format as well
		cert := pemCert.GetPem()
		certSigners := pemCert.CertSigners
		if len(certSigners) != 0 {
			certSigner := strings.Join(certSigners, ",")
			if cert != "" {
				r.caCertificatesFromMeshConfig[certSigner] = cert
			}
		}
	}
	r.mutex.Unlock()
}

func (r *KubernetesRA) GetRootCertFromMeshConfig(signerName string) ([]byte, error) {
	r.mutex.RLock()
	defer r.mutex.RUnlock()
	caCertificates := r.caCertificatesFromMeshConfig
	if len(caCertificates) == 0 {
		return nil, fmt.Errorf("no caCertificates defined in mesh config")
	}
	for signers, caCertificate := range caCertificates {
		signerList := strings.Split(signers, ",")
		if len(signerList) == 0 {
			continue
		}
		for _, signer := range signerList {
			if signer == signerName {
				return []byte(caCertificate), nil
			}
		}
	}
	return nil, fmt.Errorf("failed to find root cert for signer: %v in mesh config", signerName)
}
<|endoftext|>


<commit_before>package selenoid

import (
	"fmt"
	. "github.com/aandryashin/matchers"
	"github.com/aerokube/selenoid/config"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"testing"
)

var (
	mock *httptest.Server
)

func init() {
	mock = httptest.NewServer(mux())
	os.Setenv("DOCKER_HOST", "tcp://"+hostPort(mock.URL))
	os.Setenv("DOCKER_API_VERSION", "1.29")
}

func mux() http.Handler {
	mux := http.NewServeMux()

	//Docker Registry API mock
	mux.HandleFunc("/v2/", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusOK)
		},
	))
	mux.HandleFunc("/v2/selenoid/firefox/tags/list", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Add("Content-Type", "application/json")
			fmt.Fprintln(w, `{"name":"firefox", "tags": ["46.0", "45.0", "7.0", "latest"]}`)
		},
	))

	mux.HandleFunc("/v2/selenoid/phantomjs/tags/list", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Add("Content-Type", "application/json")
			fmt.Fprintln(w, `{"name":"phantomjs", "tags": ["2.1.1", "latest"]}`)
		},
	))

	//Docker API mock
	mux.HandleFunc("/v1.29/images/create", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusOK)
		},
	))
	return mux
}

func hostPort(input string) string {
	u, err := url.Parse(input)
	if err != nil {
		panic(err)
	}
	return u.Host
}

func TestImageWithTag(t *testing.T) {
	AssertThat(t, imageWithTag("selenoid/firefox", "tag"), EqualTo{"selenoid/firefox:tag"})
}

func TestFetchImageTags(t *testing.T) {
	c, err := NewConfigurator(mock.URL, true)
	AssertThat(t, err, Is{nil})
	defer c.Close()
	tags := c.fetchImageTags("selenoid/firefox")
	AssertThat(t, len(tags), EqualTo{3})
	AssertThat(t, tags[0], EqualTo{"46.0"})
	AssertThat(t, tags[1], EqualTo{"45.0"})
	AssertThat(t, tags[2], EqualTo{"7.0"})
}

func TestPullImages(t *testing.T) {
	c, err := NewConfigurator(mock.URL, true)
	AssertThat(t, err, Is{nil})
	defer c.Close()
	tags := c.pullImages("selenoid/firefox", []string{"46.0", "45.0"})
	AssertThat(t, len(tags), EqualTo{2})
	AssertThat(t, tags[0], EqualTo{"46.0"})
	AssertThat(t, tags[1], EqualTo{"45.0"})
}

func TestCreateConfig(t *testing.T) {
	testCreateConfig(t, true)
}

func TestLimitNoPull(t *testing.T) {
	testCreateConfig(t, false)
}

func testCreateConfig(t *testing.T, pull bool) {
	c, err := NewConfigurator(mock.URL, true)
	AssertThat(t, err, Is{nil})
	c.LastVersions = 2
	c.Pull = pull
	c.Tmpfs = 512
	defer c.Close()
	cfg := c.createConfig()
	AssertThat(t, len(cfg), EqualTo{2})

	firefoxVersions, hasFirefoxKey := cfg["firefox"]
	AssertThat(t, hasFirefoxKey, Is{true})
	AssertThat(t, firefoxVersions, Is{Not{nil}})

	tmpfsMap := make(map[string]string)
	tmpfsMap["/tmp"] = "size=512m"

	correctFFBrowsers := make(map[string]*config.Browser)
	correctFFBrowsers["46.0"] = &config.Browser{
		Image: "selenoid/firefox:46.0",
		Port:  "4444",
		Path:  "/wd/hub",
		Tmpfs: tmpfsMap,
	}
	correctFFBrowsers["45.0"] = &config.Browser{
		Image: "selenoid/firefox:45.0",
		Port:  "4444",
		Path:  "/wd/hub",
		Tmpfs: tmpfsMap,
	}
	AssertThat(t, firefoxVersions, EqualTo{config.Versions{
		Default:  "46.0",
		Versions: correctFFBrowsers,
	}})

	phantomjsVersions, hasPhantomjsKey := cfg["phantomjs"]
	AssertThat(t, hasPhantomjsKey, Is{true})
	AssertThat(t, phantomjsVersions, Is{Not{nil}})
	AssertThat(t, phantomjsVersions.Default, EqualTo{"2.1.1"})

	correctPhantomjsBrowsers := make(map[string]*config.Browser)
	correctPhantomjsBrowsers["2.1.1"] = &config.Browser{
		Image: "selenoid/phantomjs:2.1.1",
		Port:  "4444",
		Tmpfs: tmpfsMap,
	}
}
<commit_msg>Fixed issue in tests<commit_after>package selenoid

import (
	"fmt"
	. "github.com/aandryashin/matchers"
	"github.com/aerokube/selenoid/config"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"testing"
)

var (
	mock *httptest.Server
)

func init() {
	mock = httptest.NewServer(mux())
	os.Setenv("DOCKER_HOST", "tcp://"+hostPort(mock.URL))
	os.Setenv("DOCKER_API_VERSION", "1.29")
}

func mux() http.Handler {
	mux := http.NewServeMux()

	//Docker Registry API mock
	mux.HandleFunc("/v2/", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusOK)
		},
	))
	mux.HandleFunc("/v2/selenoid/firefox/tags/list", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Add("Content-Type", "application/json")
			fmt.Fprintln(w, `{"name":"firefox", "tags": ["46.0", "45.0", "7.0", "latest"]}`)
		},
	))

	mux.HandleFunc("/v2/selenoid/opera/tags/list", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Add("Content-Type", "application/json")
			fmt.Fprintln(w, `{"name":"opera", "tags": ["44.0", "latest"]}`)
		},
	))

	//Docker API mock
	mux.HandleFunc("/v1.29/images/create", http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(http.StatusOK)
		},
	))
	return mux
}

func hostPort(input string) string {
	u, err := url.Parse(input)
	if err != nil {
		panic(err)
	}
	return u.Host
}

func TestImageWithTag(t *testing.T) {
	AssertThat(t, imageWithTag("selenoid/firefox", "tag"), EqualTo{"selenoid/firefox:tag"})
}

func TestFetchImageTags(t *testing.T) {
	c, err := NewConfigurator(mock.URL, true)
	AssertThat(t, err, Is{nil})
	defer c.Close()
	tags := c.fetchImageTags("selenoid/firefox")
	AssertThat(t, len(tags), EqualTo{3})
	AssertThat(t, tags[0], EqualTo{"46.0"})
	AssertThat(t, tags[1], EqualTo{"45.0"})
	AssertThat(t, tags[2], EqualTo{"7.0"})
}

func TestPullImages(t *testing.T) {
	c, err := NewConfigurator(mock.URL, true)
	AssertThat(t, err, Is{nil})
	defer c.Close()
	tags := c.pullImages("selenoid/firefox", []string{"46.0", "45.0"})
	AssertThat(t, len(tags), EqualTo{2})
	AssertThat(t, tags[0], EqualTo{"46.0"})
	AssertThat(t, tags[1], EqualTo{"45.0"})
}

func TestCreateConfig(t *testing.T) {
	testCreateConfig(t, true)
}

func TestLimitNoPull(t *testing.T) {
	testCreateConfig(t, false)
}

func testCreateConfig(t *testing.T, pull bool) {
	c, err := NewConfigurator(mock.URL, true)
	AssertThat(t, err, Is{nil})
	c.LastVersions = 2
	c.Pull = pull
	c.Tmpfs = 512
	defer c.Close()
	cfg := c.createConfig()
	AssertThat(t, len(cfg), EqualTo{2})

	firefoxVersions, hasFirefoxKey := cfg["firefox"]
	AssertThat(t, hasFirefoxKey, Is{true})
	AssertThat(t, firefoxVersions, Is{Not{nil}})

	tmpfsMap := make(map[string]string)
	tmpfsMap["/tmp"] = "size=512m"

	correctFFBrowsers := make(map[string]*config.Browser)
	correctFFBrowsers["46.0"] = &config.Browser{
		Image: "selenoid/firefox:46.0",
		Port:  "4444",
		Path:  "/wd/hub",
		Tmpfs: tmpfsMap,
	}
	correctFFBrowsers["45.0"] = &config.Browser{
		Image: "selenoid/firefox:45.0",
		Port:  "4444",
		Path:  "/wd/hub",
		Tmpfs: tmpfsMap,
	}
	AssertThat(t, firefoxVersions, EqualTo{config.Versions{
		Default:  "46.0",
		Versions: correctFFBrowsers,
	}})

	operaVersions, hasPhantomjsKey := cfg["opera"]
	AssertThat(t, hasPhantomjsKey, Is{true})
	AssertThat(t, operaVersions, Is{Not{nil}})
	AssertThat(t, operaVersions.Default, EqualTo{"44.0"})

	correctPhantomjsBrowsers := make(map[string]*config.Browser)
	correctPhantomjsBrowsers["2.1.1"] = &config.Browser{
		Image: "selenoid/opera:44.0",
		Port:  "4444",
		Tmpfs: tmpfsMap,
	}
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT

package form

import (
	"errors"
	"fmt"
	"net/url"
	"reflect"
	"strings"
	"unicode"

	"github.com/issue9/conv"
)

// Tag 在 struct tag 中的标签名称
const Tag = "form"

// 将 v 转换成 form-data 格式的数据
//
// NOTE: form-data 中不需要考虑 omitempty 的情况，因为无法处理数组和切片在有没有 omitempty 下的区别。
func marshal(v interface{}) (url.Values, error) {
	objs := map[string]reflect.Value{}
	if err := getFields(objs, "", reflect.ValueOf(v)); err != nil {
		return nil, err
	}

	vals := url.Values{}
	for k, v := range objs {
		kind := v.Kind()
		if kind != reflect.Array && kind != reflect.Slice {
			vals.Add(k, fmt.Sprint(v))
			continue
		}

		chkSliceType(v)
		for i := 0; i < v.Len(); i++ {
			vals.Add(k, fmt.Sprint(v.Index(i).Interface()))
		}
	}

	return vals, nil
}

// 将  form-data 数据转换到 v 中
func unmarshal(vals url.Values, obj interface{}) error {
	val := reflect.ValueOf(obj)
	for k, v := range vals {
		if err := setField(val, strings.Split(k, "."), v); err != nil {
			return err
		}
	}
	return nil
}

func setField(obj reflect.Value, names []string, val []string) error {
	for obj.Kind() == reflect.Ptr {
		if obj.IsNil() {
			obj.Set(reflect.New(obj.Type().Elem()))
		}
		obj = obj.Elem()
	}

	if len(names) == 0 {
		ok := obj.Kind()
		if ok == reflect.Slice || ok == reflect.Array {
			chkSliceType(obj)
			return conv.Value(val, obj)
		}
		return conv.Value(val[0], obj)
	}

	switch obj.Kind() {
	case reflect.Struct:
		return setStructField(obj, names, val)
	case reflect.Map:
		if obj.Type().Key().Kind() != reflect.String {
			return errors.New("map 类型的键值只能是字符串")
		}

		if obj.IsNil() {
			obj.Set(reflect.MakeMap(obj.Type()))
		}
		return setMapField(obj, names, val)
	case reflect.Func, reflect.Chan:
		return nil
	default:
		return errors.New("无法找到对应的字段")
	}
}

func setStructField(obj reflect.Value, names []string, val []string) error {
	rtype := obj.Type()
	l := obj.NumField()
	for i := 0; i < l; i++ {
		rf := rtype.Field(i)
		if rf.Anonymous {
			if err := setField(obj.Field(i), names, val); err != nil {
				return err
			}
			continue
		}

		if name := parseTag(rf); name == "-" || name != names[0] { // 名称不匹配或是 struct tag 中标记为忽略
			continue
		}

		return setField(obj.Field(i), names[1:], val)
	}

	return nil
}

func setMapField(obj reflect.Value, names []string, val []string) error {
	// NOTE: map 元素是 unAddressable 的

	key := reflect.ValueOf(names[0])
	elem := reflect.New(obj.Type().Elem()).Elem()
	if e := obj.MapIndex(key); e.IsValid() {
		elem.Set(e)
	}
	if err := setField(elem, names[1:], val); err != nil {
		return err
	}
	obj.SetMapIndex(key, elem)
	return nil
}

func getFields(kv map[string]reflect.Value, name string, rval reflect.Value) error {
	for rval.Kind() == reflect.Ptr {
		if rval.IsNil() {
			rval.Set(reflect.New(rval.Type().Elem()))
		}
		rval = rval.Elem()
	}

	if rval.Kind() == reflect.Map && rval.IsNil() {
		rval.Set(reflect.MakeMap(rval.Type()))
	}

	switch rval.Kind() {
	case reflect.Struct:
		return getStructFields(kv, name, rval)
	case reflect.Map:
		if rval.Type().Key().Kind() != reflect.String {
			return errors.New("map 类型的键值只能是字符串")
		}
		return getMapFields(kv, name, rval)
	case reflect.Chan, reflect.Func:
		return nil
	default:
		kv[name] = rval
		return nil
	}
}

func getStructFields(kv map[string]reflect.Value, parent string, rval reflect.Value) error {
	rtype := rval.Type()
	for i := 0; i < rtype.NumField(); i++ {
		field := rtype.Field(i)

		if field.Anonymous {
			if err := getFields(kv, parent, rval.Field(i)); err != nil {
				return err
			}
			continue
		}

		name := parseTag(field)
		if name == "-" {
			continue
		}
		if parent != "" {
			name = parent + "." + name
		}
		if err := getFields(kv, name, rval.Field(i)); err != nil {
			return err
		}
	}

	return nil
}

func getMapFields(kv map[string]reflect.Value, parent string, rval reflect.Value) error {
	for iter := rval.MapRange(); iter.Next(); {
		name := iter.Key().String()
		if parent != "" {
			name = parent + "." + name
		}

		if err := getFields(kv, name, iter.Value()); err != nil {
			return err
		}
	}
	return nil
}

func parseTag(sf reflect.StructField) string {
	if unicode.IsLower(rune(sf.Name[0])) {
		return "-"
	}

	tag := strings.TrimSpace(sf.Tag.Get(Tag))
	switch tag {
	case "":
		return sf.Name
	default: // '-' 也原样返回即可
		return tag
	}
}

func chkSliceType(v reflect.Value) {
	k := v.Type().Elem().Kind()
	if (k < reflect.Bool || k > reflect.Float64) && k != reflect.String {
		msg := fmt.Sprintf("slice 和 array 的元素类型只能是介于 [reflect.Bool, reflect.Float64] 之间或是 reflect.String，当前为 %s", k)
		panic(msg)
	}
}
<commit_msg>style(content/form): 规范变量命名<commit_after>// SPDX-License-Identifier: MIT

package form

import (
	"errors"
	"fmt"
	"net/url"
	"reflect"
	"strings"
	"unicode"

	"github.com/issue9/conv"
)

// Tag 在 struct tag 中的标签名称
const Tag = "form"

// 将 v 转换成 form-data 格式的数据
//
// NOTE: form-data 中不需要考虑 omitempty 的情况，因为无法处理数组和切片在有没有 omitempty 下的区别。
func marshal(v interface{}) (url.Values, error) {
	objs := map[string]reflect.Value{}
	if err := getFields(objs, "", reflect.ValueOf(v)); err != nil {
		return nil, err
	}

	vals := url.Values{}
	for k, v := range objs {
		kind := v.Kind()
		if kind != reflect.Array && kind != reflect.Slice {
			vals.Add(k, fmt.Sprint(v))
			continue
		}

		chkSliceType(v)
		for i := 0; i < v.Len(); i++ {
			vals.Add(k, fmt.Sprint(v.Index(i).Interface()))
		}
	}

	return vals, nil
}

// 将  form-data 数据转换到 v 中
func unmarshal(vals url.Values, obj interface{}) error {
	val := reflect.ValueOf(obj)
	for k, v := range vals {
		if err := setField(val, strings.Split(k, "."), v); err != nil {
			return err
		}
	}
	return nil
}

func setField(obj reflect.Value, names []string, val []string) error {
	for obj.Kind() == reflect.Ptr {
		if obj.IsNil() {
			obj.Set(reflect.New(obj.Type().Elem()))
		}
		obj = obj.Elem()
	}

	if len(names) == 0 {
		ok := obj.Kind()
		if ok == reflect.Slice || ok == reflect.Array {
			chkSliceType(obj)
			return conv.Value(val, obj)
		}
		return conv.Value(val[0], obj)
	}

	switch obj.Kind() {
	case reflect.Struct:
		return setStructField(obj, names, val)
	case reflect.Map:
		if obj.Type().Key().Kind() != reflect.String {
			return errors.New("map 类型的键值只能是字符串")
		}

		if obj.IsNil() {
			obj.Set(reflect.MakeMap(obj.Type()))
		}
		return setMapField(obj, names, val)
	case reflect.Func, reflect.Chan:
		return nil
	default:
		return errors.New("无法找到对应的字段")
	}
}

func setStructField(obj reflect.Value, names []string, val []string) error {
	rtype := obj.Type()
	l := obj.NumField()
	for i := 0; i < l; i++ {
		rf := rtype.Field(i)
		if rf.Anonymous {
			if err := setField(obj.Field(i), names, val); err != nil {
				return err
			}
			continue
		}

		if name := parseTag(rf); name == "-" || name != names[0] { // 名称不匹配或是 struct tag 中标记为忽略
			continue
		}

		return setField(obj.Field(i), names[1:], val)
	}

	return nil
}

func setMapField(obj reflect.Value, names []string, val []string) error {
	// NOTE: map 元素是 unAddressable 的

	key := reflect.ValueOf(names[0])
	elem := reflect.New(obj.Type().Elem()).Elem()
	if e := obj.MapIndex(key); e.IsValid() {
		elem.Set(e)
	}
	if err := setField(elem, names[1:], val); err != nil {
		return err
	}
	obj.SetMapIndex(key, elem)
	return nil
}

func getFields(kv map[string]reflect.Value, name string, rv reflect.Value) error {
	for rv.Kind() == reflect.Ptr {
		if rv.IsNil() {
			rv.Set(reflect.New(rv.Type().Elem()))
		}
		rv = rv.Elem()
	}

	if rv.Kind() == reflect.Map && rv.IsNil() {
		rv.Set(reflect.MakeMap(rv.Type()))
	}

	switch rv.Kind() {
	case reflect.Struct:
		return getStructFields(kv, name, rv)
	case reflect.Map:
		if rv.Type().Key().Kind() != reflect.String {
			return errors.New("map 类型的键值只能是字符串")
		}
		return getMapFields(kv, name, rv)
	case reflect.Chan, reflect.Func:
		return nil
	default:
		kv[name] = rv
		return nil
	}
}

func getStructFields(kv map[string]reflect.Value, parent string, rv reflect.Value) error {
	rtype := rv.Type()
	for i := 0; i < rtype.NumField(); i++ {
		field := rtype.Field(i)

		if field.Anonymous {
			if err := getFields(kv, parent, rv.Field(i)); err != nil {
				return err
			}
			continue
		}

		name := parseTag(field)
		if name == "-" {
			continue
		}
		if parent != "" {
			name = parent + "." + name
		}
		if err := getFields(kv, name, rv.Field(i)); err != nil {
			return err
		}
	}

	return nil
}

func getMapFields(kv map[string]reflect.Value, parent string, rv reflect.Value) error {
	for iter := rv.MapRange(); iter.Next(); {
		name := iter.Key().String()
		if parent != "" {
			name = parent + "." + name
		}

		if err := getFields(kv, name, iter.Value()); err != nil {
			return err
		}
	}
	return nil
}

func parseTag(sf reflect.StructField) string {
	if unicode.IsLower(rune(sf.Name[0])) {
		return "-"
	}

	tag := strings.TrimSpace(sf.Tag.Get(Tag))
	switch tag {
	case "":
		return sf.Name
	default: // '-' 也原样返回即可
		return tag
	}
}

func chkSliceType(v reflect.Value) {
	k := v.Type().Elem().Kind()
	if (k < reflect.Bool || k > reflect.Float64) && k != reflect.String {
		msg := fmt.Sprintf("slice 和 array 的元素类型只能是介于 [reflect.Bool, reflect.Float64] 之间或是 reflect.String，当前为 %s", k)
		panic(msg)
	}
}
<|endoftext|>


<commit_before><commit_msg>Fix flaky observability tests (#1993)<commit_after><|endoftext|>