<commit_before>package trie

import (
	"fmt"
	"strings"
	"sync"
)

type MemberInfo struct {
	Value string
	Count int64
}

func (m *MemberInfo) String() string {
	return fmt.Sprintf("%s(%v)", m.Value, m.Count)
}

type Branch struct {
	sync.RWMutex
	Branches  map[byte]*Branch
	LeafValue []byte
	End       bool
	Count     int64
}

/*
NewBranch returns a new initialezed *Branch
*/
func (b *Branch) NewBranch() *Branch {
	return &Branch{
		Branches: make(map[byte]*Branch),
		Count:    0,
	}
}

/*
Add adds an entry to the Branch
*/
func (b *Branch) add(entry []byte) (addedBranch *Branch) {
	if b.LeafValue == nil && len(b.Branches) == 0 {
		if len(entry) > 0 {
			b.LeafValue = entry
		} else {
			// something came in but we already have branches for it
			// so the tail was the current branches index but no value
			// to push. just mark the current idx position as End
		}
		b.setEnd(true)
		addedBranch = b
		return
	}

	// // something came in but we already have branches for it
	// // so the tail was the current branches index but no value
	// // to push. just mark the current idx position as End
	// if len(b.LeafValue) == 0 && len(entry) == 0 {
	// 	b.setEnd(true)
	// 	addedBranch = b
	// 	return
	// }

	// check the overlap between the current LeafValue and the new entry
	newLeaf := func(LeafValue, newEntry []byte) (leaf []byte) {
		for li, b := range LeafValue {
			if li > len(newEntry)-1 {
				break
			}
			if b == newEntry[li] {
				leaf = append(leaf, b)
			} else {
				break
			}
		}
		return
	}(b.LeafValue, entry)

	newLeafLen := len(newLeaf)

	// the new leaf is smaller than the current leaf.
	// we will push the old leaf down the branch
	if newLeafLen < len(b.LeafValue) {
		tail := b.LeafValue[newLeafLen:]
		idx := tail[0]
		newBranch := b.NewBranch()
		newBranch.LeafValue = tail[1:]

		b.LeafValue = newLeaf
		newBranch.Branches, b.Branches = b.Branches, newBranch.Branches
		newBranch.End, b.End = b.End, newBranch.End
		if newBranch.End {
			if b.Count > 0 {
				newBranch.Count = b.Count
			} else {
				newBranch.Count = 1
			}
		} else {
			newBranch.Count = 0
		}
		if b.End {
			b.Count = 1
		} else {
			b.Count = 0
		}
		b.Branches[idx] = newBranch
	}

	// new leaf is smaller than the entry, which means there will be more stuff
	// that we need to push down
	if newLeafLen < len(entry) {
		tail := entry[newLeafLen:]
		idx := tail[0]

		// create new branch at idx if it does not exists yet
		if _, notPresent := b.Branches[idx]; !notPresent {
			b.Branches[idx] = b.NewBranch()
		}
		// check whether the idx itself marks an End $. if so add a new idx
		addedBranch = b.Branches[idx].add(tail[1:])
	} else {
		// if there is nothing else to be pushed down we just have to mark the
		// current branch as an end. this happens when you add a value that already
		// is covered by the index but this particular end had not been marked.
		// eg. you already have 'food' and 'foot' (shared LeafValue of 'foo') in
		// your index and now add 'foo'.
		b.setEnd(true)
		addedBranch = b
	}
	return addedBranch
}

/*
Members returns slice of all Members of the Branch prepended with `branchPrefix`
*/
func (b *Branch) members(branchPrefix []byte) (members []*MemberInfo) {
	if b.End {
		members = append(members, &MemberInfo{string(append(branchPrefix, b.LeafValue...)), b.Count})
	}
	for idx, br := range b.Branches {
		newPrefix := append(append(branchPrefix, b.LeafValue...), idx)
		members = append(members, br.members(newPrefix)...)
	}
	return
}

/*
prefixMembers returns a slice of all Members of the Branch matching the given prefix. The values returned are prepended with `branchPrefix`
*/
func (b *Branch) prefixMembers(branchPrefix []byte, searchPrefix []byte) (members []*MemberInfo) {
	leafLen := len(b.LeafValue)
	searchPrefixLen := len(searchPrefix)

	// if the searchPrefix is empty we want all members
	if searchPrefixLen == 0 {
		members = append(members, b.members(branchPrefix)...)
		return
	}

	// if the searchPrefix is shorter than the leaf we will add the LeafValue
	// if it is an End and a the searchPrefix matches
	// if searchPrefixLen < leafLen {
	if searchPrefixLen > leafLen {
		for idx, br := range b.Branches {
			// does it match the next byte?
			if idx == searchPrefix[leafLen] {
				newSearchPrefix := searchPrefix[leafLen+1:]
				members = append(members, br.prefixMembers(append(append(branchPrefix, b.LeafValue...), idx), newSearchPrefix)...)
			}
		}
	} else if searchPrefixLen == leafLen {
		for i, sb := range searchPrefix {
			if sb != b.LeafValue[i] {
				return
			}
		}
		members = append(members, b.members(branchPrefix)...)
	} else {
		if b.End {
			for i, sb := range searchPrefix {
				if sb != b.LeafValue[i] {
					return
				}
			}
			members = append(members, b.members(branchPrefix)...)
			// members = append(members, &MemberInfo{string(append(branchPrefix, b.LeafValue...)), b.Count})
		}
	}
	return
}

/*
 */
func (b *Branch) delete(entry []byte) (deleted bool) {
	leafLen := len(b.LeafValue)
	entryLen := len(entry)
	// does the leafValue match?
	if leafLen > 0 {
		if entryLen >= leafLen {
			for i, lb := range b.LeafValue {
				if entry[i] != lb {
					return false
				}
			}
		} else {
			return false
		}
	}

	// entry matches leaf. zero+ length
	// if there are branches there cant be End == true with a LeafValue.
	// if there are NO branches there MUST be End == true with either a LeafValue or not

	// we are at the leafend
	// log.Println("entryLen-leafLen", entryLen, leafLen, entryLen-leafLen)
	if b.End && ((entryLen - leafLen) == 0) {
		b.setEnd(false)
		if len(b.Branches) == 0 && b.Count == 0 {
			b.LeafValue = nil
		} else if len(b.Branches) == 1 && b.Count == 0 {
			b = b.pullUp()
		}
		return true
	}

	// if End == true and there are no Branches we can delete the branch because either the idx or the LeafValue mark the end - if it is matched it can be deleted
	// this is being checked in the branch above
	// prefix is matched. check for branches
	if leafLen < entryLen && b.hasBranch(entry[leafLen]) {
		// next branch matches. check the leaf/branches again
		nextBranch := b.Branches[entry[leafLen]]
		if len(nextBranch.Branches) == 0 && nextBranch.Count == 0 {
			delete(b.Branches, entry[leafLen])
			return true
		} else {
			deleted := nextBranch.delete(entry[leafLen+1:])
			if deleted && len(nextBranch.Branches) == 0 && !nextBranch.End {
				delete(b.Branches, entry[leafLen])
				// dangling leaf value?
				if len(b.Branches) == 0 && b.Count == 0 {
					b.LeafValue = nil
				}
			}
			return deleted
		}
	}

	return false
}

/*
 */
func (b *Branch) has(entry []byte) bool {
	if b.getBranch(entry) != nil {
		return true
	}
	return false
}

func (b *Branch) hasCount(entry []byte) (bool, int64) {
	br := b.getBranch(entry)
	if br != nil {
		return true, br.Count
	}
	return false, 0
}

func (b *Branch) getBranch(entry []byte) (be *Branch) {
	leafLen := len(b.LeafValue)
	entryLen := len(entry)

	if entryLen >= leafLen {
		for i, pb := range b.LeafValue {
			if pb != entry[i] {
				return
			}
		}
	} else {
		return
	}

	if entryLen > leafLen {
		if br, present := b.Branches[entry[leafLen]]; present {
			return br.getBranch(entry[leafLen+1:])
		} else {
			return
		}
	} else if entryLen == leafLen && b.End {
		be = b
	}
	return
}

/*
 */
func (b *Branch) hasPrefix(prefix []byte) bool {
	exists, _ := b.hasPrefixBranch(prefix)
	return exists
}

func (b *Branch) hasPrefixCount(prefix []byte) (exists bool, count int64) {
	exists, br := b.hasPrefixBranch(prefix)
	if exists {
		count = br.sumCount()
	}
	return
}

func (b *Branch) hasPrefixBranch(prefix []byte) (exists bool, branch *Branch) {
	leafLen := len(b.LeafValue)
	prefixLen := len(prefix)
	exists = false

	if leafLen > 0 {
		if prefixLen <= leafLen {
			for i, pb := range prefix {
				if pb != b.LeafValue[i] {
					return
				}
			}
		} else {
			for i, lb := range b.LeafValue {
				if prefix[i] != lb {
					return
				}
			}
		}
	}

	if prefixLen > leafLen {
		if br, present := b.Branches[prefix[leafLen]]; present {
			return br.hasPrefixBranch(prefix[leafLen+1:])
		} else {
			return
		}
	}

	return true, b
}

func (b *Branch) sumCount() (count int64) {
	// leaf itself matches
	if b.End {
		// fmt.Println("***", string(b.LeafValue), b.Count)
		count += b.Count
	}
	for _, br := range b.Branches {
		// fmt.Println(string(k), br.Count)
		count += br.sumCount()
	}
	return
}

/*
 */
func (b *Branch) Dump(depth int) (out string) {
	if len(b.LeafValue) > 0 {
		if b.End {
			out += fmt.Sprintf("%s V:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth), string(b.LeafValue), b.Count)
		} else {
			out += fmt.Sprintf("%s V:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth), string(b.LeafValue), "-")
		}
	}

	if b.End {
		out += fmt.Sprintf("%s $\n", strings.Repeat(PADDING_CHAR, depth+len(b.LeafValue)))
	}

	for idx, branch := range b.Branches {
		if branch.End && len(branch.LeafValue) == 0 {
			out += fmt.Sprintf("%s I:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth+len(b.LeafValue)), string(idx), branch.Count)
		} else {
			out += fmt.Sprintf("%s I:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth+len(b.LeafValue)), string(idx), "-")
		}
		out += branch.Dump(depth + len(b.LeafValue) + 1)
	}

	return
}

/*
 */
// func (b *Branch) hasBranches() bool {
// 	return len(b.Branches) == 0
// }

/*
 */
func (b *Branch) hasBranch(idx byte) bool {
	if _, present := b.Branches[idx]; present {
		return true
	}
	return false
}

/*
 */
// func (b *Branch) matchesLeaf(entry []byte) bool {
// 	leafLen := len(b.LeafValue)
// 	entryLen := len(entry)

// 	if leafLen == 0 && entryLen == 0 {
// 		return true
// 	}

// 	if leafLen == entryLen {
// 		for i, lb := range b.LeafValue {
// 			if entry[i] != lb {
// 				return false
// 			}
// 		}
// 	}
// 	return true
// }

/*
 */
func (b *Branch) pullUp() *Branch {
	if len(b.Branches) == 1 {
		for k, nextBranch := range b.Branches {
			if len(nextBranch.Branches) == 0 {
				b.LeafValue = append(b.LeafValue, append([]byte{k}, nextBranch.LeafValue...)...)
			} else {
				b.LeafValue = append(b.LeafValue, k)
			}
			b.End = nextBranch.End
			b.Branches = nextBranch.Branches
			b.Count = nextBranch.Count
		}
		return b.pullUp()
	}
	return b
}

func (b *Branch) setEnd(flag bool) {
	if flag {
		b.Count += 1
	} else {
		if b.End && b.Count > 0 {
			b.Count -= 1
			if b.Count > 0 {
				return
			}
		}
	}
	b.End = flag
	return
}

func (b *Branch) String() string {
	return b.Dump(0)
}

func (b *Branch) PrintDump() {
	fmt.Printf("\n%s\n\n", b)
}
<commit_msg>clenaup<commit_after>package trie

import (
	"fmt"
	"strings"
	"sync"
)

type MemberInfo struct {
	Value string
	Count int64
}

func (m *MemberInfo) String() string {
	return fmt.Sprintf("%s(%v)", m.Value, m.Count)
}

type Branch struct {
	sync.RWMutex
	Branches  map[byte]*Branch
	LeafValue []byte
	End       bool
	Count     int64
}

/*
NewBranch returns a new initialezed *Branch
*/
func (b *Branch) NewBranch() *Branch {
	return &Branch{
		Branches: make(map[byte]*Branch),
		Count:    0,
	}
}

/*
Add adds an entry to the Branch
*/
func (b *Branch) add(entry []byte) (addedBranch *Branch) {
	if b.LeafValue == nil && len(b.Branches) == 0 {
		if len(entry) > 0 {
			b.LeafValue = entry
		} else {
			// something came in but we already have branches for it
			// so the tail was the current branches index but no value
			// to push. just mark the current idx position as End
		}
		b.setEnd(true)
		addedBranch = b
		return
	}

	// check the overlap between the current LeafValue and the new entry
	newLeaf := func(LeafValue, newEntry []byte) (leaf []byte) {
		for li, b := range LeafValue {
			if li > len(newEntry)-1 {
				break
			}
			if b == newEntry[li] {
				leaf = append(leaf, b)
			} else {
				break
			}
		}
		return
	}(b.LeafValue, entry)

	newLeafLen := len(newLeaf)

	// the new leaf is smaller than the current leaf.
	// we will push the old leaf down the branch
	if newLeafLen < len(b.LeafValue) {
		tail := b.LeafValue[newLeafLen:]
		idx := tail[0]
		newBranch := b.NewBranch()
		newBranch.LeafValue = tail[1:]

		b.LeafValue = newLeaf
		newBranch.Branches, b.Branches = b.Branches, newBranch.Branches
		newBranch.End, b.End = b.End, newBranch.End
		if newBranch.End {
			if b.Count > 0 {
				newBranch.Count = b.Count
			} else {
				newBranch.Count = 1
			}
		} else {
			newBranch.Count = 0
		}
		if b.End {
			b.Count = 1
		} else {
			b.Count = 0
		}
		b.Branches[idx] = newBranch
	}

	// new leaf is smaller than the entry, which means there will be more stuff
	// that we need to push down
	if newLeafLen < len(entry) {
		tail := entry[newLeafLen:]
		idx := tail[0]

		// create new branch at idx if it does not exists yet
		if _, notPresent := b.Branches[idx]; !notPresent {
			b.Branches[idx] = b.NewBranch()
		}
		// check whether the idx itself marks an End $. if so add a new idx
		addedBranch = b.Branches[idx].add(tail[1:])
	} else {
		// if there is nothing else to be pushed down we just have to mark the
		// current branch as an end. this happens when you add a value that already
		// is covered by the index but this particular end had not been marked.
		// eg. you already have 'food' and 'foot' (shared LeafValue of 'foo') in
		// your index and now add 'foo'.
		b.setEnd(true)
		addedBranch = b
	}
	return addedBranch
}

/*
Members returns slice of all Members of the Branch prepended with `branchPrefix`
*/
func (b *Branch) members(branchPrefix []byte) (members []*MemberInfo) {
	if b.End {
		members = append(members, &MemberInfo{string(append(branchPrefix, b.LeafValue...)), b.Count})
	}
	for idx, br := range b.Branches {
		newPrefix := append(append(branchPrefix, b.LeafValue...), idx)
		members = append(members, br.members(newPrefix)...)
	}
	return
}

/*
prefixMembers returns a slice of all Members of the Branch matching the given prefix. The values returned are prepended with `branchPrefix`
*/
func (b *Branch) prefixMembers(branchPrefix []byte, searchPrefix []byte) (members []*MemberInfo) {
	leafLen := len(b.LeafValue)
	searchPrefixLen := len(searchPrefix)

	// if the searchPrefix is empty we want all members
	if searchPrefixLen == 0 {
		members = append(members, b.members(branchPrefix)...)
		return
	}

	// if the searchPrefix is shorter than the leaf we will add the LeafValue
	// if it is an End and a the searchPrefix matches
	// if searchPrefixLen < leafLen {
	if searchPrefixLen > leafLen {
		for idx, br := range b.Branches {
			// does it match the next byte?
			if idx == searchPrefix[leafLen] {
				newSearchPrefix := searchPrefix[leafLen+1:]
				members = append(members, br.prefixMembers(append(append(branchPrefix, b.LeafValue...), idx), newSearchPrefix)...)
			}
		}
	} else if searchPrefixLen == leafLen {
		for i, sb := range searchPrefix {
			if sb != b.LeafValue[i] {
				return
			}
		}
		members = append(members, b.members(branchPrefix)...)
	} else {
		if b.End {
			for i, sb := range searchPrefix {
				if sb != b.LeafValue[i] {
					return
				}
			}
			members = append(members, b.members(branchPrefix)...)
			// members = append(members, &MemberInfo{string(append(branchPrefix, b.LeafValue...)), b.Count})
		}
	}
	return
}

/*
 */
func (b *Branch) delete(entry []byte) (deleted bool) {
	leafLen := len(b.LeafValue)
	entryLen := len(entry)
	// does the leafValue match?
	if leafLen > 0 {
		if entryLen >= leafLen {
			for i, lb := range b.LeafValue {
				if entry[i] != lb {
					return false
				}
			}
		} else {
			return false
		}
	}

	// entry matches leaf. zero+ length
	// if there are branches there cant be End == true with a LeafValue.
	// if there are NO branches there MUST be End == true with either a LeafValue or not

	// we are at the leafend
	// log.Println("entryLen-leafLen", entryLen, leafLen, entryLen-leafLen)
	if b.End && ((entryLen - leafLen) == 0) {
		b.setEnd(false)
		if len(b.Branches) == 0 && b.Count == 0 {
			b.LeafValue = nil
		} else if len(b.Branches) == 1 && b.Count == 0 {
			b = b.pullUp()
		}
		return true
	}

	// if End == true and there are no Branches we can delete the branch because either the idx or the LeafValue mark the end - if it is matched it can be deleted
	// this is being checked in the branch above
	// prefix is matched. check for branches
	if leafLen < entryLen && b.hasBranch(entry[leafLen]) {
		// next branch matches. check the leaf/branches again
		nextBranch := b.Branches[entry[leafLen]]
		if len(nextBranch.Branches) == 0 && nextBranch.Count == 0 {
			delete(b.Branches, entry[leafLen])
			return true
		} else {
			deleted := nextBranch.delete(entry[leafLen+1:])
			if deleted && len(nextBranch.Branches) == 0 && !nextBranch.End {
				delete(b.Branches, entry[leafLen])
				// dangling leaf value?
				if len(b.Branches) == 0 && b.Count == 0 {
					b.LeafValue = nil
				}
			}
			return deleted
		}
	}

	return false
}

/*
 */
func (b *Branch) has(entry []byte) bool {
	if b.getBranch(entry) != nil {
		return true
	}
	return false
}

func (b *Branch) hasCount(entry []byte) (bool, int64) {
	br := b.getBranch(entry)
	if br != nil {
		return true, br.Count
	}
	return false, 0
}

func (b *Branch) getBranch(entry []byte) (be *Branch) {
	leafLen := len(b.LeafValue)
	entryLen := len(entry)

	if entryLen >= leafLen {
		for i, pb := range b.LeafValue {
			if pb != entry[i] {
				return
			}
		}
	} else {
		return
	}

	if entryLen > leafLen {
		if br, present := b.Branches[entry[leafLen]]; present {
			return br.getBranch(entry[leafLen+1:])
		} else {
			return
		}
	} else if entryLen == leafLen && b.End {
		be = b
	}
	return
}

/*
 */
func (b *Branch) hasPrefix(prefix []byte) bool {
	exists, _ := b.hasPrefixBranch(prefix)
	return exists
}

func (b *Branch) hasPrefixCount(prefix []byte) (exists bool, count int64) {
	exists, br := b.hasPrefixBranch(prefix)
	if exists {
		count = br.sumCount()
	}
	return
}

func (b *Branch) hasPrefixBranch(prefix []byte) (exists bool, branch *Branch) {
	leafLen := len(b.LeafValue)
	prefixLen := len(prefix)
	exists = false

	if leafLen > 0 {
		if prefixLen <= leafLen {
			for i, pb := range prefix {
				if pb != b.LeafValue[i] {
					return
				}
			}
		} else {
			for i, lb := range b.LeafValue {
				if prefix[i] != lb {
					return
				}
			}
		}
	}

	if prefixLen > leafLen {
		if br, present := b.Branches[prefix[leafLen]]; present {
			return br.hasPrefixBranch(prefix[leafLen+1:])
		} else {
			return
		}
	}

	return true, b
}

func (b *Branch) sumCount() (count int64) {
	// leaf itself matches
	if b.End {
		count += b.Count
	}
	for _, br := range b.Branches {
		count += br.sumCount()
	}
	return
}

/*
 */
func (b *Branch) Dump(depth int) (out string) {
	if len(b.LeafValue) > 0 {
		if b.End {
			out += fmt.Sprintf("%s V:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth), string(b.LeafValue), b.Count)
		} else {
			out += fmt.Sprintf("%s V:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth), string(b.LeafValue), "-")
		}
	}

	if b.End {
		out += fmt.Sprintf("%s $\n", strings.Repeat(PADDING_CHAR, depth+len(b.LeafValue)))
	}

	for idx, branch := range b.Branches {
		if branch.End && len(branch.LeafValue) == 0 {
			out += fmt.Sprintf("%s I:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth+len(b.LeafValue)), string(idx), branch.Count)
		} else {
			out += fmt.Sprintf("%s I:%v (%v)\n", strings.Repeat(PADDING_CHAR, depth+len(b.LeafValue)), string(idx), "-")
		}
		out += branch.Dump(depth + len(b.LeafValue) + 1)
	}

	return
}

/*
 */
// func (b *Branch) hasBranches() bool {
// 	return len(b.Branches) == 0
// }

/*
 */
func (b *Branch) hasBranch(idx byte) bool {
	if _, present := b.Branches[idx]; present {
		return true
	}
	return false
}

/*
 */
// func (b *Branch) matchesLeaf(entry []byte) bool {
// 	leafLen := len(b.LeafValue)
// 	entryLen := len(entry)

// 	if leafLen == 0 && entryLen == 0 {
// 		return true
// 	}

// 	if leafLen == entryLen {
// 		for i, lb := range b.LeafValue {
// 			if entry[i] != lb {
// 				return false
// 			}
// 		}
// 	}
// 	return true
// }

/*
 */
func (b *Branch) pullUp() *Branch {
	if len(b.Branches) == 1 {
		for k, nextBranch := range b.Branches {
			if len(nextBranch.Branches) == 0 {
				b.LeafValue = append(b.LeafValue, append([]byte{k}, nextBranch.LeafValue...)...)
			} else {
				b.LeafValue = append(b.LeafValue, k)
			}
			b.End = nextBranch.End
			b.Branches = nextBranch.Branches
			b.Count = nextBranch.Count
		}
		return b.pullUp()
	}
	return b
}

func (b *Branch) setEnd(flag bool) {
	if flag {
		b.Count += 1
	} else {
		if b.End && b.Count > 0 {
			b.Count -= 1
			if b.Count > 0 {
				return
			}
		}
	}
	b.End = flag
	return
}

func (b *Branch) String() string {
	return b.Dump(0)
}

func (b *Branch) PrintDump() {
	fmt.Printf("\n%s\n\n", b)
}
<|endoftext|>


<commit_before>package manager

import (
	"net/http"
	"regexp"
	"strconv"
	"io"
	"fmt"
//"encoding/json"
	"github.com/030io/whalefs/volume"
	"gopkg.in/redis.v2"
)

var (
	postVolumeUrl *regexp.Regexp
	postFileUrl *regexp.Regexp
	deleteFileUrl  *regexp.Regexp
)

func init() {
	var err error
	postVolumeUrl, err = regexp.Compile("/([0-9]*)/")
	if err != nil {
		panic(err)
	}

	postFileUrl, err = regexp.Compile("/([0-9]*)/([0-9]*)/(.*)")
	if err != nil {
		panic(err)
	}

	deleteFileUrl = postFileUrl
}

type Size interface {
	Size() int64
}

func (vm *VolumeManager)adminEntry(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case http.MethodGet, http.MethodHead:
		vm.publicEntry(w, r)
	case http.MethodPost:
		if postFileUrl.MatchString(r.URL.Path) {
			vm.adminPostFile(w, r)
		}else if postVolumeUrl.MatchString(r.URL.Path) {
			vm.adminPostVolume(w, r)
		}else {
			http.Error(w, r.URL.String() + " can't match", http.StatusNotFound)
		}
	case http.MethodDelete:
		if deleteFileUrl.MatchString(r.URL.Path) {
			vm.adminDeleteFile(w, r)
		}else {
			http.Error(w, r.URL.String() + " can't match", http.StatusNotFound)
		}
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func (vm *VolumeManager)adminPostVolume(w http.ResponseWriter, r *http.Request) {
	match := postVolumeUrl.FindStringSubmatch(r.URL.Path)
	vid, _ := strconv.ParseUint(match[1], 10, 64)
	volume, err := volume.NewVolume(vm.DataDir, vid)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	vm.Volumes[vid] = volume
	w.WriteHeader(http.StatusCreated)
}

func (vm *VolumeManager)adminPostFile(w http.ResponseWriter, r *http.Request) {
	match := postFileUrl.FindStringSubmatch(r.URL.Path)
	vid, _ := strconv.ParseUint(match[1], 10, 64)
	volume, ok := vm.Volumes[vid]
	if !ok {
		http.Error(w, "can't find volume", http.StatusNotFound)
		return
	}

	file, header, err := r.FormFile("file")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer file.Close()

	fileSize := file.(Size).Size()

	fid, _ := strconv.ParseUint(match[2], 10, 64)
	file_, err := volume.NewFile(fid, header.Filename, uint64(fileSize))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer func() {
		if err != nil {
			if err := volume.Delete(file_.Info.Fid, file_.Info.FileName); err != nil {
				panic(err)
			}
		}
	}()

	n, err := io.Copy(file_, file)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}else if n != fileSize {
		err = fmt.Errorf("%d != %d", n, fileSize)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}

func (vm *VolumeManager)adminDeleteFile(w http.ResponseWriter, r *http.Request) {
	match := deleteFileUrl.FindStringSubmatch(r.URL.Path)
	vid, _ := strconv.ParseUint(match[1], 10, 64)
	volume := vm.Volumes[vid]
	if volume == nil {
		http.Error(w, "can't find volume", http.StatusNotFound)
		return
	}

	fid, _ := strconv.ParseUint(match[2], 10, 64)
	err := volume.Delete(fid, match[3])
	if err == redis.Nil {
		http.NotFound(w, r)
	}else if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}else {
		w.WriteHeader(http.StatusAccepted)
	}
}
<commit_msg>修复 大文件上传volume崩溃<commit_after>package manager

import (
	"net/http"
	"regexp"
	"strconv"
	"io"
	"fmt"
	"github.com/030io/whalefs/volume"
	"gopkg.in/redis.v2"
	"os"
)

var (
	postVolumeUrl *regexp.Regexp
	postFileUrl *regexp.Regexp
	deleteFileUrl  *regexp.Regexp
)

func init() {
	var err error
	postVolumeUrl, err = regexp.Compile("/([0-9]*)/")
	if err != nil {
		panic(err)
	}

	postFileUrl, err = regexp.Compile("/([0-9]*)/([0-9]*)/(.*)")
	if err != nil {
		panic(err)
	}

	deleteFileUrl = postFileUrl
}

type Size interface {
	Size() int64
}

func (vm *VolumeManager)adminEntry(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case http.MethodGet, http.MethodHead:
		vm.publicEntry(w, r)
	case http.MethodPost:
		if postFileUrl.MatchString(r.URL.Path) {
			vm.adminPostFile(w, r)
		} else if postVolumeUrl.MatchString(r.URL.Path) {
			vm.adminPostVolume(w, r)
		} else {
			http.Error(w, r.URL.String() + " can't match", http.StatusNotFound)
		}
	case http.MethodDelete:
		if deleteFileUrl.MatchString(r.URL.Path) {
			vm.adminDeleteFile(w, r)
		} else {
			http.Error(w, r.URL.String() + " can't match", http.StatusNotFound)
		}
	default:
		w.WriteHeader(http.StatusMethodNotAllowed)
	}
}

func (vm *VolumeManager)adminPostVolume(w http.ResponseWriter, r *http.Request) {
	match := postVolumeUrl.FindStringSubmatch(r.URL.Path)
	vid, _ := strconv.ParseUint(match[1], 10, 64)
	volume, err := volume.NewVolume(vm.DataDir, vid)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	vm.Volumes[vid] = volume
	w.WriteHeader(http.StatusCreated)
}

func (vm *VolumeManager)adminPostFile(w http.ResponseWriter, r *http.Request) {
	match := postFileUrl.FindStringSubmatch(r.URL.Path)
	vid, _ := strconv.ParseUint(match[1], 10, 64)
	volume, ok := vm.Volumes[vid]
	if !ok {
		http.Error(w, "can't find volume", http.StatusNotFound)
		return
	}

	file, header, err := r.FormFile("file")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer file.Close()

	var fileSize int64
	switch file.(type) {
	case *os.File:
		s, _ := file.(*os.File).Stat()
		fileSize = s.Size()
	case Size:
		fileSize = file.(Size).Size()
	}

	fid, _ := strconv.ParseUint(match[2], 10, 64)
	file_, err := volume.NewFile(fid, header.Filename, uint64(fileSize))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer func() {
		if err != nil {
			if err := volume.Delete(file_.Info.Fid, file_.Info.FileName); err != nil {
				panic(err)
			}
		}
	}()

	n, err := io.Copy(file_, file)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else if n != fileSize {
		err = fmt.Errorf("%d != %d", n, fileSize)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}

func (vm *VolumeManager)adminDeleteFile(w http.ResponseWriter, r *http.Request) {
	match := deleteFileUrl.FindStringSubmatch(r.URL.Path)
	vid, _ := strconv.ParseUint(match[1], 10, 64)
	volume := vm.Volumes[vid]
	if volume == nil {
		http.Error(w, "can't find volume", http.StatusNotFound)
		return
	}

	fid, _ := strconv.ParseUint(match[2], 10, 64)
	err := volume.Delete(fid, match[3])
	if err == redis.Nil {
		http.NotFound(w, r)
	} else if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		w.WriteHeader(http.StatusAccepted)
	}
}
<|endoftext|>


<commit_before>package templates

import (
	"os"
	"text/template"
)

func ParseManifestFile(inputFile string, outputFile string, settings map[string]string) error {

	t, err := template.ParseFiles(inputFile)
	if err != nil {
		return err
	}

	f, err := os.Create(outputFile)
	defer f.Close()
	if err != nil {
		return err
	}

	err = t.Execute(f, settings)

	if err != nil {
		return err
	}

	return nil
}
<commit_msg>add ToLower to templates<commit_after>package templates

import (
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

func ParseManifestFile(inputFile string, outputFile string, settings map[string]string) error {
	funcMap := template.FuncMap{
		"ToUpper": strings.ToUpper,
		"ToLower": strings.ToLower,
	}

	t, err := template.
		New(filepath.Base(inputFile)).
		Funcs(funcMap).
		ParseFiles(inputFile)
	if err != nil {
		return err
	}

	f, err := os.Create(outputFile)
	defer f.Close()
	if err != nil {
		return err
	}

	err = t.Execute(f, settings)

	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package clw11

/*
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif

extern void bufferCallback(cl_mem memobj, void *user_data);

void callBufferCallback(cl_mem memobj, void *user_data)
{
	bufferCallback(memobj, user_data);
}
*/
import "C"
import (
	"unsafe"
)

type (
	Mem              C.cl_mem
	MemFlags         C.cl_mem_flags
	MemInfo          C.cl_mem_info
	MapFlags         C.cl_map_flags
	BufferRegion     C.cl_buffer_region
	BufferCreateType C.cl_buffer_create_type
)

// Bitfield.
const (
	MemReadWrite        MemFlags = C.CL_MEM_READ_WRITE
	MemWriteOnly        MemFlags = C.CL_MEM_WRITE_ONLY
	MemReadOnly         MemFlags = C.CL_MEM_READ_ONLY
	MemUseHostPointer   MemFlags = C.CL_MEM_USE_HOST_PTR
	MemAllocHostPointer MemFlags = C.CL_MEM_ALLOC_HOST_PTR
	MemCopyHostPointer  MemFlags = C.CL_MEM_COPY_HOST_PTR
)

// Bitfield.
const (
	MapRead  MapFlags = C.CL_MAP_READ
	MapWrite MapFlags = C.CL_MAP_WRITE
)

const (
	BufferCreateTypeRegion BufferCreateType = C.CL_BUFFER_CREATE_TYPE_REGION
)

const (
	MemType                MemInfo = C.CL_MEM_TYPE
	MemFlagsInfo           MemInfo = C.CL_MEM_FLAGS // Appended "Info" due to conflict with type.
	MemSize                MemInfo = C.CL_MEM_SIZE
	MemHostPtr             MemInfo = C.CL_MEM_HOST_PTR
	MemMapCount            MemInfo = C.CL_MEM_MAP_COUNT
	MemReferenceCount      MemInfo = C.CL_MEM_REFERENCE_COUNT
	MemContext             MemInfo = C.CL_MEM_CONTEXT
	MemAssociatedMemobject MemInfo = C.CL_MEM_ASSOCIATED_MEMOBJECT
	MemOffset              MemInfo = C.CL_MEM_OFFSET
)

// Creates a buffer object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clCreateBuffer.html
func CreateBuffer(context Context, flags MemFlags, size Size, host_ptr unsafe.Pointer) (Mem, error) {

	var err C.cl_int
	memory := C.clCreateBuffer(context, C.cl_mem_flags(flags), C.size_t(size), host_ptr, &err)

	return Mem(memory), toError(err)
}

// Creates a buffer object (referred to as a sub-buffer object) from an existing
// buffer object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clCreateSubBuffer.html
func CreateSubBuffer(buffer Mem, flags MemFlags, buffer_create_type BufferCreateType, buffer_create_info unsafe.Pointer,
	errcode_ret *Int) (Mem, error) {

	var err C.cl_int
	memory := C.clCreateSubBuffer(buffer, C.cl_mem_flags(flags), C.cl_buffer_create_type(buffer_create_type),
		buffer_create_info, &err)

	return Mem(memory), toError(err)
}

// Increments the memory object reference count.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clRetainMemObject.html
func RetainMemObject(memobj Mem) error {
	return toError(C.clRetainMemObject(memobj))
}

// Decrements the memory object reference count.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clReleaseMemObject.html
func ReleaseMemObject(memobj Mem) error {
	return toError(C.clReleaseMemObject(memobj))
}

// Enqueue commands to read from a buffer object to host memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReadBuffer.html
func EnqueueReadBuffer(command_queue CommandQueue, buffer Mem, blocking_read Bool, offset, cb Size,
	ptr unsafe.Pointer, wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueReadBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueue commands to write to a buffer object from host memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueWriteBuffer.html
func EnqueueWriteBuffer(command_queue CommandQueue, buffer Mem, blocking_read Bool, offset, cb Size,
	ptr unsafe.Pointer, wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueWriteBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueue commands to read from a rectangular region from a buffer object to
// host memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReadBufferRect.html
func EnqueueReadBufferRect(command_queue CommandQueue, buffer Mem, blocking_read Bool, buffer_origin, host_origin,
	region [3]Size, buffer_row_pitch, buffer_slice_pitch, host_row_pitch, host_slice_pitch Size, ptr unsafe.Pointer,
	wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueReadBufferRect(command_queue, buffer, C.cl_bool(blocking_read),
		(*C.size_t)(&buffer_origin[0]), (*C.size_t)(&host_origin[0]), (*C.size_t)(&region[0]),
		C.size_t(buffer_row_pitch), C.size_t(buffer_slice_pitch), C.size_t(host_row_pitch), C.size_t(host_slice_pitch),
		ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueue commands to write a rectangular region to a buffer object from host
// memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueWriteBufferRect.html
func EnqueueWriteBufferRect(command_queue CommandQueue, buffer Mem, blocking_read Bool, buffer_origin, host_origin,
	region [3]Size, buffer_row_pitch, buffer_slice_pitch, host_row_pitch, host_slice_pitch Size, ptr unsafe.Pointer,
	wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueWriteBufferRect(command_queue, buffer, C.cl_bool(blocking_read),
		(*C.size_t)(&buffer_origin[0]), (*C.size_t)(&host_origin[0]), (*C.size_t)(&region[0]),
		C.size_t(buffer_row_pitch), C.size_t(buffer_slice_pitch), C.size_t(host_row_pitch), C.size_t(host_slice_pitch),
		ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueues a command to copy from one buffer object to another.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueCopyBuffer.html
func EnqueueCopyBuffer(command_queue CommandQueue, src_buffer, dst_buffer Mem, src_offset, dst_offset, cb Size,
	wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueCopyBuffer(command_queue, src_buffer, dst_buffer, C.size_t(src_offset),
		C.size_t(dst_offset), C.size_t(cb), num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueues a command to copy a rectangular region from the buffer object to
// another buffer object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueCopyBufferRect.html
func EnqueueCopyBufferRect(command_queue CommandQueue, src_buffer, dst_buffer Mem, src_origin, dst_origin,
	region [3]Size, src_row_pitch, src_slice_pitch, dst_row_pitch, dst_slice_pitch Size, wait_list []Event,
	event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueCopyBufferRect(command_queue, src_buffer, dst_buffer, (*C.size_t)(&src_origin[0]),
		(*C.size_t)(&dst_origin[0]), (*C.size_t)(&region[0]), C.size_t(src_row_pitch), C.size_t(src_slice_pitch),
		C.size_t(dst_row_pitch), C.size_t(dst_slice_pitch), num_events_in_wait_list, event_wait_list,
		(*C.cl_event)(event)))
}

// Enqueues a command to map a region of the buffer object given by buffer into
// the host address space and returns a pointer to this mapped region.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueMapBuffer.html
func EnqueueMapBuffer(command_queue CommandQueue, buffer Mem, blocking_map Bool, map_flags MapFlags, offset, cb Size,
	wait_list []Event, event *Event) (unsafe.Pointer, error) {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	var err C.cl_int
	mapped := C.clEnqueueMapBuffer(command_queue, buffer, C.cl_bool(blocking_map), C.cl_map_flags(map_flags),
		C.size_t(offset), C.size_t(cb), num_events_in_wait_list, event_wait_list, (*C.cl_event)(event), &err)

	return mapped, toError(err)
}

// Enqueues a command to unmap a previously mapped region of a memory object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueUnmapMemObject.html
func EnqueueUnmapMemObject(command_queue CommandQueue, memobj Mem, mapped_ptr unsafe.Pointer, wait_list []Event,
	event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueUnmapMemObject(command_queue, memobj, mapped_ptr, num_events_in_wait_list,
		event_wait_list, (*C.cl_event)(event)))
}

func SetMemObjectDestructorCallback(memobj Mem, callback BufferCallbackFunc, user_data interface{}) error {

	key := bufferCallbacks.add(callback, user_data)

	err := toError(C.clSetMemObjectDestructorCallback(C.cl_mem(memobj), (*[0]byte)(C.callBufferCallback),
		unsafe.Pointer(key)))

	if err != nil {
		// If the C side setting of the callback failed GetCallback will remove
		// the callback from the map.
		bufferCallbacks.get(key)
	}

	return err
}

func GetMemObjectInfo(memobj Mem, param_name MemInfo, param_value_size Size, param_value unsafe.Pointer,
	param_value_size_return *Size) error {

	return toError(C.clGetMemObjectInfo(memobj, C.cl_mem_info(param_name), C.size_t(param_value_size), param_value,
		(*C.size_t)(param_value_size_return)))
}
<commit_msg>Added documentation.<commit_after>package clw11

/*
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif

extern void bufferCallback(cl_mem memobj, void *user_data);

void callBufferCallback(cl_mem memobj, void *user_data)
{
	bufferCallback(memobj, user_data);
}
*/
import "C"
import (
	"unsafe"
)

type (
	Mem              C.cl_mem
	MemFlags         C.cl_mem_flags
	MemInfo          C.cl_mem_info
	MapFlags         C.cl_map_flags
	BufferRegion     C.cl_buffer_region
	BufferCreateType C.cl_buffer_create_type
)

// Bitfield.
const (
	MemReadWrite        MemFlags = C.CL_MEM_READ_WRITE
	MemWriteOnly        MemFlags = C.CL_MEM_WRITE_ONLY
	MemReadOnly         MemFlags = C.CL_MEM_READ_ONLY
	MemUseHostPointer   MemFlags = C.CL_MEM_USE_HOST_PTR
	MemAllocHostPointer MemFlags = C.CL_MEM_ALLOC_HOST_PTR
	MemCopyHostPointer  MemFlags = C.CL_MEM_COPY_HOST_PTR
)

// Bitfield.
const (
	MapRead  MapFlags = C.CL_MAP_READ
	MapWrite MapFlags = C.CL_MAP_WRITE
)

const (
	BufferCreateTypeRegion BufferCreateType = C.CL_BUFFER_CREATE_TYPE_REGION
)

const (
	MemType                MemInfo = C.CL_MEM_TYPE
	MemFlagsInfo           MemInfo = C.CL_MEM_FLAGS // Appended "Info" due to conflict with type.
	MemSize                MemInfo = C.CL_MEM_SIZE
	MemHostPtr             MemInfo = C.CL_MEM_HOST_PTR
	MemMapCount            MemInfo = C.CL_MEM_MAP_COUNT
	MemReferenceCount      MemInfo = C.CL_MEM_REFERENCE_COUNT
	MemContext             MemInfo = C.CL_MEM_CONTEXT
	MemAssociatedMemobject MemInfo = C.CL_MEM_ASSOCIATED_MEMOBJECT
	MemOffset              MemInfo = C.CL_MEM_OFFSET
)

// Creates a buffer object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clCreateBuffer.html
func CreateBuffer(context Context, flags MemFlags, size Size, host_ptr unsafe.Pointer) (Mem, error) {

	var err C.cl_int
	memory := C.clCreateBuffer(context, C.cl_mem_flags(flags), C.size_t(size), host_ptr, &err)

	return Mem(memory), toError(err)
}

// Creates a buffer object (referred to as a sub-buffer object) from an existing
// buffer object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clCreateSubBuffer.html
func CreateSubBuffer(buffer Mem, flags MemFlags, buffer_create_type BufferCreateType, buffer_create_info unsafe.Pointer,
	errcode_ret *Int) (Mem, error) {

	var err C.cl_int
	memory := C.clCreateSubBuffer(buffer, C.cl_mem_flags(flags), C.cl_buffer_create_type(buffer_create_type),
		buffer_create_info, &err)

	return Mem(memory), toError(err)
}

// Increments the memory object reference count.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clRetainMemObject.html
func RetainMemObject(memobj Mem) error {
	return toError(C.clRetainMemObject(memobj))
}

// Decrements the memory object reference count.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clReleaseMemObject.html
func ReleaseMemObject(memobj Mem) error {
	return toError(C.clReleaseMemObject(memobj))
}

// Enqueue commands to read from a buffer object to host memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReadBuffer.html
func EnqueueReadBuffer(command_queue CommandQueue, buffer Mem, blocking_read Bool, offset, cb Size,
	ptr unsafe.Pointer, wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueReadBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueue commands to write to a buffer object from host memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueWriteBuffer.html
func EnqueueWriteBuffer(command_queue CommandQueue, buffer Mem, blocking_read Bool, offset, cb Size,
	ptr unsafe.Pointer, wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueWriteBuffer(command_queue, buffer, C.cl_bool(blocking_read), C.size_t(offset),
		C.size_t(cb), ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueue commands to read from a rectangular region from a buffer object to
// host memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReadBufferRect.html
func EnqueueReadBufferRect(command_queue CommandQueue, buffer Mem, blocking_read Bool, buffer_origin, host_origin,
	region [3]Size, buffer_row_pitch, buffer_slice_pitch, host_row_pitch, host_slice_pitch Size, ptr unsafe.Pointer,
	wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueReadBufferRect(command_queue, buffer, C.cl_bool(blocking_read),
		(*C.size_t)(&buffer_origin[0]), (*C.size_t)(&host_origin[0]), (*C.size_t)(&region[0]),
		C.size_t(buffer_row_pitch), C.size_t(buffer_slice_pitch), C.size_t(host_row_pitch), C.size_t(host_slice_pitch),
		ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueue commands to write a rectangular region to a buffer object from host
// memory.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueWriteBufferRect.html
func EnqueueWriteBufferRect(command_queue CommandQueue, buffer Mem, blocking_read Bool, buffer_origin, host_origin,
	region [3]Size, buffer_row_pitch, buffer_slice_pitch, host_row_pitch, host_slice_pitch Size, ptr unsafe.Pointer,
	wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueWriteBufferRect(command_queue, buffer, C.cl_bool(blocking_read),
		(*C.size_t)(&buffer_origin[0]), (*C.size_t)(&host_origin[0]), (*C.size_t)(&region[0]),
		C.size_t(buffer_row_pitch), C.size_t(buffer_slice_pitch), C.size_t(host_row_pitch), C.size_t(host_slice_pitch),
		ptr, num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueues a command to copy from one buffer object to another.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueCopyBuffer.html
func EnqueueCopyBuffer(command_queue CommandQueue, src_buffer, dst_buffer Mem, src_offset, dst_offset, cb Size,
	wait_list []Event, event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueCopyBuffer(command_queue, src_buffer, dst_buffer, C.size_t(src_offset),
		C.size_t(dst_offset), C.size_t(cb), num_events_in_wait_list, event_wait_list, (*C.cl_event)(event)))
}

// Enqueues a command to copy a rectangular region from the buffer object to
// another buffer object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueCopyBufferRect.html
func EnqueueCopyBufferRect(command_queue CommandQueue, src_buffer, dst_buffer Mem, src_origin, dst_origin,
	region [3]Size, src_row_pitch, src_slice_pitch, dst_row_pitch, dst_slice_pitch Size, wait_list []Event,
	event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueCopyBufferRect(command_queue, src_buffer, dst_buffer, (*C.size_t)(&src_origin[0]),
		(*C.size_t)(&dst_origin[0]), (*C.size_t)(&region[0]), C.size_t(src_row_pitch), C.size_t(src_slice_pitch),
		C.size_t(dst_row_pitch), C.size_t(dst_slice_pitch), num_events_in_wait_list, event_wait_list,
		(*C.cl_event)(event)))
}

// Enqueues a command to map a region of the buffer object given by buffer into
// the host address space and returns a pointer to this mapped region.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueMapBuffer.html
func EnqueueMapBuffer(command_queue CommandQueue, buffer Mem, blocking_map Bool, map_flags MapFlags, offset, cb Size,
	wait_list []Event, event *Event) (unsafe.Pointer, error) {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	var err C.cl_int
	mapped := C.clEnqueueMapBuffer(command_queue, buffer, C.cl_bool(blocking_map), C.cl_map_flags(map_flags),
		C.size_t(offset), C.size_t(cb), num_events_in_wait_list, event_wait_list, (*C.cl_event)(event), &err)

	return mapped, toError(err)
}

// Enqueues a command to unmap a previously mapped region of a memory object.
// http://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueUnmapMemObject.html
func EnqueueUnmapMemObject(command_queue CommandQueue, memobj Mem, mapped_ptr unsafe.Pointer, wait_list []Event,
	event *Event) error {

	event_wait_list, num_events_in_wait_list := toEventList(wait_list)

	return toError(C.clEnqueueUnmapMemObject(command_queue, memobj, mapped_ptr, num_events_in_wait_list,
		event_wait_list, (*C.cl_event)(event)))
}

// Registers a user callback function that will be called when the memory object
// is deleted and its resources freed.
// https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clSetMemObjectDestructorCallback.html
func SetMemObjectDestructorCallback(memobj Mem, callback BufferCallbackFunc, user_data interface{}) error {

	key := bufferCallbacks.add(callback, user_data)

	err := toError(C.clSetMemObjectDestructorCallback(C.cl_mem(memobj), (*[0]byte)(C.callBufferCallback),
		unsafe.Pointer(key)))

	if err != nil {
		// If the C side setting of the callback failed GetCallback will remove
		// the callback from the map.
		bufferCallbacks.get(key)
	}

	return err
}

// Used to get information that is common to all memory objects (buffer and
// image objects).
// https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clGetMemObjectInfo.html
func GetMemObjectInfo(memobj Mem, param_name MemInfo, param_value_size Size, param_value unsafe.Pointer,
	param_value_size_return *Size) error {

	return toError(C.clGetMemObjectInfo(memobj, C.cl_mem_info(param_name), C.size_t(param_value_size), param_value,
		(*C.size_t)(param_value_size_return)))
}
<|endoftext|>


<commit_before>package indicators

import (
	"container/list"
	"errors"
	"github.com/thetruetrade/gotrade"
	"math"
)

// A Lowest Low Value Bars Indicator (LlvBars), no storage, for use in other indicators
type LlvBarsWithoutStorage struct {
	*baseIndicator
	*baseIntBounds

	// private variables
	periodHistory        *list.List
	valueAvailableAction ValueAvailableActionInt
	currentLow           float64
	currentLowIndex      int64
	timePeriod           int
}

// NewLlvBarsWithoutStorage creates a Lowest Low Value Bars Indicator Indicator (LlvBars) without storage
func NewLlvBarsWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionInt) (indicator *LlvBarsWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := timePeriod - 1

	ind := LlvBarsWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseIntBounds:        newBaseIntBounds(),
		currentLow:           math.MaxFloat64,
		currentLowIndex:      0,
		periodHistory:        list.New(),
		valueAvailableAction: valueAvailableAction,
		timePeriod:           timePeriod,
	}

	return &ind, nil
}

// A Lowest Low Value Bars Indicator (LlvBars)
type LlvBars struct {
	*LlvBarsWithoutStorage
	selectData gotrade.DataSelectionFunc

	// public variables
	Data []int64
}

// NewLlvBars creates a Lowest Low Value Bars Indicator (LlvBars) for online usage
func NewLlvBars(timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind := LlvBars{selectData: selectData}
	ind.LlvBarsWithoutStorage, err = NewLlvBarsWithoutStorage(timePeriod, func(dataItem int64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultLlvBars creates a Lowest Low Value Indicator (LlvBars) for online usage with default parameters
//	- timePeriod: 25
func NewDefaultLlvBars() (indicator *LlvBars, err error) {
	timePeriod := 25
	return NewLlvBars(timePeriod, gotrade.UseClosePrice)
}

// NewLlvBarsWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage
func NewLlvBarsWithSrcLen(sourceLength uint, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind, err := NewLlvBars(timePeriod, selectData)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]int64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultLlvBarsWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage with default parameters
func NewDefaultLlvBarsWithSrcLen(sourceLength uint) (indicator *LlvBars, err error) {
	ind, err := NewDefaultLlvBars()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]int64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewLlvBarsForStream creates a Lowest Low Value Indicator (LlvBars)for online usage with a source data stream
func NewLlvBarsForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind, err := NewLlvBars(timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLlvBarsForStream creates a Lowest Low Value Indicator (LlvBars)for online usage with a source data stream
func NewDefaultLlvBarsForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LlvBars, err error) {
	ind, err := NewDefaultLlvBars()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewLlvBarsForStreamWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage with a source data stream
func NewLlvBarsForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind, err := NewLlvBarsWithSrcLen(sourceLength, timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLlvBarsForStreamWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage with a source data stream
func NewDefaultLlvBarsForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LlvBars, err error) {
	ind, err := NewDefaultLlvBarsWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *LlvBars) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	var selectedData = ind.selectData(tickData)
	ind.ReceiveTick(selectedData, streamBarIndex)
}

func (ind *LlvBarsWithoutStorage) ReceiveTick(tickData float64, streamBarIndex int) {
	ind.periodHistory.PushBack(tickData)

	// resize the history
	if ind.periodHistory.Len() > ind.timePeriod {
		first := ind.periodHistory.Front()
		ind.periodHistory.Remove(first)

		// make sure we haven't just removed the current low
		if ind.currentLowIndex == int64(ind.timePeriod-1) {
			ind.currentLow = math.MaxFloat64
			// we have we need to find the new low in the history
			var i int = ind.timePeriod - 1
			for e := ind.periodHistory.Front(); e != nil; e = e.Next() {
				value := e.Value.(float64)
				if value < ind.currentLow {
					ind.currentLow = value
					ind.currentLowIndex = int64(i)
				}
				i -= 1
			}
		} else {
			if tickData < ind.currentLow {
				ind.currentLow = tickData
				ind.currentLowIndex = 0
			} else {
				ind.currentLowIndex += 1
			}
		}

		var result = ind.currentLowIndex

		// increment the number of results this indicator can be expected to return
		ind.dataLength += 1

		if ind.validFromBar == -1 {
			// set the streamBarIndex from which this indicator returns valid results
			ind.validFromBar = streamBarIndex
		}

		// update the maximum result value
		if result > ind.maxValue {
			ind.maxValue = result
		}

		// update the minimum result value
		if result < ind.minValue {
			ind.minValue = result
		}

		// notify of a new result value though the value available action
		ind.valueAvailableAction(result, streamBarIndex)

	} else {
		if tickData < ind.currentLow {
			ind.currentLow = tickData
			ind.currentLowIndex = 0
		} else {
			ind.currentLowIndex += 1
		}

		if ind.periodHistory.Len() == ind.timePeriod {
			var result = ind.currentLowIndex

			// increment the number of results this indicator can be expected to return
			ind.dataLength += 1

			if ind.validFromBar == -1 {
				// set the streamBarIndex from which this indicator returns valid results
				ind.validFromBar = streamBarIndex
			}

			// update the maximum result value
			if result > ind.maxValue {
				ind.maxValue = result
			}

			// update the minimum result value
			if result < ind.minValue {
				ind.minValue = result
			}

			// notify of a new result value though the value available action
			ind.valueAvailableAction(result, streamBarIndex)
		}
	}

}
<commit_msg>#76 Remove duplication - llvbars<commit_after>package indicators

import (
	"container/list"
	"errors"
	"github.com/thetruetrade/gotrade"
	"math"
)

// A Lowest Low Value Bars Indicator (LlvBars), no storage, for use in other indicators
type LlvBarsWithoutStorage struct {
	*baseIndicatorWithIntBounds

	// private variables
	periodHistory   *list.List
	currentLow      float64
	currentLowIndex int64
	timePeriod      int
}

// NewLlvBarsWithoutStorage creates a Lowest Low Value Bars Indicator Indicator (LlvBars) without storage
func NewLlvBarsWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionInt) (indicator *LlvBarsWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := timePeriod - 1

	ind := LlvBarsWithoutStorage{
		baseIndicatorWithIntBounds: newBaseIndicatorWithIntBounds(lookback, valueAvailableAction),
		currentLow:                 math.MaxFloat64,
		currentLowIndex:            0,
		periodHistory:              list.New(),
		timePeriod:                 timePeriod,
	}

	return &ind, nil
}

// A Lowest Low Value Bars Indicator (LlvBars)
type LlvBars struct {
	*LlvBarsWithoutStorage
	selectData gotrade.DataSelectionFunc

	// public variables
	Data []int64
}

// NewLlvBars creates a Lowest Low Value Bars Indicator (LlvBars) for online usage
func NewLlvBars(timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind := LlvBars{selectData: selectData}
	ind.LlvBarsWithoutStorage, err = NewLlvBarsWithoutStorage(timePeriod, func(dataItem int64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultLlvBars creates a Lowest Low Value Indicator (LlvBars) for online usage with default parameters
//	- timePeriod: 25
func NewDefaultLlvBars() (indicator *LlvBars, err error) {
	timePeriod := 25
	return NewLlvBars(timePeriod, gotrade.UseClosePrice)
}

// NewLlvBarsWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage
func NewLlvBarsWithSrcLen(sourceLength uint, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind, err := NewLlvBars(timePeriod, selectData)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]int64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultLlvBarsWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage with default parameters
func NewDefaultLlvBarsWithSrcLen(sourceLength uint) (indicator *LlvBars, err error) {
	ind, err := NewDefaultLlvBars()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]int64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewLlvBarsForStream creates a Lowest Low Value Indicator (LlvBars)for online usage with a source data stream
func NewLlvBarsForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind, err := NewLlvBars(timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLlvBarsForStream creates a Lowest Low Value Indicator (LlvBars)for online usage with a source data stream
func NewDefaultLlvBarsForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LlvBars, err error) {
	ind, err := NewDefaultLlvBars()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewLlvBarsForStreamWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage with a source data stream
func NewLlvBarsForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LlvBars, err error) {
	ind, err := NewLlvBarsWithSrcLen(sourceLength, timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLlvBarsForStreamWithSrcLen creates a Lowest Low Value Indicator (LlvBars)for offline usage with a source data stream
func NewDefaultLlvBarsForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LlvBars, err error) {
	ind, err := NewDefaultLlvBarsWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *LlvBars) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	var selectedData = ind.selectData(tickData)
	ind.ReceiveTick(selectedData, streamBarIndex)
}

func (ind *LlvBarsWithoutStorage) ReceiveTick(tickData float64, streamBarIndex int) {
	ind.periodHistory.PushBack(tickData)

	// resize the history
	if ind.periodHistory.Len() > ind.timePeriod {
		first := ind.periodHistory.Front()
		ind.periodHistory.Remove(first)

		// make sure we haven't just removed the current low
		if ind.currentLowIndex == int64(ind.timePeriod-1) {
			ind.currentLow = math.MaxFloat64
			// we have we need to find the new low in the history
			var i int = ind.timePeriod - 1
			for e := ind.periodHistory.Front(); e != nil; e = e.Next() {
				value := e.Value.(float64)
				if value < ind.currentLow {
					ind.currentLow = value
					ind.currentLowIndex = int64(i)
				}
				i -= 1
			}
		} else {
			if tickData < ind.currentLow {
				ind.currentLow = tickData
				ind.currentLowIndex = 0
			} else {
				ind.currentLowIndex += 1
			}
		}

		var result = ind.currentLowIndex

		ind.UpdateIndicatorWithNewValue(result, streamBarIndex)

	} else {
		if tickData < ind.currentLow {
			ind.currentLow = tickData
			ind.currentLowIndex = 0
		} else {
			ind.currentLowIndex += 1
		}

		if ind.periodHistory.Len() == ind.timePeriod {
			var result = ind.currentLowIndex

			ind.UpdateIndicatorWithNewValue(result, streamBarIndex)
		}
	}

}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
)

type DbCfg struct {
	Database string `json:"database"`
	User     string `json:"user"`
	Password string `json:"password"`
	Host     string `json:"host"`
	Port     string `json:"port"`
}

type AnalyzerCfg struct {
	Db DbCfg `json:"db"`
}

type Config struct {
	AppDir    string      `json:"appdir"`
	UnpackDir string      `json:"unpackdir"`
	SockPath  string      `json:"sockpath"`
	Analyzer  AnalyzerCfg `json:"analyzer"`
	Db        DbCfg       `json:"-"`
}

var UnpackDir string

func Load(cfgFile string) Config {
	file, err := os.Open(cfgFile)
	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		panic("Couldn't read config file " + cfgFile)
	}
	var cfg Config
	err = json.Unmarshal(bytes, &cfg)
	if err != nil {
		panic("Error reading JSON: " + err.Error())
	}

	if cfg.AppDir == "" {
		cfg.AppDir = "./apks"
	}
	if cfg.UnpackDir == "" {
		cfg.UnpackDir = "."
	}
	if cfg.SockPath == "" {
		cfg.SockPath = "/run/apkScraper"
	}

	cfg.AppDir = path.Clean(cfg.AppDir)
	cfg.UnpackDir = path.Clean(cfg.UnpackDir)
	cfg.SockPath = path.Clean(cfg.SockPath)

	fmt.Println("Config:")
	fmt.Println("\tApp directory:", cfg.AppDir)
	fmt.Println("\tUnpacked app directory:", cfg.UnpackDir)
	fmt.Println("\tMessage socket path:", cfg.SockPath)

	return cfg
}
<commit_msg>update to use new config file structure<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
)

type DbCfg struct {
	Database string `json:"database"`
	User     string `json:"user"`
	Password string `json:"password"`
	Host     string `json:"host"`
	Port     string `json:"port"`
}

type AnalyzerCfg struct {
	Db DbCfg `json:"db"`
}

type Config struct {
	DataDir           string      `json:"datadir"`
	AppDir, UnpackDir string      `json:"-"`
	SockPath          string      `json:"sockpath"`
	Analyzer          AnalyzerCfg `json:"analyzer"`
	Db                DbCfg       `json:"-"`
}

var UnpackDir string

func Load(cfgFile string) Config {
	file, err := os.Open(cfgFile)
	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		panic("Couldn't read config file " + cfgFile)
	}
	var cfg Config
	err = json.Unmarshal(bytes, &cfg)
	if err != nil {
		panic("Error reading JSON: " + err.Error())
	}

	if cfg.DataDir == "" {
		cfg.DataDir = "/usr/local/var/xray"
	}
	cfg.AppDir = path.Join(cfg.DataDir, "apk_archive")
	cfg.UnpackDir = path.Join(cfg.DataDir, "apk_unpacked")
	if cfg.SockPath == "" {
		cfg.SockPath = "/var/run/apkScraper"
	}

	cfg.AppDir = path.Clean(cfg.AppDir)
	cfg.UnpackDir = path.Clean(cfg.UnpackDir)
	cfg.SockPath = path.Clean(cfg.SockPath)

	fmt.Println("Config:")
	fmt.Println("\tApp directory:", cfg.AppDir)
	fmt.Println("\tUnpacked app directory:", cfg.UnpackDir)
	fmt.Println("\tMessage socket path:", cfg.SockPath)

	return cfg
}
<|endoftext|>


<commit_before>package handlers

import (
	"github.com/go-openapi/runtime/middleware"
	apierrors "k8s.io/apimachinery/pkg/api/errors"

	"github.com/sapcc/kubernikus/pkg/api"
	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/api/rest/operations"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
)

func NewUpdateCluster(rt *api.Runtime) operations.UpdateClusterHandler {
	return &updateCluster{rt}
}

type updateCluster struct {
	*api.Runtime
}

func (d *updateCluster) Handle(params operations.UpdateClusterParams, principal *models.Principal) middleware.Responder {

	kluster, err := editCluster(d.Kubernikus.Kubernikus().Klusters(d.Namespace), principal, params.Name, func(kluster *v1.Kluster) {
		nodePools := params.Body.Spec.NodePools
		//set default image
		for i, pool := range nodePools {
			if pool.Image == "" {
				nodePools[i].Image = DEFAULT_IMAGE
			}
		}
		// Update nodepool
		kluster.Spec.NodePools = nodePools
		kluster.Spec.SSHPublicKey = params.Body.Spec.SSHPublicKey
		kluster.Spec.Openstack.SecurityGroupName = params.Body.Spec.Openstack.SecurityGroupName
	})
	if err != nil {
		if apierrors.IsNotFound(err) {
			return NewErrorResponse(&operations.UpdateClusterDefault{}, 404, "Not found")
		}
		return NewErrorResponse(&operations.UpdateClusterDefault{}, 500, err.Error())
	}
	return operations.NewUpdateClusterOK().WithPayload(klusterFromCRD(kluster))
}
<commit_msg>fixes nodepool update overwriting securitygroup<commit_after>package handlers

import (
	"github.com/go-openapi/runtime/middleware"
	apierrors "k8s.io/apimachinery/pkg/api/errors"

	"github.com/sapcc/kubernikus/pkg/api"
	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/api/rest/operations"
	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
)

func NewUpdateCluster(rt *api.Runtime) operations.UpdateClusterHandler {
	return &updateCluster{rt}
}

type updateCluster struct {
	*api.Runtime
}

func (d *updateCluster) Handle(params operations.UpdateClusterParams, principal *models.Principal) middleware.Responder {

	kluster, err := editCluster(d.Kubernikus.Kubernikus().Klusters(d.Namespace), principal, params.Name, func(kluster *v1.Kluster) {
		nodePools := params.Body.Spec.NodePools
		//set default image
		for i, pool := range nodePools {
			if pool.Image == "" {
				nodePools[i].Image = DEFAULT_IMAGE
			}
		}
		// Update nodepool
		kluster.Spec.NodePools = nodePools
		kluster.Spec.SSHPublicKey = params.Body.Spec.SSHPublicKey

		if params.Body.Spec.Openstack.SecurityGroupName != "" {
			kluster.Spec.Openstack.SecurityGroupName = params.Body.Spec.Openstack.SecurityGroupName
		}
	})
	if err != nil {
		if apierrors.IsNotFound(err) {
			return NewErrorResponse(&operations.UpdateClusterDefault{}, 404, "Not found")
		}
		return NewErrorResponse(&operations.UpdateClusterDefault{}, 500, err.Error())
	}
	return operations.NewUpdateClusterOK().WithPayload(klusterFromCRD(kluster))
}
<|endoftext|>


<commit_before>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import "time"

const (
	// LostEventSourceUnspec indicates an event has been lost at an unknown
	// source
	LostEventSourceUnspec = iota
	// LostEventSourcePerfRingBuffer indicates an event has been lost because
	// the perf event ring buffer was not read before it was overwritten.
	LostEventSourcePerfRingBuffer
	// LostEventSourceEventsQueue indicates that an event has been dropped
	// because the events queue was full.
	LostEventSourceEventsQueue
)

// MonitorEvent is the top-level type for all events consumed by the observer
type MonitorEvent struct {
	// Timestamp when the event was received by the consumer
	Timestamp time.Time
	// NodeName where the event occurred
	NodeName string
	// Payload is one of: AgentEvent, PerfEvent or LostEvent
	Payload interface{}
}

// AgentEvent is any agent event
type AgentEvent struct {
	// Type is a monitorAPI.MessageType* value
	Type int
	// Message is the agent message, e.g. accesslog.LogRecord, monitorAPI.AgentNotify
	Message interface{}
}

// PerfEvent is a raw event obtained from a BPF perf ring buffer
type PerfEvent struct {
	// Data is the raw data payload of the perf event
	Data []byte
	// CPU is the cpu number on which the perf event occurred
	CPU int
}

// LostEvent indicates that a number of events were lost at the indicated source
type LostEvent struct {
	// Source is where the events were dropped
	Source int
	// NumLostEvents is the number of events lost
	NumLostEvents uint64
	// CPU is the cpu number if for events lost in the perf ring buffer
	CPU int
}
<commit_msg>hubble/observer/types: fix comment for AgentEvent.Message<commit_after>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import "time"

const (
	// LostEventSourceUnspec indicates an event has been lost at an unknown
	// source
	LostEventSourceUnspec = iota
	// LostEventSourcePerfRingBuffer indicates an event has been lost because
	// the perf event ring buffer was not read before it was overwritten.
	LostEventSourcePerfRingBuffer
	// LostEventSourceEventsQueue indicates that an event has been dropped
	// because the events queue was full.
	LostEventSourceEventsQueue
)

// MonitorEvent is the top-level type for all events consumed by the observer
type MonitorEvent struct {
	// Timestamp when the event was received by the consumer
	Timestamp time.Time
	// NodeName where the event occurred
	NodeName string
	// Payload is one of: AgentEvent, PerfEvent or LostEvent
	Payload interface{}
}

// AgentEvent is any agent event
type AgentEvent struct {
	// Type is a monitorAPI.MessageType* value
	Type int
	// Message is the agent message, e.g. accesslog.LogRecord, monitorAPI.AgentNotifyMessage
	Message interface{}
}

// PerfEvent is a raw event obtained from a BPF perf ring buffer
type PerfEvent struct {
	// Data is the raw data payload of the perf event
	Data []byte
	// CPU is the cpu number on which the perf event occurred
	CPU int
}

// LostEvent indicates that a number of events were lost at the indicated source
type LostEvent struct {
	// Source is where the events were dropped
	Source int
	// NumLostEvents is the number of events lost
	NumLostEvents uint64
	// CPU is the cpu number if for events lost in the perf ring buffer
	CPU int
}
<|endoftext|>


<commit_before>package platform

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"os"

	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/drud/ddev/pkg/testcommon"
	"github.com/drud/ddev/pkg/util"
	"github.com/drud/drud-go/utils/system"
	"github.com/stretchr/testify/assert"
)

var (
	TestSites = []testcommon.TestSite{
		{
			Name:                          "TestMainPkgDrupal8",
			SourceURL:                     "https://github.com/drud/drupal8/archive/v0.6.0.tar.gz",
			ArchiveInternalExtractionPath: "drupal8-0.6.0/",
			FileURL: "https://github.com/drud/drupal8/releases/download/v0.6.0/files.tar.gz",
			DBURL:   "https://github.com/drud/drupal8/releases/download/v0.6.0/db.tar.gz",
		},
		{
			Name:                          "TestMainPkgWordpress",
			SourceURL:                     "https://github.com/drud/wordpress/archive/v0.4.0.tar.gz",
			ArchiveInternalExtractionPath: "wordpress-0.4.0/",
			FileURL: "https://github.com/drud/wordpress/releases/download/v0.4.0/files.tar.gz",
			DBURL:   "https://github.com/drud/wordpress/releases/download/v0.4.0/db.tar.gz",
		},
		{
			Name:                          "TestMainPkgDrupalKickstart",
			SourceURL:                     "https://github.com/drud/drupal-kickstart/archive/v0.4.0.tar.gz",
			ArchiveInternalExtractionPath: "drupal-kickstart-0.4.0/",
			FileURL: "https://github.com/drud/drupal-kickstart/releases/download/v0.4.0/files.tar.gz",
			DBURL:   "https://github.com/drud/drupal-kickstart/releases/download/v0.4.0/db.tar.gz",
		},
	}
)

func TestMain(m *testing.M) {
	for i := range TestSites {
		err := TestSites[i].Prepare()
		if err != nil {
			log.Fatalf("Prepare() failed on TestSite.Prepare(), err=%v", err)
		}
	}

	log.Debugln("Running tests.")
	testRun := m.Run()

	for i := range TestSites {
		TestSites[i].Cleanup()
	}

	os.Exit(testRun)
}

// TestLocalStart tests the functionality that is called when "ddev start" is executed
func TestLocalStart(t *testing.T) {

	// ensure we have docker network
	client := util.GetDockerClient()
	err := util.EnsureNetwork(client, util.NetName)
	if err != nil {
		log.Fatal(err)
	}

	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalStart", site.Name))

		testcommon.ClearDockerEnv()
		err = app.Init(site.Dir)
		assert.NoError(err)

		err = app.Start()
		assert.NoError(err)

		err = app.Wait("web")
		assert.NoError(err)

		// ensure docker-compose.yaml exists inside .ddev site folder
		composeFile := system.FileExists(app.DockerComposeYAMLPath())
		assert.True(composeFile)

		for _, containerType := range [3]string{"web", "db", "dba"} {
			containerName, err := constructContainerName(containerType, app)
			assert.NoError(err)
			check, err := testcommon.ContainerCheck(containerName, "running")
			assert.NoError(err)
			assert.True(check, containerType, "container is running")
		}

		runTime()
		cleanup()
	}

	// try to start a site of same name at different path
	another := TestSites[0]
	err = another.Prepare()
	if err != nil {
		assert.FailNow("TestLocalStart: Prepare() failed on another.Prepare(), err=%v", err)
		return
	}

	err = app.Init(another.Dir)
	assert.Error(err)
	assert.Contains(err.Error(), fmt.Sprintf("container in running state already exists for %s that was created at %s", TestSites[0].Name, TestSites[0].Dir))
	another.Cleanup()
}

// TestGetApps tests the GetApps function to ensure it accurately returns a list of running applications.
func TestGetApps(t *testing.T) {
	assert := assert.New(t)
	apps := GetApps()
	assert.Equal(len(apps["local"]), len(TestSites))

	for _, site := range TestSites {
		var found bool
		for _, siteInList := range apps["local"] {
			if site.Name == siteInList.GetName() {
				found = true
				break
			}
		}
		assert.True(found, "Found site %s in list", site.Name)
	}
}

// TestLocalImportDB tests the functionality that is called when "ddev import-db" is executed
func TestLocalImportDB(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalImportDB", site.Name))
		dbPath := filepath.Join(testcommon.CreateTmpDir("local-db"), "db.tar.gz")

		err := system.DownloadFile(dbPath, site.DBURL)
		assert.NoError(err)

		testcommon.ClearDockerEnv()
		err = app.Init(site.Dir)
		assert.NoError(err)

		err = app.ImportDB(dbPath)
		assert.NoError(err)

		err = os.Remove(dbPath)
		assert.NoError(err)

		runTime()
		cleanup()
	}
}

// TestLocalImportFiles tests the functionality that is called when "ddev import-files" is executed
func TestLocalImportFiles(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalImportFiles", site.Name))
		filePath := filepath.Join(testcommon.CreateTmpDir("local-files"), "files.tar.gz")

		err := system.DownloadFile(filePath, site.FileURL)
		assert.NoError(err)

		testcommon.ClearDockerEnv()
		err = app.Init(site.Dir)
		assert.NoError(err)

		err = app.ImportFiles(filePath)
		assert.NoError(err)

		err = os.Remove(filePath)
		assert.NoError(err)

		runTime()
		cleanup()
	}
}

// TestLocalExec tests the execution of commands inside a docker container of a site.
func TestLocalExec(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalExec", site.Name))

		err := app.Init(site.Dir)
		assert.NoError(err)

		stdout := testcommon.CaptureStdOut()
		err = app.Exec("web", true, "pwd")
		assert.NoError(err)
		out := stdout()
		assert.Contains(out, "/var/www/html/docroot")

		stdout = testcommon.CaptureStdOut()
		switch app.GetType() {
		case "drupal7":
			fallthrough
		case "drupal8":
			err := app.Exec("web", true, "drush", "status")
			assert.NoError(err)
		case "wordpress":
			err = app.Exec("web", true, "wp", "--info")
			assert.NoError(err)
		default:
		}
		out = stdout()

		assert.Contains(string(out), "/etc/php/7.0/cli/php.ini")

		runTime()
		cleanup()

	}
}

// TestLocalLogs tests the container log output functionality.
func TestLocalLogs(t *testing.T) {
	assert := assert.New(t)

	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalLogs", site.Name))

		err := app.Init(site.Dir)
		assert.NoError(err)

		stdout := testcommon.CaptureStdOut()
		err = app.Logs("web", false, false, "")
		assert.NoError(err)
		out := stdout()
		assert.Contains(out, "Server started")

		stdout = testcommon.CaptureStdOut()
		err = app.Logs("db", false, false, "")
		assert.NoError(err)
		out = stdout()
		assert.Contains(out, "Database initialized")

		stdout = testcommon.CaptureStdOut()
		err = app.Logs("db", false, false, "2")
		assert.NoError(err)
		out = stdout()
		assert.Contains(out, "MySQL init process done. Ready for start up.")
		assert.False(strings.Contains(out, "Database initialized"))

		runTime()
		cleanup()
	}
}

// TestLocalStop tests the functionality that is called when "ddev stop" is executed
func TestLocalStop(t *testing.T) {
	assert := assert.New(t)

	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalStop", site.Name))

		testcommon.ClearDockerEnv()
		err := app.Init(site.Dir)
		assert.NoError(err)

		err = app.Stop()
		assert.NoError(err)

		for _, containerType := range [3]string{"web", "db", "dba"} {
			containerName, err := constructContainerName(containerType, app)
			assert.NoError(err)
			check, err := testcommon.ContainerCheck(containerName, "exited")
			assert.NoError(err)
			assert.True(check, containerType, "container has exited")
		}

		runTime()
		cleanup()
	}
}

// TestDescribeStopped tests that the describe command works properly on a stopped site.
func TestDescribeStopped(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()

		testcommon.ClearDockerEnv()
		err := app.Init(site.Dir)
		assert.NoError(err)

		out, err := app.Describe()
		assert.NoError(err)

		assert.Contains(out, SiteStopped, "Output did not include the word stopped when describing a stopped site.")
		cleanup()
	}
}

// TestLocalRemove tests the functionality that is called when "ddev rm" is executed
func TestLocalRemove(t *testing.T) {
	assert := assert.New(t)

	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()

		testcommon.ClearDockerEnv()
		err := app.Init(site.Dir)
		assert.NoError(err)

		// start the previously stopped containers -
		// stopped/removed have the same state
		err = app.Start()
		assert.NoError(err)

		err = app.Wait("web")
		assert.NoError(err)

		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalRemove", site.Name))
		err = app.Down()
		assert.NoError(err)

		for _, containerType := range [3]string{"web", "db", "dba"} {
			_, err := constructContainerName(containerType, app)
			assert.Error(err, "Received error on containerName search: ", err)
		}

		runTime()
		cleanup()
	}
}

// TestCleanupWithoutCompose ensures app containers can be properly cleaned up without a docker-compose config file present.
func TestCleanupWithoutCompose(t *testing.T) {
	assert := assert.New(t)
	site := TestSites[0]

	revertDir := site.Chdir()
	app, err := GetPluginApp("local")
	assert.NoError(err)

	testcommon.ClearDockerEnv()
	err = app.Init(site.Dir)
	assert.NoError(err)

	// Start a site so we have something to cleanup
	err = app.Start()
	assert.NoError(err)

	err = app.Wait("web")
	assert.NoError(err)

	// Call the Cleanup command()
	err = Cleanup(app)
	assert.NoError(err)

	for _, containerType := range [3]string{"web", "db", "dba"} {
		_, err := constructContainerName(containerType, app)
		assert.Error(err)
	}

	revertDir()
}

// TestGetappsEmpty ensures that GetApps returns an empty list when no applications are running.
func TestGetAppsEmpty(t *testing.T) {
	assert := assert.New(t)
	apps := GetApps()
	assert.Equal(len(apps["local"]), 0)
}

// TestRouterNotRunning ensures the router is shut down after all sites are stopped.
func TestRouterNotRunning(t *testing.T) {
	assert := assert.New(t)
	containers, err := util.GetDockerContainers(false)
	assert.NoError(err)

	for _, container := range containers {
		assert.NotEqual(util.ContainerName(container), "nginx-proxy", "Found nginx proxy running")
	}
}

// constructContainerName builds a container name given the type (web/db/dba) and the app
func constructContainerName(containerType string, app App) (string, error) {
	container, err := app.FindContainerByType(containerType)
	if err != nil {
		return "", err
	}
	name := util.ContainerName(container)
	return name, nil
}
<commit_msg>fail the local plugin tests if sites are already running (#279)<commit_after>package platform

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"os"

	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/drud/ddev/pkg/testcommon"
	"github.com/drud/ddev/pkg/util"
	"github.com/drud/drud-go/utils/system"
	"github.com/stretchr/testify/assert"
)

var (
	TestSites = []testcommon.TestSite{
		{
			Name:                          "TestMainPkgDrupal8",
			SourceURL:                     "https://github.com/drud/drupal8/archive/v0.6.0.tar.gz",
			ArchiveInternalExtractionPath: "drupal8-0.6.0/",
			FileURL: "https://github.com/drud/drupal8/releases/download/v0.6.0/files.tar.gz",
			DBURL:   "https://github.com/drud/drupal8/releases/download/v0.6.0/db.tar.gz",
		},
		{
			Name:                          "TestMainPkgWordpress",
			SourceURL:                     "https://github.com/drud/wordpress/archive/v0.4.0.tar.gz",
			ArchiveInternalExtractionPath: "wordpress-0.4.0/",
			FileURL: "https://github.com/drud/wordpress/releases/download/v0.4.0/files.tar.gz",
			DBURL:   "https://github.com/drud/wordpress/releases/download/v0.4.0/db.tar.gz",
		},
		{
			Name:                          "TestMainPkgDrupalKickstart",
			SourceURL:                     "https://github.com/drud/drupal-kickstart/archive/v0.4.0.tar.gz",
			ArchiveInternalExtractionPath: "drupal-kickstart-0.4.0/",
			FileURL: "https://github.com/drud/drupal-kickstart/releases/download/v0.4.0/files.tar.gz",
			DBURL:   "https://github.com/drud/drupal-kickstart/releases/download/v0.4.0/db.tar.gz",
		},
	}
)

func TestMain(m *testing.M) {
	if len(GetApps()) > 0 {
		log.Fatalf("Local plugin tests require no sites running. You have %v site(s) running.", len(GetApps()))
	}

	for i := range TestSites {
		err := TestSites[i].Prepare()
		if err != nil {
			log.Fatalf("Prepare() failed on TestSite.Prepare(), err=%v", err)
		}
	}

	log.Debugln("Running tests.")
	testRun := m.Run()

	for i := range TestSites {
		TestSites[i].Cleanup()
	}

	os.Exit(testRun)
}

// TestLocalStart tests the functionality that is called when "ddev start" is executed
func TestLocalStart(t *testing.T) {

	// ensure we have docker network
	client := util.GetDockerClient()
	err := util.EnsureNetwork(client, util.NetName)
	if err != nil {
		log.Fatal(err)
	}

	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalStart", site.Name))

		testcommon.ClearDockerEnv()
		err = app.Init(site.Dir)
		assert.NoError(err)

		err = app.Start()
		assert.NoError(err)

		err = app.Wait("web")
		assert.NoError(err)

		// ensure docker-compose.yaml exists inside .ddev site folder
		composeFile := system.FileExists(app.DockerComposeYAMLPath())
		assert.True(composeFile)

		for _, containerType := range [3]string{"web", "db", "dba"} {
			containerName, err := constructContainerName(containerType, app)
			assert.NoError(err)
			check, err := testcommon.ContainerCheck(containerName, "running")
			assert.NoError(err)
			assert.True(check, containerType, "container is running")
		}

		runTime()
		cleanup()
	}

	// try to start a site of same name at different path
	another := TestSites[0]
	err = another.Prepare()
	if err != nil {
		assert.FailNow("TestLocalStart: Prepare() failed on another.Prepare(), err=%v", err)
		return
	}

	err = app.Init(another.Dir)
	assert.Error(err)
	assert.Contains(err.Error(), fmt.Sprintf("container in running state already exists for %s that was created at %s", TestSites[0].Name, TestSites[0].Dir))
	another.Cleanup()
}

// TestGetApps tests the GetApps function to ensure it accurately returns a list of running applications.
func TestGetApps(t *testing.T) {
	assert := assert.New(t)
	apps := GetApps()
	assert.Equal(len(apps["local"]), len(TestSites))

	for _, site := range TestSites {
		var found bool
		for _, siteInList := range apps["local"] {
			if site.Name == siteInList.GetName() {
				found = true
				break
			}
		}
		assert.True(found, "Found site %s in list", site.Name)
	}
}

// TestLocalImportDB tests the functionality that is called when "ddev import-db" is executed
func TestLocalImportDB(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalImportDB", site.Name))
		dbPath := filepath.Join(testcommon.CreateTmpDir("local-db"), "db.tar.gz")

		err := system.DownloadFile(dbPath, site.DBURL)
		assert.NoError(err)

		testcommon.ClearDockerEnv()
		err = app.Init(site.Dir)
		assert.NoError(err)

		err = app.ImportDB(dbPath)
		assert.NoError(err)

		err = os.Remove(dbPath)
		assert.NoError(err)

		runTime()
		cleanup()
	}
}

// TestLocalImportFiles tests the functionality that is called when "ddev import-files" is executed
func TestLocalImportFiles(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalImportFiles", site.Name))
		filePath := filepath.Join(testcommon.CreateTmpDir("local-files"), "files.tar.gz")

		err := system.DownloadFile(filePath, site.FileURL)
		assert.NoError(err)

		testcommon.ClearDockerEnv()
		err = app.Init(site.Dir)
		assert.NoError(err)

		err = app.ImportFiles(filePath)
		assert.NoError(err)

		err = os.Remove(filePath)
		assert.NoError(err)

		runTime()
		cleanup()
	}
}

// TestLocalExec tests the execution of commands inside a docker container of a site.
func TestLocalExec(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalExec", site.Name))

		err := app.Init(site.Dir)
		assert.NoError(err)

		stdout := testcommon.CaptureStdOut()
		err = app.Exec("web", true, "pwd")
		assert.NoError(err)
		out := stdout()
		assert.Contains(out, "/var/www/html/docroot")

		stdout = testcommon.CaptureStdOut()
		switch app.GetType() {
		case "drupal7":
			fallthrough
		case "drupal8":
			err := app.Exec("web", true, "drush", "status")
			assert.NoError(err)
		case "wordpress":
			err = app.Exec("web", true, "wp", "--info")
			assert.NoError(err)
		default:
		}
		out = stdout()

		assert.Contains(string(out), "/etc/php/7.0/cli/php.ini")

		runTime()
		cleanup()

	}
}

// TestLocalLogs tests the container log output functionality.
func TestLocalLogs(t *testing.T) {
	assert := assert.New(t)

	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalLogs", site.Name))

		err := app.Init(site.Dir)
		assert.NoError(err)

		stdout := testcommon.CaptureStdOut()
		err = app.Logs("web", false, false, "")
		assert.NoError(err)
		out := stdout()
		assert.Contains(out, "Server started")

		stdout = testcommon.CaptureStdOut()
		err = app.Logs("db", false, false, "")
		assert.NoError(err)
		out = stdout()
		assert.Contains(out, "Database initialized")

		stdout = testcommon.CaptureStdOut()
		err = app.Logs("db", false, false, "2")
		assert.NoError(err)
		out = stdout()
		assert.Contains(out, "MySQL init process done. Ready for start up.")
		assert.False(strings.Contains(out, "Database initialized"))

		runTime()
		cleanup()
	}
}

// TestLocalStop tests the functionality that is called when "ddev stop" is executed
func TestLocalStop(t *testing.T) {
	assert := assert.New(t)

	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()
		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalStop", site.Name))

		testcommon.ClearDockerEnv()
		err := app.Init(site.Dir)
		assert.NoError(err)

		err = app.Stop()
		assert.NoError(err)

		for _, containerType := range [3]string{"web", "db", "dba"} {
			containerName, err := constructContainerName(containerType, app)
			assert.NoError(err)
			check, err := testcommon.ContainerCheck(containerName, "exited")
			assert.NoError(err)
			assert.True(check, containerType, "container has exited")
		}

		runTime()
		cleanup()
	}
}

// TestDescribeStopped tests that the describe command works properly on a stopped site.
func TestDescribeStopped(t *testing.T) {
	assert := assert.New(t)
	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()

		testcommon.ClearDockerEnv()
		err := app.Init(site.Dir)
		assert.NoError(err)

		out, err := app.Describe()
		assert.NoError(err)

		assert.Contains(out, SiteStopped, "Output did not include the word stopped when describing a stopped site.")
		cleanup()
	}
}

// TestLocalRemove tests the functionality that is called when "ddev rm" is executed
func TestLocalRemove(t *testing.T) {
	assert := assert.New(t)

	app, err := GetPluginApp("local")
	assert.NoError(err)

	for _, site := range TestSites {
		cleanup := site.Chdir()

		testcommon.ClearDockerEnv()
		err := app.Init(site.Dir)
		assert.NoError(err)

		// start the previously stopped containers -
		// stopped/removed have the same state
		err = app.Start()
		assert.NoError(err)

		err = app.Wait("web")
		assert.NoError(err)

		runTime := testcommon.TimeTrack(time.Now(), fmt.Sprintf("%s LocalRemove", site.Name))
		err = app.Down()
		assert.NoError(err)

		for _, containerType := range [3]string{"web", "db", "dba"} {
			_, err := constructContainerName(containerType, app)
			assert.Error(err, "Received error on containerName search: ", err)
		}

		runTime()
		cleanup()
	}
}

// TestCleanupWithoutCompose ensures app containers can be properly cleaned up without a docker-compose config file present.
func TestCleanupWithoutCompose(t *testing.T) {
	assert := assert.New(t)
	site := TestSites[0]

	revertDir := site.Chdir()
	app, err := GetPluginApp("local")
	assert.NoError(err)

	testcommon.ClearDockerEnv()
	err = app.Init(site.Dir)
	assert.NoError(err)

	// Start a site so we have something to cleanup
	err = app.Start()
	assert.NoError(err)

	err = app.Wait("web")
	assert.NoError(err)

	// Call the Cleanup command()
	err = Cleanup(app)
	assert.NoError(err)

	for _, containerType := range [3]string{"web", "db", "dba"} {
		_, err := constructContainerName(containerType, app)
		assert.Error(err)
	}

	revertDir()
}

// TestGetappsEmpty ensures that GetApps returns an empty list when no applications are running.
func TestGetAppsEmpty(t *testing.T) {
	assert := assert.New(t)
	apps := GetApps()
	assert.Equal(len(apps["local"]), 0)
}

// TestRouterNotRunning ensures the router is shut down after all sites are stopped.
func TestRouterNotRunning(t *testing.T) {
	assert := assert.New(t)
	containers, err := util.GetDockerContainers(false)
	assert.NoError(err)

	for _, container := range containers {
		assert.NotEqual(util.ContainerName(container), "nginx-proxy", "Found nginx proxy running")
	}
}

// constructContainerName builds a container name given the type (web/db/dba) and the app
func constructContainerName(containerType string, app App) (string, error) {
	container, err := app.FindContainerByType(containerType)
	if err != nil {
		return "", err
	}
	name := util.ContainerName(container)
	return name, nil
}
<|endoftext|>