<commit_before>package ssh

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path"
	"strconv"
	"strings"

	gossh "github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh"

	"github.com/coreos/fleet/pkg"
)

const (
	DefaultKnownHostsFile = "~/.fleetctl/known_hosts"

	sshDefaultPort = 22  /* ssh.h */
	sshHashDelim   = "|" /* hostfile.h */

	warningRemoteHostChanged = `@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@    WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!     @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!
Someone could be eavesdropping on you right now (man-in-the-middle attack)!
It is also possible that a host key has just been changed.
The fingerprint for the %v key sent by the remote host is
%v.
Please contact your system administrator.
Add correct host key in %v to get rid of this message.
Host key verification failed.
`
)

var (
	ErrUnparsableKey   = errors.New("unparsable key bytes")
	ErrUnsetTrustFunc  = errors.New("unset trustHost function")
	ErrUntrustHost     = errors.New("unauthorized host")
	ErrUnmatchKey      = errors.New("host key mismatch")
	ErrUnfoundHostAddr = errors.New("cannot find host address")
)

// HostKeyChecker implements gossh.HostKeyChecker interface
// It is used for validation during the cryptographic handshake
type HostKeyChecker struct {
	m         HostKeyManager
	trustHost func(addr, algo, fingerprint string) bool
	// errLog is used to print out error/warning message
	errLog *log.Logger
}

// NewHostKeyChecker returns new HostKeyChecker
// m manages existing host keys, trustHost tells whether or not to trust
// new host, errWriter indicates the place to write error msg
func NewHostKeyChecker(m HostKeyManager, trustHost func(addr, algo, fingerprint string) bool, errWriter io.Writer) *HostKeyChecker {
	if errWriter == nil {
		errWriter = os.Stderr
	}

	return &HostKeyChecker{m, trustHost, log.New(errWriter, "", 0)}
}

// SetTrustHost sets trustHost field
func (kc *HostKeyChecker) SetTrustHost(trustHost func(addr, algo, fingerprint string) bool) {
	kc.trustHost = trustHost
}

// matchHost tries to match the given host name against a comma-separated
// sequence of subpatterns s (each possibly preceded by ! to indicate negation).
// It returns a boolean indicating whether or not a positive match was made.
// Any matched negations take precedence over any other possible matches in the
// pattern.
func matchHost(host, pattern string) bool {
	subpatterns := strings.Split(pattern, ",")
	found := false
	for _, s := range subpatterns {
		/* If the host name matches a negated pattern, it is not
		accepted even if it matches other patterns on that line. */
		if strings.HasPrefix(s, "!") && match(host, s[1:]) {
			return false
		}
		/* Otherwise, check for a normal match */
		if match(host, s) {
			found = true
		}
	}
	/* Return success if we found a positive match.  If there was a negative
	   match, we have already returned false and never get here. */
	return found
}

// match compares the input string s to the pattern p, which may contain
// single and multi-character wildcards (? and * respectively). It returns a
// boolean indicating whether the string matches the pattern.
func match(s, p string) bool {
	var i, j int
	for i < len(p) {
		if p[i] == '*' {
			/* Skip the asterisk. */
			i++

			/* If at end of pattern, accept immediately. */
			if i == len(p) {
				return true
			}

			/* If next character in pattern is known, optimize. */
			if p[i] != '?' && p[i] != '*' {
				/* Look for instances of the next character in
				   pattern, and try to match starting from those. */
				for ; j < len(s); j++ {
					if s[j] == p[i] && match(s[j:], p[i:]) {
						return true
					}
				}
				/* Failed */
				return false
			}

			/*
			 * Move ahead one character at a time and try to
			 * match at each position.
			 */
			for ; j < len(s); j++ {
				if match(s[j:], p[i:]) {
					return true
				}
			}
			/* Failed. */
			return false
		}
		/*
		 * There must be at least one more character in the string.
		 * If we are at the end, fail.
		 */
		if j == len(s) {
			return false
		}

		/* Check if the next character of the string is acceptable. */
		if p[i] != '?' && p[i] != s[j] {
			return false
		}

		/* Move to the next character, both in string and in pattern. */
		i++
		j++
	}
	/* If at end of pattern, accept if also at end of string. */
	return j == len(s)
}

// Check is called during the handshake to check the server's public key for
// unexpected changes. The key argument is in SSH wire format. It can be parsed
// using ssh.ParsePublicKey. The address before DNS resolution is passed in the
// addr argument, so the key can also be checked against the hostname.
func (kc *HostKeyChecker) Check(addr string, remote net.Addr, key gossh.PublicKey) error {
	remoteAddr, err := kc.addrToHostPort(remote.String())
	if err != nil {
		return err
	}

	algoStr := algoString(key.Type())
	keyFingerprintStr := md5String(md5.Sum(key.Marshal()))

	hostKeys, err := kc.m.GetHostKeys()
	_, ok := err.(*os.PathError)
	if err != nil && !ok {
		kc.errLog.Println("Failed to read known_hosts file %v: %v", kc.m.String(), err)
	}

	mismatched := false
	for pattern, keys := range hostKeys {
		if !matchHost(remoteAddr, pattern) {
			continue
		}
		for _, hostKey := range keys {
			/* Any matching key is considered a success, irrespective of previous */
			if hostKey.Type() == key.Type() && bytes.Compare(hostKey.Marshal(), key.Marshal()) == 0 {
				return nil
			} else {
				mismatched = true
			}
		}
	}

	if mismatched {
		kc.errLog.Printf(warningRemoteHostChanged, algoStr, keyFingerprintStr, kc.m.String())
		return ErrUnmatchKey
	}

	/* If we get this far, we haven't matched on any of the hostname patterns */

	if kc.trustHost == nil {
		return ErrUnsetTrustFunc
	}
	if !kc.trustHost(remoteAddr, algoStr, keyFingerprintStr) {
		kc.errLog.Println("Host key verification failed.")
		return ErrUntrustHost
	}

	if err := kc.m.PutHostKey(remoteAddr, key); err != nil {
		kc.errLog.Printf("Failed to add the host to the list of known hosts (%v).\n", kc.m)
		return nil
	}

	kc.errLog.Printf("Warning: Permanently added '%v' (%v) to the list of known hosts.\n", remoteAddr, algoStr)
	return nil
}

// addrToHostPort takes the given address and parses it into a string suitable
// for use in the 'hostnames' field in a known_hosts file.  For more details,
// see the `SSH_KNOWN_HOSTS FILE FORMAT` section of `man 8 sshd`
func (kc *HostKeyChecker) addrToHostPort(a string) (string, error) {
	if !strings.Contains(a, ":") {
		// No port, so return unadulterated
		return a, nil
	}
	host, p, err := net.SplitHostPort(a)
	if err != nil {
		kc.errLog.Printf("Unable to parse addr %s: %v", a, err)
		return "", err
	}

	port, err := strconv.Atoi(p)
	if err != nil {
		kc.errLog.Printf("Error parsing port %s: %v", p, err)
		return "", err
	}

	// see `put_host_port` in openssh/misc.c
	if port == 0 || port == sshDefaultPort {
		// IPv6 addresses must be enclosed in square brackets
		if strings.Contains(host, ":") {
			host = "[" + host + "]"
		}
		return host, nil
	}

	return net.JoinHostPort(host, p), nil
}

// HostKeyManager gives the interface to manage host keys
type HostKeyManager interface {
	String() string
	// GetHostKeys returns a map from host patterns to a list of PublicKeys
	GetHostKeys() (map[string][]gossh.PublicKey, error)
	// put new host key under management
	PutHostKey(addr string, hostKey gossh.PublicKey) error
}

// HostKeyFile is an implementation of HostKeyManager interface
// Host keys are saved and loaded from file
type HostKeyFile struct {
	path string
}

// NewHostKeyFile returns new HostKeyFile using file path
func NewHostKeyFile(path string) *HostKeyFile {
	return &HostKeyFile{pkg.ParseFilepath(path)}
}

func (f *HostKeyFile) String() string {
	return f.path
}

func (f *HostKeyFile) GetHostKeys() (map[string][]gossh.PublicKey, error) {
	in, err := os.Open(f.path)
	if err != nil {
		return nil, err
	}
	defer in.Close()

	hostKeys := make(map[string][]gossh.PublicKey)
	n := 0
	s := bufio.NewScanner(in)
	for s.Scan() {
		n++
		line := s.Bytes()

		/* Skip any leading whitespace. */
		line = bytes.TrimLeft(line, "\t ")

		/* Skip comments and empty lines. */
		if bytes.HasPrefix(line, []byte("#")) || len(line) == 0 {
			continue
		}

		/* Skip markers. */
		if bytes.HasPrefix(line, []byte("@")) {
			log.Printf("Marker functionality not implemented - skipping line %d", n)
			continue
		}

		/* Find the end of the host name(s) portion. */
		end := bytes.IndexAny(line, "\t ")
		if end <= 0 {
			log.Printf("Bad format (insufficient fields) - skipping line %d", n)
			continue
		}
		hosts := string(line[:end])
		keyBytes := line[end+1:]

		/* Check for hashed host names. */
		if strings.HasPrefix(hosts, sshHashDelim) {
			log.Printf("Hashed hosts not implemented - skipping line %d", n)
			continue
		}

		key, _, _, _, err := gossh.ParseAuthorizedKey(keyBytes)
		if err != nil {
			log.Printf("Error parsing key, skipping line %d: %v", n, err)
			continue
		}

		/* It is permissible to have several lines for the same host name(s) */
		hostKeys[hosts] = append(hostKeys[hosts], key)
	}

	return hostKeys, nil
}

func (f *HostKeyFile) PutHostKey(addr string, hostKey gossh.PublicKey) error {
	// Make necessary directories if needed
	err := os.MkdirAll(path.Dir(f.path), 0700)
	if err != nil {
		return err
	}

	out, err := os.OpenFile(f.path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = out.Write(renderHostLine(addr, hostKey))
	if err != nil {
		return err
	}
	return nil
}

func parseHostLine(line []byte) (string, gossh.PublicKey, error) {
	end := bytes.IndexByte(line, ' ')
	if end <= 0 {
		return "", nil, ErrUnfoundHostAddr
	}
	addr := string(line[:end])
	keyByte := line[end+1:]
	key, _, _, _, err := gossh.ParseAuthorizedKey(keyByte)
	if err != nil {
		return "", nil, err
	}
	return addr, key, nil
}

func renderHostLine(addr string, key gossh.PublicKey) []byte {
	keyByte := gossh.MarshalAuthorizedKey(key)
	// allocate line space in advance
	length := len(addr) + 1 + len(keyByte)
	line := make([]byte, 0, length)

	w := bytes.NewBuffer(line)
	w.Write([]byte(addr))
	w.WriteByte(' ')
	w.Write(keyByte)
	return w.Bytes()
}

func algoString(algo string) string {
	switch algo {
	case gossh.KeyAlgoRSA:
		return "RSA"
	case gossh.KeyAlgoDSA:
		return "DSA"
	case gossh.KeyAlgoECDSA256:
		return "ECDSA256"
	case gossh.KeyAlgoECDSA384:
		return "ECDSA384"
	case gossh.KeyAlgoECDSA521:
		return "ECDSA521"
	}
	return algo
}

func md5String(md5Sum [16]byte) string {
	md5Str := fmt.Sprintf("% x", md5Sum)
	md5Str = strings.Replace(md5Str, " ", ":", -1)
	return md5Str
}
<commit_msg>chore(ssh): change comments to slashes<commit_after>package ssh

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path"
	"strconv"
	"strings"

	gossh "github.com/coreos/fleet/third_party/code.google.com/p/gosshnew/ssh"

	"github.com/coreos/fleet/pkg"
)

const (
	DefaultKnownHostsFile = "~/.fleetctl/known_hosts"

	sshDefaultPort = 22  // ssh.h
	sshHashDelim   = "|" // hostfile.h

	warningRemoteHostChanged = `@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@    WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!     @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!
Someone could be eavesdropping on you right now (man-in-the-middle attack)!
It is also possible that a host key has just been changed.
The fingerprint for the %v key sent by the remote host is
%v.
Please contact your system administrator.
Add correct host key in %v to get rid of this message.
Host key verification failed.
`
)

var (
	ErrUnparsableKey   = errors.New("unparsable key bytes")
	ErrUnsetTrustFunc  = errors.New("unset trustHost function")
	ErrUntrustHost     = errors.New("unauthorized host")
	ErrUnmatchKey      = errors.New("host key mismatch")
	ErrUnfoundHostAddr = errors.New("cannot find host address")
)

// HostKeyChecker implements the gossh.HostKeyChecker interface
// It is used for key validation during the cryptographic handshake
type HostKeyChecker struct {
	m         HostKeyManager
	trustHost func(addr, algo, fingerprint string) bool
	// errLog is used to print out error/warning message
	errLog *log.Logger
}

// NewHostKeyChecker returns new HostKeyChecker
// m manages existing host keys, trustHost tells whether or not to trust
// new host, errWriter indicates the place to write error msg
func NewHostKeyChecker(m HostKeyManager, trustHost func(addr, algo, fingerprint string) bool, errWriter io.Writer) *HostKeyChecker {
	if errWriter == nil {
		errWriter = os.Stderr
	}

	return &HostKeyChecker{m, trustHost, log.New(errWriter, "", 0)}
}

// SetTrustHost sets trustHost field
func (kc *HostKeyChecker) SetTrustHost(trustHost func(addr, algo, fingerprint string) bool) {
	kc.trustHost = trustHost
}

// matchHost tries to match the given host name against a comma-separated
// sequence of subpatterns s (each possibly preceded by ! to indicate negation).
// It returns a boolean indicating whether or not a positive match was made.
// Any matched negations take precedence over any other possible matches in the
// pattern.
func matchHost(host, pattern string) bool {
	subpatterns := strings.Split(pattern, ",")
	found := false
	for _, s := range subpatterns {
		// If the host name matches a negated pattern, it is not
		// accepted even if it matches other patterns on that line.
		if strings.HasPrefix(s, "!") && match(host, s[1:]) {
			return false
		}
		// Otherwise, check for a normal match
		if match(host, s) {
			found = true
		}
	}
	// Return success if we found a positive match.  If there was a negative
	// match, we have already returned false and never get here.
	return found
}

// match compares the input string s to the pattern p, which may contain
// single and multi-character wildcards (? and * respectively). It returns a
// boolean indicating whether the string matches the pattern.
func match(s, p string) bool {
	var i, j int
	for i < len(p) {
		if p[i] == '*' {
			// Skip the asterisk.
			i++

			// If at end of pattern, accept immediately.
			if i == len(p) {
				return true
			}

			// If next character in pattern is known, optimize.
			if p[i] != '?' && p[i] != '*' {
				// Look for instances of the next character in
				// pattern, and try to match starting from those.
				for ; j < len(s); j++ {
					if s[j] == p[i] && match(s[j:], p[i:]) {
						return true
					}
				}
				// Failed.
				return false
			}

			// Move ahead one character at a time and try to
			// match at each position.
			for ; j < len(s); j++ {
				if match(s[j:], p[i:]) {
					return true
				}
			}
			// Failed.
			return false
		}

		// There must be at least one more character in the string.
		// If we are at the end, fail.
		if j == len(s) {
			return false
		}

		// Check if the next character of the string is acceptable.
		if p[i] != '?' && p[i] != s[j] {
			return false
		}

		// Move to the next character, both in string and in pattern.
		i++
		j++
	}
	// If at end of pattern, accept if also at end of string.
	return j == len(s)
}

// Check is called during the handshake to check the server's public key for
// unexpected changes. The key argument is in SSH wire format. It can be parsed
// using ssh.ParsePublicKey. The address before DNS resolution is passed in the
// addr argument, so the key can also be checked against the hostname.
func (kc *HostKeyChecker) Check(addr string, remote net.Addr, key gossh.PublicKey) error {
	remoteAddr, err := kc.addrToHostPort(remote.String())
	if err != nil {
		return err
	}

	algoStr := algoString(key.Type())
	keyFingerprintStr := md5String(md5.Sum(key.Marshal()))

	hostKeys, err := kc.m.GetHostKeys()
	_, ok := err.(*os.PathError)
	if err != nil && !ok {
		kc.errLog.Println("Failed to read known_hosts file %v: %v", kc.m.String(), err)
	}

	mismatched := false
	for pattern, keys := range hostKeys {
		if !matchHost(remoteAddr, pattern) {
			continue
		}
		for _, hostKey := range keys {
			// Any matching key is considered a success, irrespective of previous
			if hostKey.Type() == key.Type() && bytes.Compare(hostKey.Marshal(), key.Marshal()) == 0 {
				return nil
			} else {
				mismatched = true
			}
		}
	}

	if mismatched {
		kc.errLog.Printf(warningRemoteHostChanged, algoStr, keyFingerprintStr, kc.m.String())
		return ErrUnmatchKey
	}

	// If we get this far, we haven't matched on any of the hostname patterns

	if kc.trustHost == nil {
		return ErrUnsetTrustFunc
	}
	if !kc.trustHost(remoteAddr, algoStr, keyFingerprintStr) {
		kc.errLog.Println("Host key verification failed.")
		return ErrUntrustHost
	}

	if err := kc.m.PutHostKey(remoteAddr, key); err != nil {
		kc.errLog.Printf("Failed to add the host to the list of known hosts (%v).\n", kc.m)
		return nil
	}

	kc.errLog.Printf("Warning: Permanently added '%v' (%v) to the list of known hosts.\n", remoteAddr, algoStr)
	return nil
}

// addrToHostPort takes the given address and parses it into a string suitable
// for use in the 'hostnames' field in a known_hosts file.  For more details,
// see the `SSH_KNOWN_HOSTS FILE FORMAT` section of `man 8 sshd`
func (kc *HostKeyChecker) addrToHostPort(a string) (string, error) {
	if !strings.Contains(a, ":") {
		// No port, so return unadulterated
		return a, nil
	}
	host, p, err := net.SplitHostPort(a)
	if err != nil {
		kc.errLog.Printf("Unable to parse addr %s: %v", a, err)
		return "", err
	}

	port, err := strconv.Atoi(p)
	if err != nil {
		kc.errLog.Printf("Error parsing port %s: %v", p, err)
		return "", err
	}

	// see `put_host_port` in openssh/misc.c
	if port == 0 || port == sshDefaultPort {
		// IPv6 addresses must be enclosed in square brackets
		if strings.Contains(host, ":") {
			host = "[" + host + "]"
		}
		return host, nil
	}

	return net.JoinHostPort(host, p), nil
}

// HostKeyManager gives the interface to manage host keys
type HostKeyManager interface {
	String() string
	// GetHostKeys returns a map from host patterns to a list of PublicKeys
	GetHostKeys() (map[string][]gossh.PublicKey, error)
	// put new host key under management
	PutHostKey(addr string, hostKey gossh.PublicKey) error
}

// HostKeyFile is an implementation of HostKeyManager interface
// Host keys are saved and loaded from file
type HostKeyFile struct {
	path string
}

// NewHostKeyFile returns new HostKeyFile using file path
func NewHostKeyFile(path string) *HostKeyFile {
	return &HostKeyFile{pkg.ParseFilepath(path)}
}

func (f *HostKeyFile) String() string {
	return f.path
}

func (f *HostKeyFile) GetHostKeys() (map[string][]gossh.PublicKey, error) {
	in, err := os.Open(f.path)
	if err != nil {
		return nil, err
	}
	defer in.Close()

	hostKeys := make(map[string][]gossh.PublicKey)
	n := 0
	s := bufio.NewScanner(in)
	for s.Scan() {
		n++
		line := s.Bytes()

		// Skip any leading whitespace.
		line = bytes.TrimLeft(line, "\t ")

		// Skip comments and empty lines.
		if bytes.HasPrefix(line, []byte("#")) || len(line) == 0 {
			continue
		}

		// Skip markers.
		if bytes.HasPrefix(line, []byte("@")) {
			log.Printf("Marker functionality not implemented - skipping line %d", n)
			continue
		}

		// Find the end of the host name(s) portion.
		end := bytes.IndexAny(line, "\t ")
		if end <= 0 {
			log.Printf("Bad format (insufficient fields) - skipping line %d", n)
			continue
		}
		hosts := string(line[:end])
		keyBytes := line[end+1:]

		// Check for hashed host names.
		if strings.HasPrefix(hosts, sshHashDelim) {
			log.Printf("Hashed hosts not implemented - skipping line %d", n)
			continue
		}

		key, _, _, _, err := gossh.ParseAuthorizedKey(keyBytes)
		if err != nil {
			log.Printf("Error parsing key, skipping line %d: %v", n, err)
			continue
		}

		// It is permissible to have several lines for the same host name(s)
		hostKeys[hosts] = append(hostKeys[hosts], key)
	}

	return hostKeys, nil
}

func (f *HostKeyFile) PutHostKey(addr string, hostKey gossh.PublicKey) error {
	// Make necessary directories if needed
	err := os.MkdirAll(path.Dir(f.path), 0700)
	if err != nil {
		return err
	}

	out, err := os.OpenFile(f.path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	defer out.Close()

	_, err = out.Write(renderHostLine(addr, hostKey))
	if err != nil {
		return err
	}
	return nil
}

func parseHostLine(line []byte) (string, gossh.PublicKey, error) {
	end := bytes.IndexByte(line, ' ')
	if end <= 0 {
		return "", nil, ErrUnfoundHostAddr
	}
	addr := string(line[:end])
	keyByte := line[end+1:]
	key, _, _, _, err := gossh.ParseAuthorizedKey(keyByte)
	if err != nil {
		return "", nil, err
	}
	return addr, key, nil
}

func renderHostLine(addr string, key gossh.PublicKey) []byte {
	keyByte := gossh.MarshalAuthorizedKey(key)
	// allocate line space in advance
	length := len(addr) + 1 + len(keyByte)
	line := make([]byte, 0, length)

	w := bytes.NewBuffer(line)
	w.Write([]byte(addr))
	w.WriteByte(' ')
	w.Write(keyByte)
	return w.Bytes()
}

func algoString(algo string) string {
	switch algo {
	case gossh.KeyAlgoRSA:
		return "RSA"
	case gossh.KeyAlgoDSA:
		return "DSA"
	case gossh.KeyAlgoECDSA256:
		return "ECDSA256"
	case gossh.KeyAlgoECDSA384:
		return "ECDSA384"
	case gossh.KeyAlgoECDSA521:
		return "ECDSA521"
	}
	return algo
}

func md5String(md5Sum [16]byte) string {
	md5Str := fmt.Sprintf("% x", md5Sum)
	md5Str = strings.Replace(md5Str, " ", ":", -1)
	return md5Str
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssh

import (
	"bytes"
	crypto_rand "crypto/rand"
	"io"
	"math/rand"
	"testing"
)

// windowTestBytes is the number of bytes that we'll send to the SSH server.
const windowTestBytes = 16000 * 200

// CopyNRandomly copies n bytes from src to dst. It uses a variable, and random,
// buffer size to exercise more code paths.
func CopyNRandomly(dst io.Writer, src io.Reader, n int64) (written int64, err error) {
	buf := make([]byte, 32*1024)
	for written < n {
		l := (rand.Intn(30) + 1) * 1024
		if d := n - written; d < int64(l) {
			l = int(d)
		}
		nr, er := src.Read(buf[0:l])
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}

func TestServerWindow(t *testing.T) {
	addr := startSSHServer(t)
	runSSHClient(t, addr)
}

// runSSHClient writes random data to the server. The server is expected to echo
// the same data back, which is compared against the original.
func runSSHClient(t *testing.T, addr string) {
	conn, err := Dial("tcp", addr, &ClientConfig{})
	if err != nil {
		t.Fatal(err)
	}

	session, err := conn.NewSession()
	if err != nil {
		t.Fatal(err)
	}

	origBuf := bytes.NewBuffer(make([]byte, 0, windowTestBytes))
	echoedBuf := bytes.NewBuffer(make([]byte, 0, windowTestBytes))
	io.CopyN(origBuf, crypto_rand.Reader, windowTestBytes)
	origBytes := origBuf.Bytes()

	wait := make(chan bool)

	// Read back the data from the server.
	go func() {
		defer session.Close()
		serverStdout, err := session.StdoutPipe()
		if err != nil {
			t.Fatal(err)
		}

		n, err := CopyNRandomly(echoedBuf, serverStdout, windowTestBytes)
		if err != nil && err != io.EOF {
			t.Fatal(err)
		}
		if n != windowTestBytes {
			t.Fatalf("Read only %d bytes from server, expected %d", n, windowTestBytes)
		}
		wait <- true
	}()

	serverStdin, err := session.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}

	written, err := CopyNRandomly(serverStdin, origBuf, windowTestBytes)
	if err != nil {
		t.Fatal(err)
	}
	if written != windowTestBytes {
		t.Fatalf("Wrote only %d of %d bytes to server", written, windowTestBytes)
	}

	<-wait

	if !bytes.Equal(origBytes, echoedBuf.Bytes()) {
		t.Error("Echoed buffer differed from original")
	}
}

func startSSHServer(t *testing.T) (addr string) {
	config := &ServerConfig{
		NoClientAuth: true,
	}

	err := config.SetRSAPrivateKey([]byte(testServerPrivateKey))
	if err != nil {
		t.Fatalf("Failed to parse private key: %s", err.Error())
	}

	listener, err := Listen("tcp", "127.0.0.1:0", config)
	if err != nil {
		t.Fatalf("Bind error: %s", err)
	}

	addr = listener.Addr().String()

	go func() {
		for {
			sConn, err := listener.Accept()

			err = sConn.Handshake()
			if err != nil {
				if err != io.EOF {
					t.Fatalf("failed to handshake: %s", err)
				}
				return
			}

			go connRun(t, sConn)
		}
	}()

	return
}

func connRun(t *testing.T, sConn *ServerConn) {
	for {
		channel, err := sConn.Accept()
		if err != nil {
			if err == io.EOF {
				break
			}
			t.Fatalf("ServerConn.Accept failed: %s", err)
		}

		if channel.ChannelType() != "session" {
			channel.Reject(UnknownChannelType, "unknown channel type")
			continue
		}
		err = channel.Accept()
		if err != nil {
			t.Fatalf("Channel.Accept failed: %s", err)
		}

		go func() {
			defer channel.Close()

			n, err := CopyNRandomly(channel, channel, windowTestBytes)
			if err != nil && err != io.EOF {
				if err == io.ErrShortWrite {
					t.Fatalf("short write, wrote %d, expected %d", n, windowTestBytes)
				}
				t.Fatal(err)
			}
		}()
	}
}
<commit_msg>go.crypto/ssh: improve TestServerWindow robustness<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssh

import (
	"bytes"
	crypto_rand "crypto/rand"
	"io"
	"math/rand"
	"testing"
)

// windowTestBytes is the number of bytes that we'll send to the SSH server.
const windowTestBytes = 16000 * 200

// CopyNRandomly copies n bytes from src to dst. It uses a variable, and random,
// buffer size to exercise more code paths.
func CopyNRandomly(dst io.Writer, src io.Reader, n int) (written int, err error) {
	buf := make([]byte, 32*1024)
	for written < n {
		l := (rand.Intn(30) + 1) * 1024
		if d := n - written; d < l {
			l = d
		}
		nr, er := src.Read(buf[0:l])
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw > 0 {
				written += nw
			}
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = io.ErrShortWrite
				break
			}
		}
		if er != nil {
			err = er
			break
		}
	}
	return written, err
}

func TestServerWindow(t *testing.T) {
	addr := startSSHServer(t)
	runSSHClient(t, addr)
}

// runSSHClient writes random data to the server. The server is expected to echo
// the same data back, which is compared against the original.
func runSSHClient(t *testing.T, addr string) {
	conn, err := Dial("tcp", addr, &ClientConfig{})
	if err != nil {
		t.Fatal(err)
	}

	session, err := conn.NewSession()
	if err != nil {
		t.Fatal(err)
	}

	origBuf := bytes.NewBuffer(make([]byte, 0, windowTestBytes))
	echoedBuf := bytes.NewBuffer(make([]byte, 0, windowTestBytes))
	io.CopyN(origBuf, crypto_rand.Reader, windowTestBytes)
	origBytes := origBuf.Bytes()

	wait := make(chan bool)

	// Read back the data from the server.
	go func() {
		defer session.Close()
		defer close(wait)
		serverStdout, err := session.StdoutPipe()
		if err != nil {
			t.Fatal(err)
		}

		n, err := CopyNRandomly(echoedBuf, serverStdout, windowTestBytes)
		if err != nil && err != io.EOF {
			t.Fatal(err)
		}
		if n != windowTestBytes {
			t.Fatalf("Read only %d bytes from server, expected %d", n, windowTestBytes)
		}
	}()

	serverStdin, err := session.StdinPipe()
	if err != nil {
		t.Fatal(err)
	}

	written, err := CopyNRandomly(serverStdin, origBuf, windowTestBytes)
	if err != nil {
		t.Fatal(err)
	}
	if written != windowTestBytes {
		t.Fatalf("Wrote only %d of %d bytes to server", written, windowTestBytes)
	}

	<-wait

	if !bytes.Equal(origBytes, echoedBuf.Bytes()) {
		t.Error("Echoed buffer differed from original")
	}
}

func startSSHServer(t *testing.T) (addr string) {
	config := &ServerConfig{
		NoClientAuth: true,
	}

	err := config.SetRSAPrivateKey([]byte(testServerPrivateKey))
	if err != nil {
		t.Fatalf("Failed to parse private key: %s", err.Error())
	}

	listener, err := Listen("tcp", "127.0.0.1:0", config)
	if err != nil {
		t.Fatalf("Bind error: %s", err)
	}

	addr = listener.Addr().String()
	go func() {
		defer listener.Close()
		for {
			sConn, err := listener.Accept()
			err = sConn.Handshake()
			if err != nil {
				if err != io.EOF {
					t.Fatalf("failed to handshake: %s", err)
				}
				return
			}

			go connRun(t, sConn)
		}
	}()

	return
}

func connRun(t *testing.T, sConn *ServerConn) {
	defer sConn.Close()
	for {
		channel, err := sConn.Accept()
		if err != nil {
			if err == io.EOF {
				break
			}
			t.Fatalf("ServerConn.Accept failed: %s", err)
		}

		if channel.ChannelType() != "session" {
			channel.Reject(UnknownChannelType, "unknown channel type")
			continue
		}
		err = channel.Accept()
		if err != nil {
			t.Fatalf("Channel.Accept failed: %s", err)
		}

		go func() {
			defer channel.Close()
			n, err := CopyNRandomly(channel, channel, windowTestBytes)
			if err != nil && err != io.EOF {
				if err == io.ErrShortWrite {
					t.Fatalf("short write, wrote %d, expected %d", n, windowTestBytes)
				}
				t.Fatal(err)
			}
		}()
	}
}
<|endoftext|>


<commit_before>package hmm

import (
	"code.google.com/p/biogo.matrix"
	"github.com/akualab/gjoa/model"
	"github.com/akualab/gjoa/model/gaussian"
	//"math"
	"testing"
)

// Tests

/*
   DISCUSSION:
   If you look at the sample data and model params. I manufactured the
   data as if it was emitted with the following sequence:

   t:  0   1   2   3   4   5   6   7   8   9   10  11
   q:  s0  s0  s0  s0  s0  s0  s1  s1  s1  s1  s0  s0
   o:  0.1 0.3 1.1 1.2 0.7 0.7 5.5 7.8 10  5.2 1.1 1.3 <=
   data I created given the Gaussians [1,1] and [4,4]

   I got the following gamma:

   γ0: -0.03 -0.03 -0.05 -0.05 -0.04 -0.11 -9.02 -21 -36 -7.8 -0.15 -0.11
   γ1: -3.35 -3.41 -3.01 -2.92 -3.13 -2.24 -0.00 -0  -0  -0   -1.91 -2.21

   As you can see choosing the gamma with highest prob for each state give
   us the hidden sequence of states.

   gamma gives you the most likely state at time t. In this case the result is what we expect.

   Viterbi gives you the P(q | O,  model), that is, it maximizes of over the whole sequence.
*/

// Test ColumnAt() function.
func TestColumnAt(t *testing.T) {

	mat, e := matrix.NewDense([][]float64{
		{10, 11, 12, 13}, {20, 21, 22, 23}, {30, 31, 32, 33}})
	if e != nil {
		t.Fatal(e)
	}

	col := ColumnAt(mat, 1)

	t.Logf("col: \n%+v", col)

	for i, expected := range []float64{11.0, 21.0, 31.0} {
		v := col.At(i, 0)
		if !model.Comparef64(expected, v) {
			t.Errorf("Wrong value. Expected: [%f], Got: [%f]", expected, v)
		}
	}
}

func MakeNewDenseMatrix(t *testing.T, m [][]float64) *matrix.Dense {
	mm, emm := matrix.NewDense(m)
	if emm != nil {
		t.Fatal(emm)
	}
	return mm
}

func MakeHMM(t *testing.T) *HMM {

	// Gaussian 1.
	mean1 := MakeNewDenseMatrix(t, [][]float64{{1}})
	var1 := MakeNewDenseMatrix(t, [][]float64{{1}})
	g1, eg1 := gaussian.NewGaussian(1, mean1, var1, true, true, "g1")
	if eg1 != nil {
		t.Fatal(eg1)
	}
	// Gaussian 2.
	mean2 := MakeNewDenseMatrix(t, [][]float64{{4}})
	var2 := MakeNewDenseMatrix(t, [][]float64{{4}})
	var2, ev2 := matrix.NewDense([][]float64{{4}})
	if ev2 != nil {
		t.Fatal(ev2)
	}
	g2, eg2 := gaussian.NewGaussian(1, mean2, var2, true, true, "g2")
	if eg2 != nil {
		t.Fatal(eg2)
	}

	initialStateProbs := MakeNewDenseMatrix(t, [][]float64{{0.8}, {0.2}})

	transProbs := MakeNewDenseMatrix(t, [][]float64{{0.9, 0.1}, {0.3, 0.7}})

	// These are the models.
	models := []*gaussian.Gaussian{g1, g2}

	// To pass the to an HMM we need to convert []*gaussian.Gaussian[]
	// to []model.Modeler
	// see http://golang.org/doc/faq#convert_slice_of_interface
	m := make([]model.Modeler, len(models))
	for i, v := range models {
		m[i] = v
	}
	hmm, e := NewHMM(transProbs, initialStateProbs, m)
	if e != nil {
		t.Fatal(e)
	}
	return hmm
}

var (
	obs0 = [][]float64{
		{0.1, 0.3, 1.1, 1.2, 0.7, 0.7, 5.5, 7.8, 10.0, 5.2, 1.1, 1.3}}
	alpha0 = []float64{
		-1.54708208451888,
		-2.80709238811418,
		-3.83134003758912,
		-4.86850442594034,
		-5.92973730403429,
		-6.99328952650412,
		-18.1370692144982,
		-36.3195887463382,
		-57.4758051059185,
		-32.2645657649804,
		-25.5978716740632,
		-26.5391830081456}
	alpha1 = []float64{
		-5.12277362619872,
		-6.99404330419337,
		-7.67194890763762,
		-8.58593275227677,
		-9.98735773434079,
		-11.0914094981902,
		-11.0792560557189,
		-14.8528937698143,
		-21.3216544274498,
		-23.4704150851531,
		-26.4904040834703,
		-29.0712307616184}
	beta0 = []float64{
		-24.9258011954291,
		-23.661415171904,
		-22.6397641116887,
		-21.6045197079498,
		-20.549461075003,
		-19.579339900188,
		-17.3294657178329,
		-13.5557050615525,
		-7.07931720879328,
		-2.06607429111337,
		-1.04620524392834,
		0}
	beta1 = []float64{
		-25.9309053603105,
		-24.6182012641994,
		-23.5726285099546,
		-22.4540945910146,
		-20.5873818254665,
		-17.6335597285231,
		-15.3835555701327,
		-11.6097949124971,
		-5.14103425479379,
		-2.99265648819389,
		-1.76872378444132,
		0}
	gamma0 = []float64{
		-0.0101945977044363,
		-0.00581887777458709,
		-0.00841546703429051,
		-0.0103354516465726,
		-0.0165096967936958,
		-0.10994074444856,
		-9.00384625008746,
		-23.4126051256471,
		-38.0924336324682,
		-7.86795137385019,
		-0.181388235747997,
		-0.076494325902054}
	gamma1 = []float64{
		-4.59099030426571,
		-5.14955588614921,
		-4.78188873534866,
		-4.5773386610478,
		-4.11205087756371,
		-2.2622805444697,
		-0.000122943608043396,
		-6.79257761172257e-11,
		0,
		-0.000382891103450269,
		-1.79643918566812,
		-2.60854207937483}
)

func CompareSliceFloat(t *testing.T, expected []float64, actual *matrix.Dense, row int, message string) {
	for i, _ := range expected {
		if !model.Comparef64(expected[i], actual.At(row, i)) {
			t.Errorf("[%s]. Expected: [%f], Got: [%f]",
				message, expected[i], actual.At(i, 0))
		}
	}
}

func CompareFloats(t *testing.T, expected float64, actual float64, message string) {
	if !model.Comparef64(expected, actual) {
		t.Errorf("[%s]. Expected: [%f], Got: [%f]",
			message, expected, actual)
	}
}

func TestEvaluationAlpha(t *testing.T) {

	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	alpha, logProb, err_alpha := hmm.alpha(obs)
	if err_alpha != nil {
		t.Fatal(err_alpha)
	}
	expectedLogProb := -26.4626886822436
	CompareFloats(t, expectedLogProb, logProb, "Error in logProb")
	message := "Error in alpha"
	CompareSliceFloat(t, alpha0, alpha, 0, message)
	CompareSliceFloat(t, alpha1, alpha, 1, message)
}

func TestEvaluationBeta(t *testing.T) {

	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	beta, err_beta := hmm.beta(obs)
	if err_beta != nil {
		t.Fatal(err_beta)
	}
	message := "Error in beta"
	CompareSliceFloat(t, beta0, beta, 0, message)
	CompareSliceFloat(t, beta1, beta, 1, message)
}

func TestEvaluationGamma(t *testing.T) {

	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	alpha, _, err_alpha := hmm.alpha(obs)
	if err_alpha != nil {
		t.Fatal(err_alpha)
	}
	beta, err_beta := hmm.beta(obs)
	if err_beta != nil {
		t.Fatal(err_beta)
	}
	gamma, err_gamma := hmm.gamma(alpha, beta)
	if err_gamma != nil {
		t.Fatal(err_gamma)
	}
	message := "Error in gamma"
	CompareSliceFloat(t, gamma0, gamma, 0, message)
	CompareSliceFloat(t, gamma1, gamma, 1, message)
}

/*
func TestEvaluationXi(t *testing.T) {
	obs := MakeNewDenseMatrix(t, obs0)
	xi, err_xi := hmm.xi(obs, alpha, beta)
	if err_xi != nil {
		t.Fatal(err_xi)
	}
	t.Logf("xi:\n%+v\n", xi)
}*/
<commit_msg>All test pass<commit_after>package hmm

import (
	"code.google.com/p/biogo.matrix"
	"github.com/akualab/gjoa/model"
	"github.com/akualab/gjoa/model/gaussian"
	//"math"
	"testing"
)

// Tests

/*
   DISCUSSION:
   If you look at the sample data and model params. I manufactured the
   data as if it was emitted with the following sequence:

   t:  0   1   2   3   4   5   6   7   8   9   10  11
   q:  s0  s0  s0  s0  s0  s0  s1  s1  s1  s1  s0  s0
   o:  0.1 0.3 1.1 1.2 0.7 0.7 5.5 7.8 10  5.2 1.1 1.3 <=
   data I created given the Gaussians [1,1] and [4,4]

   I got the following gamma:

   γ0: -0.03 -0.03 -0.05 -0.05 -0.04 -0.11 -9.02 -21 -36 -7.8 -0.15 -0.11
   γ1: -3.35 -3.41 -3.01 -2.92 -3.13 -2.24 -0.00 -0  -0  -0   -1.91 -2.21

   As you can see choosing the gamma with highest prob for each state give
   us the hidden sequence of states.

   gamma gives you the most likely state at time t. In this case the result is what we expect.

   Viterbi gives you the P(q | O,  model), that is, it maximizes of over the whole sequence.
*/

// Test ColumnAt() function.
func TestColumnAt(t *testing.T) {

	mat, e := matrix.NewDense([][]float64{
		{10, 11, 12, 13}, {20, 21, 22, 23}, {30, 31, 32, 33}})
	if e != nil {
		t.Fatal(e)
	}

	col := ColumnAt(mat, 1)

	for i, expected := range []float64{11.0, 21.0, 31.0} {
		v := col.At(i, 0)
		if !model.Comparef64(expected, v) {
			t.Errorf("Wrong value. Expected: [%f], Got: [%f]", expected, v)
		}
	}
}

func MakeNewDenseMatrix(t *testing.T, m [][]float64) *matrix.Dense {
	mm, emm := matrix.NewDense(m)
	if emm != nil {
		t.Fatal(emm)
	}
	return mm
}

func MakeHMM(t *testing.T) *HMM {

	// Gaussian 1.
	mean1 := MakeNewDenseMatrix(t, [][]float64{{1}})
	var1 := MakeNewDenseMatrix(t, [][]float64{{1}})
	g1, eg1 := gaussian.NewGaussian(1, mean1, var1, true, true, "g1")
	if eg1 != nil {
		t.Fatal(eg1)
	}
	// Gaussian 2.
	mean2 := MakeNewDenseMatrix(t, [][]float64{{4}})
	var2 := MakeNewDenseMatrix(t, [][]float64{{4}})
	var2, ev2 := matrix.NewDense([][]float64{{4}})
	if ev2 != nil {
		t.Fatal(ev2)
	}
	g2, eg2 := gaussian.NewGaussian(1, mean2, var2, true, true, "g2")
	if eg2 != nil {
		t.Fatal(eg2)
	}

	initialStateProbs := MakeNewDenseMatrix(t, [][]float64{{0.8}, {0.2}})

	transProbs := MakeNewDenseMatrix(t, [][]float64{{0.9, 0.1}, {0.3, 0.7}})

	// These are the models.
	models := []*gaussian.Gaussian{g1, g2}

	// To pass the to an HMM we need to convert []*gaussian.Gaussian[]
	// to []model.Modeler
	// see http://golang.org/doc/faq#convert_slice_of_interface
	m := make([]model.Modeler, len(models))
	for i, v := range models {
		m[i] = v
	}
	hmm, e := NewHMM(transProbs, initialStateProbs, m)
	if e != nil {
		t.Fatal(e)
	}
	return hmm
}

var (
	obs0 = [][]float64{
		{0.1, 0.3, 1.1, 1.2, 0.7, 0.7, 5.5, 7.8, 10.0, 5.2, 1.1, 1.3}}
	alpha01 = []float64{
		-1.54708208451888,
		-2.80709238811418,
		-3.83134003758912,
		-4.86850442594034,
		-5.92973730403429,
		-6.99328952650412,
		-18.1370692144982,
		-36.3195887463382,
		-57.4758051059185,
		-32.2645657649804,
		-25.5978716740632,
		-26.5391830081456,
		-5.12277362619872,
		-6.99404330419337,
		-7.67194890763762,
		-8.58593275227677,
		-9.98735773434079,
		-11.0914094981902,
		-11.0792560557189,
		-14.8528937698143,
		-21.3216544274498,
		-23.4704150851531,
		-26.4904040834703,
		-29.0712307616184}
	beta01 = []float64{
		-24.9258011954291,
		-23.661415171904,
		-22.6397641116887,
		-21.6045197079498,
		-20.549461075003,
		-19.579339900188,
		-17.3294657178329,
		-13.5557050615525,
		-7.07931720879328,
		-2.06607429111337,
		-1.04620524392834,
		0,
		-25.9309053603105,
		-24.6182012641994,
		-23.5726285099546,
		-22.4540945910146,
		-20.5873818254665,
		-17.6335597285231,
		-15.3835555701327,
		-11.6097949124971,
		-5.14103425479379,
		-2.99265648819389,
		-1.76872378444132,
		0}
	gamma01 = []float64{
		-0.0101945977044363,
		-0.00581887777458709,
		-0.00841546703429051,
		-0.0103354516465726,
		-0.0165096967936958,
		-0.10994074444856,
		-9.00384625008746,
		-23.4126051256471,
		-38.0924336324682,
		-7.86795137385019,
		-0.181388235747997,
		-0.076494325902054,
		-4.59099030426571,
		-5.14955588614921,
		-4.78188873534866,
		-4.5773386610478,
		-4.11205087756371,
		-2.2622805444697,
		-0.000122943608043396,
		-6.79257761172257e-11,
		0,
		-0.000382891103450269,
		-1.79643918566812,
		-2.60854207937483}
	xsi = []float64{
		-0.0151076230417916, -0.0134668664218517, -0.0174701121578483, -0.0245758675622469,
		-0.11568757084122, -9.00936561095593, -29.3743846426695, -58.4605163217504,
		-42.9234897636508, -7.87738137552765, -0.204482040682161, 0,
		-5.32851547323339, -4.88295302258388, -4.71741675311881, -4.26911837592192,
		-2.37652915707246, -0.110077221151965, -9.0038462515104, -23.4126051256471,
		-38.1004437186275, -12.5365216739368, -3.96110379857833, 0,
		-4.68941145338974, -5.29903007116915, -4.95669127087446, -4.84061648256679,
		-5.27192028981584, -14.2060978713101, -23.4151837725584, -38.0924336338947,
		-7.86795137385019, -0.181842984368506, -2.1956267387574, 0,
		-6.95829686585791, -7.12399378960776, -6.612115474112, -6.04063655320304,
		-4.48823943832366, -2.26228704378271, -0.000122943675802531, -6.79259981618307e-11,
		-0.000382891103450158, -1.79646084505424, -2.90772605893015, 0}
)

func CompareSliceFloat(t *testing.T, expected []float64, actual []float64, message string) {
	for i, _ := range expected {
		if !model.Comparef64(expected[i], actual[i]) {
			t.Errorf("[%s]. Expected: [%f], Got: [%f]",
				message, expected[i], actual[i])
		}
	}
}

func CompareFloats(t *testing.T, expected float64, actual float64, message string) {
	if !model.Comparef64(expected, actual) {
		t.Errorf("[%s]. Expected: [%f], Got: [%f]",
			message, expected, actual)
	}
}

func TestEvaluationAlpha(t *testing.T) {

	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	alpha, logProb, err_alpha := hmm.alpha(obs)
	if err_alpha != nil {
		t.Fatal(err_alpha)
	}
	expectedLogProb := -26.4626886822436
	CompareFloats(t, expectedLogProb, logProb, "Error in logProb")
	message := "Error in alpha"
	CompareSliceFloat(t, alpha01, alpha.ElementsVector(), message)
}

func TestEvaluationBeta(t *testing.T) {

	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	beta, err_beta := hmm.beta(obs)
	if err_beta != nil {
		t.Fatal(err_beta)
	}
	message := "Error in beta"
	CompareSliceFloat(t, beta01, beta.ElementsVector(), message)
}

func TestEvaluationGamma(t *testing.T) {

	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	alpha, _, err_alpha := hmm.alpha(obs)
	if err_alpha != nil {
		t.Fatal(err_alpha)
	}
	beta, err_beta := hmm.beta(obs)
	if err_beta != nil {
		t.Fatal(err_beta)
	}
	gamma, err_gamma := hmm.gamma(alpha, beta)
	if err_gamma != nil {
		t.Fatal(err_gamma)
	}
	message := "Error in gamma"
	CompareSliceFloat(t, gamma01, gamma.ElementsVector(), message)
}

func Convert3DSlideTo1D(s3 [][][]float64) []float64 {
	s1 := make([]float64, 0, 100)
	for _, v1 := range s3 {
		for _, v2 := range v1 {
			for _, v3 := range v2 {
				s1 = append(s1, v3)
			}
		}
	}
	return s1
}

func TestEvaluationXi(t *testing.T) {
	hmm := MakeHMM(t)
	obs := MakeNewDenseMatrix(t, obs0)
	alpha, _, err_alpha := hmm.alpha(obs)
	if err_alpha != nil {
		t.Fatal(err_alpha)
	}
	beta, err_beta := hmm.beta(obs)
	if err_beta != nil {
		t.Fatal(err_beta)
	}
	xi, err_xi := hmm.xi(obs, alpha, beta)
	if err_xi != nil {
		t.Fatal(err_xi)
	}
	xsi1 := Convert3DSlideTo1D(xi)
	message := "Error in xi"
	CompareSliceFloat(t, xsi, xsi1, message)
}
<|endoftext|>


<commit_before>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restapi

import (
	"encoding/json"
	"github.com/cloudawan/cloudone/deploy"
	"github.com/cloudawan/cloudone/image"
	"github.com/cloudawan/cloudone/utility/configuration"
	"github.com/emicklei/go-restful"
	"net/http"
)

type ImageInformationCreateInput struct {
	Name           string
	Kind           string
	Description    string
	CurrentVersion string
	BuildParameter interface{}
}

type ImageInformationUpgradeInput struct {
	ImageInformationName string
	Description          string
}

func registerWebServiceImageInformation() {
	ws := new(restful.WebService)
	ws.Path("/api/v1/imageinformations")
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_JSON)
	restful.Add(ws)

	ws.Route(ws.GET("/").Filter(authorize).Filter(auditLog).To(getAllImageInformation).
		Doc("Get all of the image information").
		Do(returns200AllImageInformation, returns422, returns500))

	ws.Route(ws.DELETE("/{imageinformationname}").Filter(authorize).Filter(auditLog).To(deleteImageInformationAndRelatedRecords).
		Doc("Delete image information and related records").
		Param(ws.PathParameter("imageinformationname", "Image information name").DataType("string")).
		Do(returns200, returns400, returns422, returns500))

	ws.Route(ws.POST("/create").Filter(authorize).Filter(auditLog).To(postImageInformationCreate).
		Doc("Create image build from source code").
		Do(returns200, returns400, returns422, returns500).
		Reads(ImageInformationCreateInput{}))

	ws.Route(ws.PUT("/upgrade").Filter(authorize).Filter(auditLog).To(putImageInformationUpgrade).
		Doc("Upgrade image build from source code").
		Do(returns200, returns400, returns404, returns422, returns500).
		Reads(ImageInformationUpgradeInput{}))
}

func getAllImageInformation(request *restful.Request, response *restful.Response) {
	imageInformationSlice, err := image.GetStorage().LoadAllImageInformation()
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Get all image information failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}

	response.WriteJson(imageInformationSlice, "[]InformationSlice")
}

func deleteImageInformationAndRelatedRecords(request *restful.Request, response *restful.Response) {
	imageInformationName := request.PathParameter("imageinformationname")

	used, err := deploy.IsImageInformationUsed(imageInformationName)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Check whether image information is used failure"
		jsonMap["ErrorMessage"] = err.Error()
		jsonMap["imageInformationName"] = imageInformationName
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}
	if used {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Can't delete the used image information"
		jsonMap["imageInformationName"] = imageInformationName
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(400, string(errorMessageByteSlice))
		return
	}

	err = image.DeleteImageInformationAndRelatedRecord(imageInformationName)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Delete image information and related records failure"
		jsonMap["ErrorMessage"] = err.Error()
		jsonMap["imageInformationName"] = imageInformationName
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}
}

func postImageInformationCreate(request *restful.Request, response *restful.Response) {
	imageInformation := new(image.ImageInformation)
	err := request.ReadEntity(&imageInformation)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Read body failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(400, string(errorMessageByteSlice))
		return
	}

	// Save image information before finish the building
	err = image.GetStorage().SaveImageInformation(imageInformation)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Save image information failure"
		jsonMap["ErrorMessage"] = err.Error()
		jsonMap["imageInformation"] = imageInformation
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}

	// Initialize the output file for websocket
	image.TouchOutMessageFile(imageInformation.Name)

	go func() {
		image.BuildCreate(imageInformation)
	}()
	/*
		outputMessage, err := image.BuildCreate(imageInformation)

		resultJsonMap := make(map[string]interface{})
		resultJsonMap["OutputMessage"] = outputMessage
		statusCode := 200
		if err != nil {
			statusCode = 422
			resultJsonMap["Error"] = "Create build failure"
			resultJsonMap["ErrorMessage"] = err.Error()
			resultJsonMap["imageInformation"] = imageInformation
			log.Error(resultJsonMap)
		}
		result, err := json.Marshal(resultJsonMap)
		if err != nil {
			jsonMap := make(map[string]interface{})
			jsonMap["Error"] = "Marshal output message failure"
			jsonMap["ErrorMessage"] = err.Error()
			jsonMap["resultJsonMap"] = resultJsonMap
			errorMessageByteSlice, _ := json.Marshal(jsonMap)
			log.Error(jsonMap)
			response.WriteErrorString(422, string(errorMessageByteSlice))
			return
		}

		response.WriteErrorString(statusCode, string(result))
		return
	*/
}

func putImageInformationUpgrade(request *restful.Request, response *restful.Response) {
	kubeApiServerEndPoint, kubeApiServerToken, err := configuration.GetAvailablekubeApiServerEndPoint()
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Get kube apiserver endpoint and token failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(404, string(errorMessageByteSlice))
		return
	}

	imageInformationUpgradeInput := new(ImageInformationUpgradeInput)
	err = request.ReadEntity(&imageInformationUpgradeInput)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Read body failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(400, string(errorMessageByteSlice))
		return
	}

	// Initialize the output file for websocket
	image.TouchOutMessageFile(imageInformationUpgradeInput.ImageInformationName)

	go func() {
		_, err := image.BuildUpgrade(
			imageInformationUpgradeInput.ImageInformationName,
			imageInformationUpgradeInput.Description)
		if err == nil {
			// Build sccessfully
			// Auto rolling update the deployment if configured
			imageInformation, err := image.GetStorage().LoadImageInformation(imageInformationUpgradeInput.ImageInformationName)
			if err != nil {
				log.Error(err)
			} else {
				deployInformationSlice, err := deploy.GetDeployInformationWithAutoUpdateForNewBuild(imageInformationUpgradeInput.ImageInformationName)
				if err != nil {
					log.Error(err)
				} else {
					for _, deployInformation := range deployInformationSlice {
						description := "Trigged by version " + imageInformation.CurrentVersion
						err := deploy.DeployUpdate(
							kubeApiServerEndPoint,
							kubeApiServerToken,
							deployInformation.Namespace,
							imageInformation.Name,
							imageInformation.CurrentVersion,
							description,
							deployInformation.EnvironmentSlice)
						if err != nil {
							log.Error(err)
						}
					}
				}
			}
		}
	}()
	/*
		outputMessage, err := image.BuildUpgrade(
			imageInformationUpgradeInput.ImageInformationName,
			imageInformationUpgradeInput.Description)

		resultJsonMap := make(map[string]interface{})
		resultJsonMap["OutputMessage"] = outputMessage
		statusCode := 200
		if err != nil {
			statusCode = 422
			resultJsonMap["Error"] = "Upgrade build failure"
			resultJsonMap["ErrorMessage"] = err.Error()
			resultJsonMap["imageInformationUpgradeInput"] = imageInformationUpgradeInput
			log.Error(resultJsonMap)
		}
		result, err := json.Marshal(resultJsonMap)
		if err != nil {
			jsonMap := make(map[string]interface{})
			jsonMap["Error"] = "Marshal output message failure"
			jsonMap["ErrorMessage"] = err.Error()
			jsonMap["resultJsonMap"] = resultJsonMap
			errorMessageByteSlice, _ := json.Marshal(jsonMap)
			log.Error(jsonMap)
			response.WriteErrorString(422, string(errorMessageByteSlice))
			return
		}

		response.WriteErrorString(statusCode, string(result))
		return
	*/
}

func returns200AllImageInformation(b *restful.RouteBuilder) {
	b.Returns(http.StatusOK, "OK", []image.ImageInformation{})
}
<commit_msg>Check whether the repository is created<commit_after>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restapi

import (
	"encoding/json"
	"github.com/cloudawan/cloudone/deploy"
	"github.com/cloudawan/cloudone/image"
	"github.com/cloudawan/cloudone/utility/configuration"
	"github.com/emicklei/go-restful"
	"net/http"
)

type ImageInformationCreateInput struct {
	Name           string
	Kind           string
	Description    string
	CurrentVersion string
	BuildParameter interface{}
}

type ImageInformationUpgradeInput struct {
	ImageInformationName string
	Description          string
}

func registerWebServiceImageInformation() {
	ws := new(restful.WebService)
	ws.Path("/api/v1/imageinformations")
	ws.Consumes(restful.MIME_JSON)
	ws.Produces(restful.MIME_JSON)
	restful.Add(ws)

	ws.Route(ws.GET("/").Filter(authorize).Filter(auditLog).To(getAllImageInformation).
		Doc("Get all of the image information").
		Do(returns200AllImageInformation, returns422, returns500))

	ws.Route(ws.DELETE("/{imageinformationname}").Filter(authorize).Filter(auditLog).To(deleteImageInformationAndRelatedRecords).
		Doc("Delete image information and related records").
		Param(ws.PathParameter("imageinformationname", "Image information name").DataType("string")).
		Do(returns200, returns400, returns422, returns500))

	ws.Route(ws.POST("/create").Filter(authorize).Filter(auditLog).To(postImageInformationCreate).
		Doc("Create image build from source code").
		Do(returns200, returns400, returns403, returns422, returns500).
		Reads(ImageInformationCreateInput{}))

	ws.Route(ws.PUT("/upgrade").Filter(authorize).Filter(auditLog).To(putImageInformationUpgrade).
		Doc("Upgrade image build from source code").
		Do(returns200, returns400, returns404, returns422, returns500).
		Reads(ImageInformationUpgradeInput{}))
}

func getAllImageInformation(request *restful.Request, response *restful.Response) {
	imageInformationSlice, err := image.GetStorage().LoadAllImageInformation()
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Get all image information failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}

	response.WriteJson(imageInformationSlice, "[]InformationSlice")
}

func deleteImageInformationAndRelatedRecords(request *restful.Request, response *restful.Response) {
	imageInformationName := request.PathParameter("imageinformationname")

	used, err := deploy.IsImageInformationUsed(imageInformationName)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Check whether image information is used failure"
		jsonMap["ErrorMessage"] = err.Error()
		jsonMap["imageInformationName"] = imageInformationName
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}
	if used {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Can't delete the used image information"
		jsonMap["imageInformationName"] = imageInformationName
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(400, string(errorMessageByteSlice))
		return
	}

	err = image.DeleteImageInformationAndRelatedRecord(imageInformationName)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Delete image information and related records failure"
		jsonMap["ErrorMessage"] = err.Error()
		jsonMap["imageInformationName"] = imageInformationName
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}
}

func postImageInformationCreate(request *restful.Request, response *restful.Response) {
	imageInformation := new(image.ImageInformation)
	err := request.ReadEntity(&imageInformation)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Read body failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(400, string(errorMessageByteSlice))
		return
	}

	oldImageInformation, _ := image.GetStorage().LoadImageInformation(imageInformation.Name)
	if oldImageInformation != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Duplicated repository error"
		jsonMap["ErrorMessage"] = "Already exists"
		jsonMap["imageInformation"] = imageInformation
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(401, string(errorMessageByteSlice))
		return
	}

	// Save image information before finish the building
	err = image.GetStorage().SaveImageInformation(imageInformation)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Save image information failure"
		jsonMap["ErrorMessage"] = err.Error()
		jsonMap["imageInformation"] = imageInformation
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(422, string(errorMessageByteSlice))
		return
	}

	// Initialize the output file for websocket
	image.TouchOutMessageFile(imageInformation.Name)

	go func() {
		image.BuildCreate(imageInformation)
	}()
	/*
		outputMessage, err := image.BuildCreate(imageInformation)

		resultJsonMap := make(map[string]interface{})
		resultJsonMap["OutputMessage"] = outputMessage
		statusCode := 200
		if err != nil {
			statusCode = 422
			resultJsonMap["Error"] = "Create build failure"
			resultJsonMap["ErrorMessage"] = err.Error()
			resultJsonMap["imageInformation"] = imageInformation
			log.Error(resultJsonMap)
		}
		result, err := json.Marshal(resultJsonMap)
		if err != nil {
			jsonMap := make(map[string]interface{})
			jsonMap["Error"] = "Marshal output message failure"
			jsonMap["ErrorMessage"] = err.Error()
			jsonMap["resultJsonMap"] = resultJsonMap
			errorMessageByteSlice, _ := json.Marshal(jsonMap)
			log.Error(jsonMap)
			response.WriteErrorString(422, string(errorMessageByteSlice))
			return
		}

		response.WriteErrorString(statusCode, string(result))
		return
	*/
}

func putImageInformationUpgrade(request *restful.Request, response *restful.Response) {
	kubeApiServerEndPoint, kubeApiServerToken, err := configuration.GetAvailablekubeApiServerEndPoint()
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Get kube apiserver endpoint and token failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(404, string(errorMessageByteSlice))
		return
	}

	imageInformationUpgradeInput := new(ImageInformationUpgradeInput)
	err = request.ReadEntity(&imageInformationUpgradeInput)
	if err != nil {
		jsonMap := make(map[string]interface{})
		jsonMap["Error"] = "Read body failure"
		jsonMap["ErrorMessage"] = err.Error()
		errorMessageByteSlice, _ := json.Marshal(jsonMap)
		log.Error(jsonMap)
		response.WriteErrorString(400, string(errorMessageByteSlice))
		return
	}

	// Initialize the output file for websocket
	image.TouchOutMessageFile(imageInformationUpgradeInput.ImageInformationName)

	go func() {
		_, err := image.BuildUpgrade(
			imageInformationUpgradeInput.ImageInformationName,
			imageInformationUpgradeInput.Description)
		if err == nil {
			// Build sccessfully
			// Auto rolling update the deployment if configured
			imageInformation, err := image.GetStorage().LoadImageInformation(imageInformationUpgradeInput.ImageInformationName)
			if err != nil {
				log.Error(err)
			} else {
				deployInformationSlice, err := deploy.GetDeployInformationWithAutoUpdateForNewBuild(imageInformationUpgradeInput.ImageInformationName)
				if err != nil {
					log.Error(err)
				} else {
					for _, deployInformation := range deployInformationSlice {
						description := "Trigged by version " + imageInformation.CurrentVersion
						err := deploy.DeployUpdate(
							kubeApiServerEndPoint,
							kubeApiServerToken,
							deployInformation.Namespace,
							imageInformation.Name,
							imageInformation.CurrentVersion,
							description,
							deployInformation.EnvironmentSlice)
						if err != nil {
							log.Error(err)
						}
					}
				}
			}
		}
	}()
	/*
		outputMessage, err := image.BuildUpgrade(
			imageInformationUpgradeInput.ImageInformationName,
			imageInformationUpgradeInput.Description)

		resultJsonMap := make(map[string]interface{})
		resultJsonMap["OutputMessage"] = outputMessage
		statusCode := 200
		if err != nil {
			statusCode = 422
			resultJsonMap["Error"] = "Upgrade build failure"
			resultJsonMap["ErrorMessage"] = err.Error()
			resultJsonMap["imageInformationUpgradeInput"] = imageInformationUpgradeInput
			log.Error(resultJsonMap)
		}
		result, err := json.Marshal(resultJsonMap)
		if err != nil {
			jsonMap := make(map[string]interface{})
			jsonMap["Error"] = "Marshal output message failure"
			jsonMap["ErrorMessage"] = err.Error()
			jsonMap["resultJsonMap"] = resultJsonMap
			errorMessageByteSlice, _ := json.Marshal(jsonMap)
			log.Error(jsonMap)
			response.WriteErrorString(422, string(errorMessageByteSlice))
			return
		}

		response.WriteErrorString(statusCode, string(result))
		return
	*/
}

func returns200AllImageInformation(b *restful.RouteBuilder) {
	b.Returns(http.StatusOK, "OK", []image.ImageInformation{})
}
<|endoftext|>


<commit_before>// Package metric provides tools for comparing datasets.
package metric

import (
	"math"

	"github.com/ready-steady/linear/metric"
	"github.com/ready-steady/sort"
	"github.com/ready-steady/statistics/distribution"
)

// MSE computes the mean-square error.
//
// https://en.wikipedia.org/wiki/Mean_squared_error
func MSE(y, yhat []float64) float64 {
	Σ := 0.0
	for i := range y {
		Δ := yhat[i] - y[i]
		Σ += Δ * Δ
	}
	return Σ / float64(len(y))
}

// RMSE computes the root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation
func RMSE(y, yhat []float64) float64 {
	return math.Sqrt(MSE(y, yhat))
}

// NRMSE computes the normalized root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation#Normalized_root-mean-square_deviation
func NRMSE(y, yhat []float64) float64 {
	count := len(y)

	min, max := y[0], y[0]
	for i := 1; i < count; i++ {
		if y[i] < min {
			min = y[i]
		}
		if y[i] > max {
			max = y[i]
		}
	}

	return RMSE(y, yhat) / (max - min)
}

// KolmogorovSmirnov computes the Kolmogorov–Smirnov statistic for two samples.
//
// https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
func KolmogorovSmirnov(data1, data2 []float64) float64 {
	edges := detect(data1, data2)

	cdf1 := distribution.CDF(data1, edges)
	cdf2 := distribution.CDF(data2, edges)

	return metric.Uniform(cdf1, cdf2)
}

// KullbackLeibler computes the Kullback–Leibler divergence of q from p where p
// and q are two discrete probability distributions. The distribution p is
// assumed to be absolutely continuous with respect the distribution q, that is,
// q[i] = 0 implies that p[i] = 0.
//
// https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
func KullbackLeibler(p, q []float64) float64 {
	divergence := 0.0
	for i := range p {
		if p[i] > 0 {
			divergence += p[i] * math.Log(p[i]/q[i])
		}
	}
	return divergence
}

func detect(data1, data2 []float64) []float64 {
	n1, n2 := len(data1), len(data2)

	edges := make([]float64, 1+n1+n2+1)
	edges[0] = math.Inf(-1)
	copy(edges[1:], data1)
	copy(edges[1+n1:], data2)
	edges[1+n1+n2] = -edges[0]

	return sort.Unique(edges)
}
<commit_msg>metric: add MSPE and RMSPE<commit_after>// Package metric provides tools for comparing datasets.
package metric

import (
	"math"

	"github.com/ready-steady/linear/metric"
	"github.com/ready-steady/sort"
	"github.com/ready-steady/statistics/distribution"
)

// MSE computes the mean-square error.
//
// https://en.wikipedia.org/wiki/Mean_squared_error
func MSE(y, yhat []float64) float64 {
	Σ := 0.0
	for i := range y {
		ε := yhat[i] - y[i]
		Σ += ε * ε
	}
	return Σ / float64(len(y))
}

// MSPE computes the mean-square-percentage error.
func MSPE(y, yhat []float64) float64 {
	Σ := 0.0
	for i := range y {
		ε := (yhat[i] - y[i]) / y[i]
		Σ += ε * ε
	}
	return Σ / float64(len(y))
}

// RMSE computes the root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation
func RMSE(y, yhat []float64) float64 {
	return math.Sqrt(MSE(y, yhat))
}

// RMSPE computes the root-mean-square-percentage error.
func RMSPE(y, yhat []float64) float64 {
	return math.Sqrt(MSPE(y, yhat))
}

// NRMSE computes the normalized root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation#Normalized_root-mean-square_deviation
func NRMSE(y, yhat []float64) float64 {
	count := len(y)

	min, max := y[0], y[0]
	for i := 1; i < count; i++ {
		if y[i] < min {
			min = y[i]
		}
		if y[i] > max {
			max = y[i]
		}
	}

	return RMSE(y, yhat) / (max - min)
}

// KolmogorovSmirnov computes the Kolmogorov–Smirnov statistic for two samples.
//
// https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
func KolmogorovSmirnov(data1, data2 []float64) float64 {
	edges := detect(data1, data2)

	cdf1 := distribution.CDF(data1, edges)
	cdf2 := distribution.CDF(data2, edges)

	return metric.Uniform(cdf1, cdf2)
}

// KullbackLeibler computes the Kullback–Leibler divergence of q from p where p
// and q are two discrete probability distributions. The distribution p is
// assumed to be absolutely continuous with respect the distribution q, that is,
// q[i] = 0 implies that p[i] = 0.
//
// https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
func KullbackLeibler(p, q []float64) float64 {
	divergence := 0.0
	for i := range p {
		if p[i] > 0 {
			divergence += p[i] * math.Log(p[i]/q[i])
		}
	}
	return divergence
}

func detect(data1, data2 []float64) []float64 {
	n1, n2 := len(data1), len(data2)

	edges := make([]float64, 1+n1+n2+1)
	edges[0] = math.Inf(-1)
	copy(edges[1:], data1)
	copy(edges[1+n1:], data2)
	edges[1+n1+n2] = -edges[0]

	return sort.Unique(edges)
}
<|endoftext|>


<commit_before>package models

import (
	"encoding/json"
	"log"
	"time"

	"github.com/go-pg/pg/orm"
	"github.com/harrisbaird/dailyteedeals/utils"
)

type SpiderItem struct {
	ID          int
	SpiderJobID int
	ProductID   int
	ItemData    string
	Error       string
	CreatedAt   time.Time

	SpiderJob *SpiderJob
	Product   *Product
}

func FindSpiderItem(db orm.DB, id int) (*SpiderItem, error) {
	var item SpiderItem
	err := db.Model(&item).Column("spider_item.*", "SpiderJob").Where("spider_item.id=?", id).First()
	return &item, err
}

func CreateSpiderItem(db orm.DB, spiderJobID int, data string) (*SpiderItem, error) {
	item := SpiderItem{SpiderJobID: spiderJobID, ItemData: data}
	err := db.Insert(&item)
	return &item, err
}

func (item *SpiderItem) UpdateError(db orm.DB, err error) {
	item.Error = err.Error()
	if err := db.Update(item); err != nil {
		log.Println(err.Error())
	}
}

func (item *SpiderItem) ParseItemData(db orm.DB, minioConn *utils.MinioConnection) error {
	var data ScrapydItem
	if err := json.Unmarshal([]byte(item.ItemData), &data); err != nil {
		return err
	}

	artist, err := FindOrCreateArtist(db, data.ArtistName, data.ArtistUrls)
	if err != nil {
		return err
	}

	design, err := FindOrCreateDesign(db, artist.ID, data.Name)
	if err != nil {
		return err
	}

	product := Product{
		DesignID:   design.ID,
		SiteID:     item.SpiderJob.SiteID,
		URL:        data.URL,
		Prices:     data.StringPrices(),
		Active:     data.Active,
		Deal:       data.Deal,
		LastChance: data.LastChance,
		Tags:       data.Tags,
	}

	_, err = db.Model(&product).OnConflict("(design_id, site_id) DO UPDATE").
		Set("url = ?url, prices = ?prices, active = ?active, deal = ?deal, last_chance = ?last_chance").
		Insert()
	if err != nil {
		log.Println(err)
		return err
	}

	db.Model(item).Set("product_id=?", product.ID).Update()

	return product.UpdateImageIfExpired(db, minioConn, data.ImageURL)
}
<commit_msg>Log failure<commit_after>package models

import (
	"encoding/json"
	"log"
	"time"

	"github.com/go-pg/pg/orm"
	"github.com/harrisbaird/dailyteedeals/utils"
)

type SpiderItem struct {
	ID          int
	SpiderJobID int
	ProductID   int
	ItemData    string
	Error       string
	CreatedAt   time.Time

	SpiderJob *SpiderJob
	Product   *Product
}

func FindSpiderItem(db orm.DB, id int) (*SpiderItem, error) {
	var item SpiderItem
	err := db.Model(&item).Column("spider_item.*", "SpiderJob").Where("spider_item.id=?", id).First()
	return &item, err
}

func CreateSpiderItem(db orm.DB, spiderJobID int, data string) (*SpiderItem, error) {
	item := SpiderItem{SpiderJobID: spiderJobID, ItemData: data}
	err := db.Insert(&item)
	return &item, err
}

func (item *SpiderItem) UpdateError(db orm.DB, err error) {
	item.Error = err.Error()
	if err := db.Update(item); err != nil {
		log.Println(err.Error())
	}
}

func (item *SpiderItem) ParseItemData(db orm.DB, minioConn *utils.MinioConnection) error {
	var data ScrapydItem
	if err := json.Unmarshal([]byte(item.ItemData), &data); err != nil {
		return err
	}

	artist, err := FindOrCreateArtist(db, data.ArtistName, data.ArtistUrls)
	if err != nil {
		return err
	}

	design, err := FindOrCreateDesign(db, artist.ID, data.Name)
	if err != nil {
		return err
	}

	product := Product{
		DesignID:   design.ID,
		SiteID:     item.SpiderJob.SiteID,
		URL:        data.URL,
		Prices:     data.StringPrices(),
		Active:     data.Active,
		Deal:       data.Deal,
		LastChance: data.LastChance,
		Tags:       data.Tags,
	}

	_, err = db.Model(&product).OnConflict("(design_id, site_id) DO UPDATE").
		Set("url = ?url, prices = ?prices, active = ?active, deal = ?deal, last_chance = ?last_chance").
		Insert()
	if err != nil {
		log.Println(err)
		return err
	}

	_, err = db.Model(item).Set("product_id=?", product.ID).Update()
	if err != nil {
		log.Printf("Unable to update SpiderItem productID - %s\n", err.Error())
	}

	return product.UpdateImageIfExpired(db, minioConn, data.ImageURL)
}
<|endoftext|>


<commit_before>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"fmt"
	"github.com/oxtoacart/go-ringbuffer/ringbuff"
	"io"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	streamingInterval = 30 * time.Second
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
	updatesCacheSize      = 1 * 24 * 2 // 1 day worth of updates at 30 minute intervals
	oldUpdates            = ringbuff.New(updatesCacheSize)
)

type streamingClient struct {
	ws      *websocket.Conn
	updates chan *streamingUpdate
	id      chan int
}

type streamingUpdate struct {
	asOf    time.Time
	allDims map[string]*json.RawMessage // using pointer because of this bug - https://code.google.com/p/go/issues/detail?id=6528
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	AsOfSeconds int64                       `json:"asOfSeconds"`
	Dims        map[string]*json.RawMessage `json:"dims"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(streamingInterval).Add(streamingInterval)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId

			log.Printf("Buffered updates count: %d", oldUpdates.Size())

			// Send buffered updates to client
			oldUpdates.ForEach(func(item interface{}) {
				update := item.(*streamingUpdate)
				client.updates <- update
			})
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Encode dims as JSON
				jsonDims := make(map[string]*json.RawMessage)
				for name, dim := range dims {
					encoded, err := json.Marshal(dim)
					if err != nil {
						log.Printf("Unable to json encode dim %s: %s", name, err)
					} else {
						raw := json.RawMessage(encoded)
						jsonDims[name] = &raw
					}
				}

				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, allDims: jsonDims}
				for _, client := range streamingClients {
					client.updates <- update
				}

				// Remember updates at 30 minute intervals
				if update.asOf.Truncate(15*time.Second) == update.asOf {
					log.Printf("Snapshotting streaming stats at %s", update.asOf)
					oldUpdates.Add(update)
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	dim, err := extractid(ws.Request())
	if err != nil {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Unable to extract id from request: %s", err)})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	dimNames := dimNamesFor(dim)

	client := &streamingClient{ws: ws, updates: make(chan *streamingUpdate, updatesCacheSize*2), id: make(chan int)}

	var wg sync.WaitGroup
	wg.Add(1)

	// Write updates to the client
	go func() {
		for {
			// This gets data for all dims
			update := <-client.updates
			dims := update.allDims
			if dimNames != nil {
				// Grab only the dimensions that this client is interested in
				dims = make(map[string]*json.RawMessage)
				for _, name := range dimNames {
					dims[name] = update.allDims[name]
				}
			}
			resp := &StreamingQueryResponse{Response: Response{Succeeded: true}, AsOfSeconds: update.asOf.Unix(), Dims: dims}
			encoded, err := json.Marshal(resp)
			if err != nil {
				log.Printf("Unable to marshal json: %s", err)
			} else {
				ws.Write(encoded)
			}
		}
	}()

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}
<commit_msg>Stat snapshots fixed<commit_after>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"fmt"
	"github.com/oxtoacart/go-ringbuffer/ringbuff"
	"io"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	streamingInterval = 30 * time.Second
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
	updatesCacheSize      = 1 * 24 * 2 // 1 day worth of updates at 30 minute intervals
	oldUpdates            = ringbuff.New(updatesCacheSize)
)

type streamingClient struct {
	ws      *websocket.Conn
	updates chan *streamingUpdate
	id      chan int
}

type streamingUpdate struct {
	asOf    time.Time
	allDims map[string]*json.RawMessage // using pointer because of this bug - https://code.google.com/p/go/issues/detail?id=6528
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	AsOfSeconds int64                       `json:"asOfSeconds"`
	Dims        map[string]*json.RawMessage `json:"dims"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(streamingInterval).Add(streamingInterval)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId

			// Send buffered updates to client
			oldUpdates.ForEach(func(item interface{}) {
				update := item.(*streamingUpdate)
				client.updates <- update
			})
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Encode dims as JSON
				jsonDims := make(map[string]*json.RawMessage)
				for name, dim := range dims {
					encoded, err := json.Marshal(dim)
					if err != nil {
						log.Printf("Unable to json encode dim %s: %s", name, err)
					} else {
						raw := json.RawMessage(encoded)
						jsonDims[name] = &raw
					}
				}

				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, allDims: jsonDims}
				for _, client := range streamingClients {
					client.updates <- update
				}

				// Remember updates at 30 minute intervals
				if update.asOf.Truncate(30*time.Minute) == update.asOf {
					log.Printf("Snapshotting streaming stats at %s", update.asOf)
					oldUpdates.Add(update)
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	dim, err := extractid(ws.Request())
	if err != nil {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Unable to extract id from request: %s", err)})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	dimNames := dimNamesFor(dim)

	client := &streamingClient{ws: ws, updates: make(chan *streamingUpdate, updatesCacheSize*2), id: make(chan int)}

	var wg sync.WaitGroup
	wg.Add(1)

	// Write updates to the client
	go func() {
		for {
			// This gets data for all dims
			update := <-client.updates
			dims := update.allDims
			if dimNames != nil {
				// Grab only the dimensions that this client is interested in
				dims = make(map[string]*json.RawMessage)
				for _, name := range dimNames {
					dims[name] = update.allDims[name]
				}
			}
			resp := &StreamingQueryResponse{Response: Response{Succeeded: true}, AsOfSeconds: update.asOf.Unix(), Dims: dims}
			encoded, err := json.Marshal(resp)
			if err != nil {
				log.Printf("Unable to marshal json: %s", err)
			} else {
				ws.Write(encoded)
			}
		}
	}()

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}
<|endoftext|>


<commit_before>package miggle

import (
	"fmt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"
)

func init() {
	http.Handle("/", Router())
}

func Router() *mux.Router {
	var res *Resource = NewResource()
	r := mux.NewRouter()

	r.HandleFunc("/test", res.RequestHandler(new(EndPoint)))
	r.HandleFunc("/get/status/{status:[0-9]+}", res.RequestHandler(new(EndPointStatus)))

	return r
}

type EndPoint struct{}

func (this *EndPoint) Init(request *http.Request) error {
	return nil
}

func (this *EndPoint) Get(request *http.Request) (int, interface{}, http.Header) {
	return 200, "data", nil
}

func TestBasicGet(t *testing.T) {
	r, err := http.NewRequest("GET", "/test", nil)
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	Router().ServeHTTP(w, r)
	if w.Code != http.StatusOK {
		t.Error("Response status code does not equal")
	}

	body, _ := ioutil.ReadAll(w.Body)

	if string(body) != "data" {
		t.Error("Response body does not equal")
	}

	if header, ok := w.HeaderMap["Content-Type"]; ok {
		if header[0] != "plain/text" {
			t.Error("Response header Content-Type: %s does not exists", header[0])
		}
	} else {
		t.Error("Response header Content-Type: does not exists", header[0])
	}
}

type EndPointStatus struct {
	Status int
}

func (this *EndPointStatus) Init(request *http.Request) error {
	vars := mux.Vars(request)
	if id, ok := vars["status"]; ok {
		status, err := strconv.Atoi(id)

		if err != nil {
			return err
		}
		this.Status = status
	}
	return nil
}

func (this *EndPointStatus) Get(request *http.Request) (int, interface{}, http.Header) {
	return this.Status, "data", http.Header{"Content-Type": {"application/json; charset=utf-8"}}
}

func TestAdvanceGet(t *testing.T) {
	statuses := []int{100, 200, 300, 400, 404, 500}
	for _, status := range statuses {
		r, err := http.NewRequest("GET", fmt.Sprintf("/get/status/%d", status), nil)
		if err != nil {
			t.Error(err)
		}

		w := httptest.NewRecorder()
		Router().ServeHTTP(w, r)
		if w.Code != status {
			t.Error("Response status code does not equal")
		}
		body, _ := ioutil.ReadAll(w.Body)

		if string(body) != "data" {
			t.Error("Response body does not equal")
		}

		if header, ok := w.HeaderMap["Content-Type"]; ok {
			if header[0] != "application/json; charset=utf-8" {
				t.Error("Response header Content-Type: %s does not exists", header[0])
			}
		} else {
			t.Error("Response header Content-Type' does not exists", header)
		}
	}

	r, err := http.NewRequest("GET", "/get/status/string", nil)
	if err != nil {
		t.Error(err)
	}

	w := httptest.NewRecorder()
	Router().ServeHTTP(w, r)
	if w.Code != 404 {
		t.Error("Response status code does not equal")
	}

}
<commit_msg>Add tests for POST http method<commit_after>package miggle

import (
	"fmt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"testing"
)

func init() {
	http.Handle("/", Router())
}

func Router() *mux.Router {
	var res *Resource = NewResource()
	r := mux.NewRouter()

	r.HandleFunc("/test", res.RequestHandler(new(EndPoint)))
	r.HandleFunc("/status/{status:[0-9]+}", res.RequestHandler(new(EndPointAdvance)))
	return r
}

type EndPoint struct{}

func (this *EndPoint) Init(request *http.Request) error {
	return nil
}

func (this *EndPoint) Get(request *http.Request) (int, interface{}, http.Header) {
	return 200, "data", nil
}

func TestBasicGet(t *testing.T) {
	r, err := http.NewRequest("GET", "/test", nil)
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	Router().ServeHTTP(w, r)
	if w.Code != http.StatusOK {
		t.Error("Response status code does not equal")
	}

	body, _ := ioutil.ReadAll(w.Body)

	if string(body) != "data" {
		t.Error("Response body does not equal")
	}

	if header, ok := w.HeaderMap["Content-Type"]; ok {
		if header[0] != "plain/text" {
			t.Error("Response header Content-Type: %s does not exists", header[0])
		}
	} else {
		t.Error("Response header Content-Type: does not exists", header[0])
	}
}

type EndPointAdvance struct {
	Status int
}

func (this *EndPointAdvance) Init(request *http.Request) error {
	vars := mux.Vars(request)
	if id, ok := vars["status"]; ok {
		status, err := strconv.Atoi(id)

		if err != nil {
			return err
		}
		this.Status = status
	}
	return nil
}

func (this *EndPointAdvance) Get(request *http.Request) (int, interface{}, http.Header) {
	return this.Status, "data", http.Header{"Content-Type": {"application/json; charset=utf-8"}}
}

func (this *EndPointAdvance) Post(request *http.Request) (int, interface{}, http.Header) {
	data, _ := ioutil.ReadAll(request.Body)
	return this.Status, data, http.Header{"Content-Type": {"application/json; charset=utf-8"}}
}

func TestAdvanceGet(t *testing.T) {
	statuses := []int{100, 200, 300, 400, 404, 500}
	for _, status := range statuses {
		r, err := http.NewRequest("GET", fmt.Sprintf("/status/%d", status), nil)
		if err != nil {
			t.Error(err)
		}

		w := httptest.NewRecorder()
		Router().ServeHTTP(w, r)
		if w.Code != status {
			t.Error("Response status code does not equal")
		}
		body, _ := ioutil.ReadAll(w.Body)

		if string(body) != "data" {
			t.Error("Response body does not equal")
		}

		if header, ok := w.HeaderMap["Content-Type"]; ok {
			if header[0] != "application/json; charset=utf-8" {
				t.Error("Response header Content-Type: %s does not exists", header[0])
			}
		} else {
			t.Error("Response header Content-Type' does not exists", header)
		}
	}

	r, err := http.NewRequest("GET", "/status/not_a_number", nil)
	if err != nil {
		t.Error(err)
	}

	w := httptest.NewRecorder()
	Router().ServeHTTP(w, r)
	if w.Code != 404 {
		t.Error("Response status code should be 404")
	}

}

func TestAdvancePost(t *testing.T) {
	status := 201
	data := "data data"
	r, err := http.NewRequest("POST", fmt.Sprintf("/status/%d", status), strings.NewReader(data))
	if err != nil {
		t.Error(err)
	}

	w := httptest.NewRecorder()
	Router().ServeHTTP(w, r)
	if w.Code != status {
		t.Error("Response status code does not equal")
	}
	body, _ := ioutil.ReadAll(w.Body)

	if string(body) != data {
		t.Error("Response body does not equal")
	}

	if header, ok := w.HeaderMap["Content-Type"]; ok {
		if header[0] != "application/json; charset=utf-8" {
			t.Error("Response header Content-Type: %s does not exists", header[0])
		}
	} else {
		t.Error("Response header Content-Type' does not exists", header)
	}

}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"time"

	"github.com/Machiel/slugify"
	"github.com/jinzhu/gorm"
)

const (
	// BuildUsernameMinLength is the minimum length of the username.
	BuildUsernameMinLength = "3"

	// BuildUsernameMaxLength is the maximum length of the username.
	BuildUsernameMaxLength = "255"
)

// PackDefaultOrder is the default ordering for pack listings.
func BuildDefaultOrder(db *gorm.DB) *gorm.DB {
	return db.Order(
		"builds.name ASC",
	)
}

// Builds is simply a collection of build structs.
type Builds []*Build

// Build represents a build model definition.
type Build struct {
	ID          int64      `json:"id" gorm:"primary_key"`
	Pack        *Pack      `json:"pack"`
	PackID      int64      `json:"pack_id" sql:"index"`
	Minecraft   *Minecraft `json:"minecraft"`
	MinecraftID int64      `json:"minecraft_id" sql:"index"`
	Forge       *Forge     `json:"forge"`
	ForgeID     int64      `json:"forge_id" sql:"index"`
	Slug        string     `json:"slug" sql:"unique_index"`
	Name        string     `json:"name" sql:"unique_index"`
	MinJava     string     `json:"min_java"`
	MinMemory   string     `json:"min_memory"`
	Published   bool       `json:"published" sql:"default:false"`
	Private     bool       `json:"private" sql:"default:false"`
	CreatedAt   time.Time  `json:"created_at"`
	UpdatedAt   time.Time  `json:"updated_at"`
	Mods        *Mods      `json:"mods" gorm:"many2many:build_mods;"`
}

// BeforeSave invokes required actions before persisting.
func (u *Build) BeforeSave(db *gorm.DB) (err error) {
	if u.Slug == "" {

		u.Slug = slugify.Slugify(u.Name)
		// Fill the slug with slugified name

	}

	return nil
}

// Defaults prefills the struct with some default values.
func (u *Build) Defaults() {
	// Currently no default values required.
}

// Validate does some validation to be able to store the record.
func (u *Build) Validate(db *gorm.DB) {
	if u.Name == "" {
		db.AddError(fmt.Errorf("Name is a required attribute"))
	}
}
<commit_msg>Fixed comment within build model<commit_after>package model

import (
	"fmt"
	"time"

	"github.com/Machiel/slugify"
	"github.com/jinzhu/gorm"
)

const (
	// BuildUsernameMinLength is the minimum length of the username.
	BuildUsernameMinLength = "3"

	// BuildUsernameMaxLength is the maximum length of the username.
	BuildUsernameMaxLength = "255"
)

// BuildDefaultOrder is the default ordering for pack listings.
func BuildDefaultOrder(db *gorm.DB) *gorm.DB {
	return db.Order(
		"builds.name ASC",
	)
}

// Builds is simply a collection of build structs.
type Builds []*Build

// Build represents a build model definition.
type Build struct {
	ID          int64      `json:"id" gorm:"primary_key"`
	Pack        *Pack      `json:"pack"`
	PackID      int64      `json:"pack_id" sql:"index"`
	Minecraft   *Minecraft `json:"minecraft"`
	MinecraftID int64      `json:"minecraft_id" sql:"index"`
	Forge       *Forge     `json:"forge"`
	ForgeID     int64      `json:"forge_id" sql:"index"`
	Slug        string     `json:"slug" sql:"unique_index"`
	Name        string     `json:"name" sql:"unique_index"`
	MinJava     string     `json:"min_java"`
	MinMemory   string     `json:"min_memory"`
	Published   bool       `json:"published" sql:"default:false"`
	Private     bool       `json:"private" sql:"default:false"`
	CreatedAt   time.Time  `json:"created_at"`
	UpdatedAt   time.Time  `json:"updated_at"`
	Mods        *Mods      `json:"mods" gorm:"many2many:build_mods;"`
}

// BeforeSave invokes required actions before persisting.
func (u *Build) BeforeSave(db *gorm.DB) (err error) {
	if u.Slug == "" {

		u.Slug = slugify.Slugify(u.Name)
		// Fill the slug with slugified name

	}

	return nil
}

// Defaults prefills the struct with some default values.
func (u *Build) Defaults() {
	// Currently no default values required.
}

// Validate does some validation to be able to store the record.
func (u *Build) Validate(db *gorm.DB) {
	if u.Name == "" {
		db.AddError(fmt.Errorf("Name is a required attribute"))
	}
}
<|endoftext|>


<commit_before>// Package subsub just shows how navigation works for packages under other packages.
//
// Nothing more to see, you've read everything, congrats!
package subsub
<commit_msg>add some wrap up at the end.<commit_after>// Package subsub wraps up the article.
//
// This is the end of the article/repo whatever you want to call it.  I hope you
// learned something.  I certainly did just by writing it all down and doing
// some experimenting to make sure what I thought was true actually was true.
//
// Feel free to send a pull request if you think something can be clarified or
// if I missed anything.
//
// Remember - documentation is how you keep other people from bugging you about
// your code.  It's also an opportunity to explain your rationale behind the API
// you have chosen to expose.  Use examples liberally, often times the functions
// and types alone don't properly convey how to use your code.
package subsub
<|endoftext|>


<commit_before>package model

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/boltdb/bolt"

	"../cfg"
)

const (
	BUCKET_INDEX = "index"
)

type Meta struct {
	Posts     int
	PerPage   int
	Pages     int
	Drafts    int
	UpdatedAt time.Time
}

func (meta *Meta) init() {
	meta.Posts = 0
	meta.PerPage = cfg.Int("ui.per_page")
	meta.Pages = 0
	meta.Drafts = 0
	meta.UpdatedAt = time.Now()
}

func (meta *Meta) Load() {
	meta.init()
	db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte([]byte(BUCKET_INDEX)))
		if bucket != nil {
			jsonMeta := bucket.Get([]byte("meta"))
			json.Unmarshal(jsonMeta, &meta)
		}

		return nil
	})
}

func RebuildIndex() error {
	cfg.Log("Rebuilding index...")

	meta := new(Meta)
	meta.init()

	pathMap := make(map[string]string)
	pageMap := make(map[string][]*Post)
	var draftList []*Post

	if err := db.View(func(tx *bolt.Tx) error {
		bucketPosts := tx.Bucket([]byte(BUCKET_POSTS))

		currentPage := 0
		pageKey := fmt.Sprintf("page-%d", currentPage)
		bucketPosts.ForEach(func(k, v []byte) error {
			post := new(Post)
			json.Unmarshal(v, &post)

			if post.Draft {
				meta.Drafts++
				draftList = append(draftList, post)
			} else {
				meta.Posts++
				pathMap[post.Path] = post.UUID

				pageMap[pageKey] = append(pageMap[pageKey], post)
				if len(pageMap[pageKey]) >= meta.PerPage {
					currentPage++
					meta.Pages++
					pageKey = fmt.Sprintf("page-%d", currentPage)
				}
			}

			return nil
		})

		return nil
	}); err != nil {
		return err
	}

	if meta.Pages == 0 {
		meta.Pages = 1
	}

	if err := db.Update(func(tx *bolt.Tx) error {
		if err := tx.DeleteBucket([]byte(BUCKET_INDEX)); err != nil {
			return err
		}
		bucketIndex, err := tx.CreateBucketIfNotExists([]byte(BUCKET_INDEX))
		if err != nil {
			return err
		}
		if err := tx.DeleteBucket([]byte(BUCKET_MAP)); err != nil {
			return err
		}
		bucketMap, err := tx.CreateBucketIfNotExists([]byte(BUCKET_MAP))
		if err != nil {
			return err
		}

		jsonMeta, _ := json.Marshal(meta)
		cfg.Log(string(jsonMeta))
		if err := bucketIndex.Put([]byte("meta"), []byte(jsonMeta)); err != nil {
			return err
		}

		for pageKey, postsPage := range pageMap {
			jsonPage, _ := json.Marshal(postsPage)
			if err := bucketIndex.Put([]byte(pageKey), []byte(jsonPage)); err != nil {
				return err
			}
		}

		jsonDrafts, _ := json.Marshal(draftList)
		if err := bucketIndex.Put([]byte("drafts"), []byte(jsonDrafts)); err != nil {
			return err
		}

		for path, uuid := range pathMap {
			if err := bucketMap.Put([]byte(path), []byte(uuid)); err != nil {
				return err
			}
		}

		return nil
	}); err != nil {
		return err
	}

	cfg.Log("Index rebuilt!")
	return nil
}

func GetPostsPage(page int) ([]Post, error) {
	pageKey := fmt.Sprintf("page-%d", page)
	var posts []Post

	if err := db.View(func(tx *bolt.Tx) error {
		bucketIndex := tx.Bucket([]byte(BUCKET_INDEX))
		if bucketIndex == nil {
			panic("Bucket index not found!")
		}

		jsonPosts := bucketIndex.Get([]byte(pageKey))
		json.Unmarshal(jsonPosts, &posts)

		return nil
	}); err != nil {
		return posts, err
	}

	return posts, nil
}
<commit_msg>tags index<commit_after>package model

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/boltdb/bolt"

	"../cfg"
)

const (
	BUCKET_INDEX = "index"
)

type Meta struct {
	Posts     int
	PerPage   int
	Pages     int
	Drafts    int
	UpdatedAt time.Time
}

func (meta *Meta) init() {
	meta.Posts = 0
	meta.PerPage = cfg.Int("ui.per_page")
	meta.Pages = 0
	meta.Drafts = 0
	meta.UpdatedAt = time.Now()
}

func (meta *Meta) Load() {
	meta.init()
	db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte([]byte(BUCKET_INDEX)))
		if bucket != nil {
			jsonMeta := bucket.Get([]byte("meta"))
			json.Unmarshal(jsonMeta, &meta)
		}

		return nil
	})
}

func RebuildIndex() error {
	cfg.Log("Rebuilding index...")

	meta := new(Meta)
	meta.init()

	pathMap := make(map[string]string)
	pageMap := make(map[string][]*Post)
	tagMap := make(map[string][]*Post)
	var draftList []*Post

	if err := db.View(func(tx *bolt.Tx) error {
		bucketPosts := tx.Bucket([]byte(BUCKET_POSTS))

		currentPage := 0
		pageKey := fmt.Sprintf("page-%d", currentPage)
		bucketPosts.ForEach(func(k, v []byte) error {
			post := new(Post)
			json.Unmarshal(v, &post)

			if post.Draft {
				meta.Drafts++
				draftList = append(draftList, post)
			} else {
				meta.Posts++
				pathMap[post.Path] = post.UUID

				pageMap[pageKey] = append(pageMap[pageKey], post)
				if len(pageMap[pageKey]) >= meta.PerPage {
					currentPage++
					meta.Pages++
					pageKey = fmt.Sprintf("page-%d", currentPage)
				}

				for _, tag := range post.Tags {
					tagKey := fmt.Sprintf("tag-%s", tag)
					tagMap[tagKey] = append(tagMap[tagKey], post)
				}
			}

			return nil
		})

		return nil
	}); err != nil {
		return err
	}

	if meta.Pages == 0 {
		meta.Pages = 1
	}

	if err := db.Update(func(tx *bolt.Tx) error {
		if err := tx.DeleteBucket([]byte(BUCKET_INDEX)); err != nil {
			return err
		}
		bucketIndex, err := tx.CreateBucketIfNotExists([]byte(BUCKET_INDEX))
		if err != nil {
			return err
		}
		if err := tx.DeleteBucket([]byte(BUCKET_MAP)); err != nil {
			return err
		}
		bucketMap, err := tx.CreateBucketIfNotExists([]byte(BUCKET_MAP))
		if err != nil {
			return err
		}

		jsonMeta, _ := json.Marshal(meta)
		cfg.Log(string(jsonMeta))
		if err := bucketIndex.Put([]byte("meta"), []byte(jsonMeta)); err != nil {
			return err
		}

		for pageKey, postsPage := range pageMap {
			jsonPage, _ := json.Marshal(postsPage)
			if err := bucketIndex.Put([]byte(pageKey), []byte(jsonPage)); err != nil {
				return err
			}
		}

		jsonDrafts, _ := json.Marshal(draftList)
		if err := bucketIndex.Put([]byte("drafts"), []byte(jsonDrafts)); err != nil {
			return err
		}

		for path, uuid := range pathMap {
			if err := bucketMap.Put([]byte(path), []byte(uuid)); err != nil {
				return err
			}
		}

		for tagKey, postsTag := range tagMap {
			jsonPosts, _ := json.Marshal(postsTag)
			if err := bucketIndex.Put([]byte(tagKey), []byte(jsonPosts)); err != nil {
				return err
			}
		}

		return nil
	}); err != nil {
		return err
	}

	cfg.Log("Index rebuilt!")
	return nil
}

func GetPostsPage(page int) ([]Post, error) {
	pageKey := fmt.Sprintf("page-%d", page)
	var posts []Post

	if err := db.View(func(tx *bolt.Tx) error {
		bucketIndex := tx.Bucket([]byte(BUCKET_INDEX))
		if bucketIndex == nil {
			panic("Bucket index not found!")
		}

		jsonPosts := bucketIndex.Get([]byte(pageKey))
		json.Unmarshal(jsonPosts, &posts)

		return nil
	}); err != nil {
		return posts, err
	}

	return posts, nil
}

func GetTagPosts(tag string) ([]Post, error) {
	tagKey := fmt.Sprintf("tag-%s", tag)
	var posts []Post

	if err := db.View(func(tx *bolt.Tx) error {
		bucketIndex := tx.Bucket([]byte(BUCKET_INDEX))
		if bucketIndex == nil {
			panic("Bucket index not found!")
		}

		jsonPosts := bucketIndex.Get([]byte(tagKey))
		json.Unmarshal(jsonPosts, &posts)

		return nil
	}); err != nil {
		return posts, err
	}

	return posts, nil
}
<|endoftext|>