<commit_before>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	"encoding/json"
	"io"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/coreos/clair/api/context"
	"github.com/coreos/clair/database"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/worker"
)

// maxBodySize restricts client requests to 1MiB.
const maxBodySize int64 = 1048576

func decodeJSON(r *http.Request, v interface{}) error {
	defer r.Body.Close()
	return json.NewDecoder(io.LimitReader(r.Body, maxBodySize)).Decode(v)
}

func writeResponse(w io.Writer, resp interface{}) {
	err := json.NewEncoder(w).Encode(resp)
	if err != nil {
		panic("v1: failed to marshal response: " + err.Error())
	}
}

func writeHeader(w http.ResponseWriter, status int) int {
	w.WriteHeader(status)
	return status
}

func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	request := LayerEnvelope{}
	err := decodeJSON(r, &request)
	if err != nil {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if request.Layer == nil {
		writeResponse(w, LayerEnvelope{Error: &Error{"failed to provide layer"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Format)
	if err != nil {
		if _, ok := err.(*cerrors.ErrBadRequest); ok {
			writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
			return writeHeader(w, http.StatusBadRequest)
		}
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusCreated)
}

func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	_, withFeatures := r.URL.Query()["features"]
	_, withVulnerabilities := r.URL.Query()["vulnerabilities"]

	dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
	if err == cerrors.ErrNotFound {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	layer := LayerFromDatabaseModel(dbLayer, withFeatures, withVulnerabilities)

	writeResponse(w, LayerEnvelope{Layer: &layer})
	return writeHeader(w, http.StatusOK)
}

func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	err := ctx.Store.DeleteLayer(p.ByName("layerName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	dbNamespaces, err := ctx.Store.ListNamespaces()
	if err != nil {
		writeResponse(w, NamespaceEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}
	var namespaces []string
	for _, dbNamespace := range dbNamespaces {
		namespaces = append(namespaces, dbNamespace.Name)
	}

	writeResponse(w, NamespaceEnvelope{Namespaces: &namespaces})
	return writeHeader(w, http.StatusOK)
}

func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	request := VulnerabilityEnvelope{}
	err := decodeJSON(r, &request)
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if request.Vulnerability == nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{"failed to provide vulnerability"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	vuln, err := request.Vulnerability.DatabaseModel()
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln})
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusCreated)
}

func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	_, withFixedIn := r.URL.Query()["fixedIn"]

	dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	vuln := VulnerabilityFromDatabaseModel(dbVuln, withFixedIn)

	writeResponse(w, VulnerabilityEnvelope{Vulnerability: &vuln})
	return writeHeader(w, http.StatusOK)
}

func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	request := VulnerabilityEnvelope{}
	err := decodeJSON(r, &request)
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if request.Vulnerability == nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{"failed to provide vulnerability"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if len(request.Vulnerability.FixedIn) != 0 {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{"Vulnerability.FixedIn must be empty"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	vuln, err := request.Vulnerability.DatabaseModel()
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln})
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func postFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}
func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}
func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}
func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}

func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	query := r.URL.Query()

	limitStrs, limitExists := query["limit"]
	if !limitExists {
		writeResponse(w, NotificationEnvelope{Error: &Error{"must provide limit query parameter"}})
		return writeHeader(w, http.StatusBadRequest)
	}
	limit, err := strconv.Atoi(limitStrs[0])
	if err != nil {
		writeResponse(w, NotificationEnvelope{Error: &Error{"invalid limit format: " + err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	page := database.VulnerabilityNotificationFirstPage
	pageStrs, pageExists := query["page"]
	if pageExists {
		page, err = pageStringToDBPageNumber(pageStrs[0])
		if err != nil {
			writeResponse(w, NotificationEnvelope{Error: &Error{"invalid page format: " + err.Error()}})
			return writeHeader(w, http.StatusBadRequest)
		}
	}

	dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page)
	if err != nil {
		writeResponse(w, NotificationEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	notification := NotificationFromDatabaseModel(dbNotification, limit, page, nextPage)

	writeResponse(w, NotificationEnvelope{Notification: &notification})
	return writeHeader(w, http.StatusOK)
}

func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	// ez
	return 0
}

func getMetrics(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	prometheus.Handler().ServeHTTP(w, r)
	return 0
}
<commit_msg>api: implement delete notification<commit_after>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	"encoding/json"
	"io"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/coreos/clair/api/context"
	"github.com/coreos/clair/database"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/worker"
)

// maxBodySize restricts client requests to 1MiB.
const maxBodySize int64 = 1048576

func decodeJSON(r *http.Request, v interface{}) error {
	defer r.Body.Close()
	return json.NewDecoder(io.LimitReader(r.Body, maxBodySize)).Decode(v)
}

func writeResponse(w io.Writer, resp interface{}) {
	err := json.NewEncoder(w).Encode(resp)
	if err != nil {
		panic("v1: failed to marshal response: " + err.Error())
	}
}

func writeHeader(w http.ResponseWriter, status int) int {
	w.WriteHeader(status)
	return status
}

func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	request := LayerEnvelope{}
	err := decodeJSON(r, &request)
	if err != nil {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if request.Layer == nil {
		writeResponse(w, LayerEnvelope{Error: &Error{"failed to provide layer"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	err = worker.Process(ctx.Store, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Format)
	if err != nil {
		if _, ok := err.(*cerrors.ErrBadRequest); ok {
			writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
			return writeHeader(w, http.StatusBadRequest)
		}
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusCreated)
}

func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	_, withFeatures := r.URL.Query()["features"]
	_, withVulnerabilities := r.URL.Query()["vulnerabilities"]

	dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
	if err == cerrors.ErrNotFound {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	layer := LayerFromDatabaseModel(dbLayer, withFeatures, withVulnerabilities)

	writeResponse(w, LayerEnvelope{Layer: &layer})
	return writeHeader(w, http.StatusOK)
}

func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	err := ctx.Store.DeleteLayer(p.ByName("layerName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, LayerEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	dbNamespaces, err := ctx.Store.ListNamespaces()
	if err != nil {
		writeResponse(w, NamespaceEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}
	var namespaces []string
	for _, dbNamespace := range dbNamespaces {
		namespaces = append(namespaces, dbNamespace.Name)
	}

	writeResponse(w, NamespaceEnvelope{Namespaces: &namespaces})
	return writeHeader(w, http.StatusOK)
}

func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	request := VulnerabilityEnvelope{}
	err := decodeJSON(r, &request)
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if request.Vulnerability == nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{"failed to provide vulnerability"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	vuln, err := request.Vulnerability.DatabaseModel()
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln})
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusCreated)
}

func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	_, withFixedIn := r.URL.Query()["fixedIn"]

	dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	vuln := VulnerabilityFromDatabaseModel(dbVuln, withFixedIn)

	writeResponse(w, VulnerabilityEnvelope{Vulnerability: &vuln})
	return writeHeader(w, http.StatusOK)
}

func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	request := VulnerabilityEnvelope{}
	err := decodeJSON(r, &request)
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if request.Vulnerability == nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{"failed to provide vulnerability"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	if len(request.Vulnerability.FixedIn) != 0 {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{"Vulnerability.FixedIn must be empty"}})
		return writeHeader(w, http.StatusBadRequest)
	}

	vuln, err := request.Vulnerability.DatabaseModel()
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln})
	if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, VulnerabilityEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func postFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}
func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}
func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}
func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	return 0
}

func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	query := r.URL.Query()

	limitStrs, limitExists := query["limit"]
	if !limitExists {
		writeResponse(w, NotificationEnvelope{Error: &Error{"must provide limit query parameter"}})
		return writeHeader(w, http.StatusBadRequest)
	}
	limit, err := strconv.Atoi(limitStrs[0])
	if err != nil {
		writeResponse(w, NotificationEnvelope{Error: &Error{"invalid limit format: " + err.Error()}})
		return writeHeader(w, http.StatusBadRequest)
	}

	page := database.VulnerabilityNotificationFirstPage
	pageStrs, pageExists := query["page"]
	if pageExists {
		page, err = pageStringToDBPageNumber(pageStrs[0])
		if err != nil {
			writeResponse(w, NotificationEnvelope{Error: &Error{"invalid page format: " + err.Error()}})
			return writeHeader(w, http.StatusBadRequest)
		}
	}

	dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page)
	if err != nil {
		writeResponse(w, NotificationEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	notification := NotificationFromDatabaseModel(dbNotification, limit, page, nextPage)

	writeResponse(w, NotificationEnvelope{Notification: &notification})
	return writeHeader(w, http.StatusOK)
}

func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	err := ctx.Store.DeleteNotification(p.ByName("notificationName"))
	if err == cerrors.ErrNotFound {
		writeResponse(w, NotificationEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusNotFound)
	} else if err != nil {
		writeResponse(w, NotificationEnvelope{Error: &Error{err.Error()}})
		return writeHeader(w, http.StatusInternalServerError)
	}

	return writeHeader(w, http.StatusOK)
}

func getMetrics(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) int {
	prometheus.Handler().ServeHTTP(w, r)
	return 0
}
<|endoftext|>


<commit_before>package apig

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"text/template"

	"github.com/wantedly/apig/msg"
	"github.com/wantedly/apig/util"
)

func generateSkeleton(detail *Detail, outDir string) error {
	var wg sync.WaitGroup
	errCh := make(chan error, 1)
	done := make(chan bool, 1)

	r := regexp.MustCompile(`_templates/skeleton/.*\.tmpl$`)
	for _, skeleton := range AssetNames() {
		wg.Add(1)
		go func(s string) {
			defer wg.Done()

			if !r.MatchString(s) {
				return
			}

			trim := strings.Replace(s, "_templates/skeleton/", "", 1)
			path := strings.Replace(trim, ".tmpl", "", 1)
			dstPath := filepath.Join(outDir, path)

			body, err := Asset(s)
			if err != nil {
				errCh <- err
			}

			tmpl, err := template.New("complex").Parse(string(body))
			if err != nil {
				errCh <- err
			}

			var buf bytes.Buffer

			if err := tmpl.Execute(&buf, detail); err != nil {
				errCh <- err
			}

			if !util.FileExists(filepath.Dir(dstPath)) {
				if err := util.Mkdir(filepath.Dir(dstPath)); err != nil {
					errCh <- err
				}
			}

			if err := ioutil.WriteFile(dstPath, buf.Bytes(), 0644); err != nil {
				errCh <- err
			}

			msg.Printf("\t\x1b[32m%s\x1b[0m %s\n", "create", dstPath)
		}(skeleton)
	}

	wg.Wait()
	close(done)

	select {
	case <-done:
	case err := <-errCh:
		if err != nil {
			return err
		}
	}

	return nil
}

func Skeleton(gopath, vcs, username, project, namespace, database string) int {
	detail := &Detail{
		VCS:       vcs,
		User:      username,
		Project:   project,
		Namespace: namespace,
		Database:  database,
	}
	outDir := filepath.Join(gopath, "src", detail.VCS, detail.User, detail.Project)
	if util.FileExists(outDir) {
		fmt.Fprintf(os.Stderr, "%s is already exists", outDir)
		return 1
	}
	if err := generateSkeleton(detail, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}

	msg.Printf("===> Created %s", outDir)
	return 0
}
<commit_msg>Store regexp in global variable<commit_after>package apig

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"text/template"

	"github.com/wantedly/apig/msg"
	"github.com/wantedly/apig/util"
)

var r = regexp.MustCompile(`_templates/skeleton/.*\.tmpl$`)

func generateSkeleton(detail *Detail, outDir string) error {
	var wg sync.WaitGroup
	errCh := make(chan error, 1)
	done := make(chan bool, 1)

	for _, skeleton := range AssetNames() {
		wg.Add(1)
		go func(s string) {
			defer wg.Done()

			if !r.MatchString(s) {
				return
			}

			trim := strings.Replace(s, "_templates/skeleton/", "", 1)
			path := strings.Replace(trim, ".tmpl", "", 1)
			dstPath := filepath.Join(outDir, path)

			body, err := Asset(s)
			if err != nil {
				errCh <- err
			}

			tmpl, err := template.New("complex").Parse(string(body))
			if err != nil {
				errCh <- err
			}

			var buf bytes.Buffer

			if err := tmpl.Execute(&buf, detail); err != nil {
				errCh <- err
			}

			if !util.FileExists(filepath.Dir(dstPath)) {
				if err := util.Mkdir(filepath.Dir(dstPath)); err != nil {
					errCh <- err
				}
			}

			if err := ioutil.WriteFile(dstPath, buf.Bytes(), 0644); err != nil {
				errCh <- err
			}

			msg.Printf("\t\x1b[32m%s\x1b[0m %s\n", "create", dstPath)
		}(skeleton)
	}

	wg.Wait()
	close(done)

	select {
	case <-done:
	case err := <-errCh:
		if err != nil {
			return err
		}
	}

	return nil
}

func Skeleton(gopath, vcs, username, project, namespace, database string) int {
	detail := &Detail{
		VCS:       vcs,
		User:      username,
		Project:   project,
		Namespace: namespace,
		Database:  database,
	}
	outDir := filepath.Join(gopath, "src", detail.VCS, detail.User, detail.Project)
	if util.FileExists(outDir) {
		fmt.Fprintf(os.Stderr, "%s is already exists", outDir)
		return 1
	}
	if err := generateSkeleton(detail, outDir); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}

	msg.Printf("===> Created %s", outDir)
	return 0
}
<|endoftext|>


<commit_before><commit_msg>Minimal change in documentation<commit_after><|endoftext|>


<commit_before><commit_msg>fixed error shadowing to make gometalinter stop complaining<commit_after><|endoftext|>


<commit_before>package main

import (
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/jcuga/golongpoll"
)

const (
	ALL_CHATS = "all_chats"
)

func main() {
	// Our chat server is just a longpoll/pub-sub server.
	manager, err := golongpoll.StartLongpoll(golongpoll.Options{})
	if err != nil {
		log.Fatalf("Failed to create chat longpoll manager: %q", err)
	}
	http.HandleFunc("/", Index)
	http.HandleFunc("/post", getChatPostClosure(manager))
	http.HandleFunc("/subscribe", manager.SubscriptionHandler)
	listenAddress := ":8080"
	log.Printf("Launching chat server on %s", listenAddress)
	http.ListenAndServe(listenAddress, nil)
}

type ChatPost struct {
	DisplayName string `json:"display_name"`
	Message     string `json:"message"`
	Topic       string `json:"topic"`
}

// Create a closure that contains a ref to our longpoll manager so we can
// call Publish() from within web handler
// NOTE: the manager is safe to call this way because it relies on channels
func getChatPostClosure(manager *golongpoll.LongpollManager) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.Error(w, "Invalid request method.", 405)
			return
		}
		err := r.ParseForm()
		if err != nil {
			http.Error(w, "Invalid form data.", 405)
			return
		}
		topic := r.PostFormValue("topic")
		display_name := r.PostFormValue("display_name")
		message := r.PostFormValue("message")
		if len(strings.TrimSpace(topic)) == 0 || len(strings.TrimSpace(display_name)) == 0 ||
			len(strings.TrimSpace(message)) == 0 {
			http.Error(w, "Invalid request.  Blank topic, display_name, or message.", 400)
			return
		}
		chat := ChatPost{DisplayName: display_name, Message: message, Topic: topic}
		manager.Publish(topic, chat)
		// show on the all-chats channel as well that shows on the homepage when you
		// haven't filtered to a specific topic.
		manager.Publish(ALL_CHATS, chat)
		// redirect to the chat page for the given topic
		http.Redirect(w, r, "/?topic="+topic+"&display_name="+display_name, http.StatusSeeOther)
	}
}

func Index(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Invalid request method.", 405)
		return
	}
	topic := r.URL.Query().Get("topic")
	displayName := r.URL.Query().Get("display_name")
	t := template.New("chat_homepage")
	t, _ = t.Parse(getIndexTemplateString())
	templateData := struct {
		Topic       string
		DisplayName string
		AllChats    string
	}{topic, displayName, ALL_CHATS}
	t.Execute(w, templateData)
}

func getIndexTemplateString() string {
	// TODO: box up top for name, chat, topic (hidden vs text field depending on
	// if topic filter on.)  when posting redir to topic feed
	// TODO: finish html/js, add readme and screenshots
	return `

      <title>micro-chat</title>

			<form method="POST" action="/post">
				{{ if .Topic }}
				  <input type="hidden" name="topic" value="{{ .Topic }}">
				{{ else }}
				  Topic:
<input type="text" name="topic">

				{{ end }}
				Post as

				<input type="text" name="display_name" value="{{if .DisplayName}}{{.DisplayName}}{{end}}">

				Message

				<textarea rows="2" cols="50" name="message"></textarea>

				<input type="submit" value="post">
			</form>

			{{ if .Topic }}

Showing chats about: {{ .Topic}}

				Show all chats.
      {{ else }}

Showing all chats

      {{ end }}

  `
}
<commit_msg>removed finished todo comments<commit_after>package main

import (
	"html/template"
	"log"
	"net/http"
	"strings"

	"github.com/jcuga/golongpoll"
)

const (
	ALL_CHATS = "all_chats"
)

func main() {
	// Our chat server is just a longpoll/pub-sub server.
	manager, err := golongpoll.StartLongpoll(golongpoll.Options{})
	if err != nil {
		log.Fatalf("Failed to create chat longpoll manager: %q", err)
	}
	http.HandleFunc("/", Index)
	http.HandleFunc("/post", getChatPostClosure(manager))
	http.HandleFunc("/subscribe", manager.SubscriptionHandler)
	listenAddress := ":8080"
	log.Printf("Launching chat server on %s", listenAddress)
	http.ListenAndServe(listenAddress, nil)
}

type ChatPost struct {
	DisplayName string `json:"display_name"`
	Message     string `json:"message"`
	Topic       string `json:"topic"`
}

// Create a closure that contains a ref to our longpoll manager so we can
// call Publish() from within web handler
// NOTE: the manager is safe to call this way because it relies on channels
func getChatPostClosure(manager *golongpoll.LongpollManager) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.Error(w, "Invalid request method.", 405)
			return
		}
		err := r.ParseForm()
		if err != nil {
			http.Error(w, "Invalid form data.", 405)
			return
		}
		topic := r.PostFormValue("topic")
		display_name := r.PostFormValue("display_name")
		message := r.PostFormValue("message")
		if len(strings.TrimSpace(topic)) == 0 || len(strings.TrimSpace(display_name)) == 0 ||
			len(strings.TrimSpace(message)) == 0 {
			http.Error(w, "Invalid request.  Blank topic, display_name, or message.", 400)
			return
		}
		chat := ChatPost{DisplayName: display_name, Message: message, Topic: topic}
		manager.Publish(topic, chat)
		// show on the all-chats channel as well that shows on the homepage when you
		// haven't filtered to a specific topic.
		manager.Publish(ALL_CHATS, chat)
		// redirect to the chat page for the given topic
		http.Redirect(w, r, "/?topic="+topic+"&display_name="+display_name, http.StatusSeeOther)
	}
}

func Index(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		http.Error(w, "Invalid request method.", 405)
		return
	}
	topic := r.URL.Query().Get("topic")
	displayName := r.URL.Query().Get("display_name")
	t := template.New("chat_homepage")
	t, _ = t.Parse(getIndexTemplateString())
	templateData := struct {
		Topic       string
		DisplayName string
		AllChats    string
	}{topic, displayName, ALL_CHATS}
	t.Execute(w, templateData)
}

func getIndexTemplateString() string {
	return `

      <title>micro-chat</title>

			<form method="POST" action="/post">
				{{ if .Topic }}
				  <input type="hidden" name="topic" value="{{ .Topic }}">
				{{ else }}
				  Topic:
<input type="text" name="topic">

				{{ end }}
				Post as

				<input type="text" name="display_name" value="{{if .DisplayName}}{{.DisplayName}}{{end}}">

				Message

				<textarea rows="2" cols="50" name="message"></textarea>

				<input type="submit" value="post">
			</form>

			{{ if .Topic }}

Showing chats about: {{ .Topic}}

				Show all chats.
      {{ else }}

Showing all chats

      {{ end }}

  `
}
<|endoftext|>


<commit_before>// Copyright 2015 Lars Wiegman. All rights reserved. Use of this source code is
// governed by a BSD-style license that can be found in the LICENSE file.

/*

Microdata is a package for the Go programming language to extract HTML
Microdata from HTML5 documents. It depends on the
golang.org/x/net/html HTML5-compliant parser.

Usage:
	var result microdata.Result
	result, _ = microdata.ParseURL("http://example.com/blogposting")
	b, _ := json.MarshalIndent(result, "", "  ")
	os.Stdout.Write(b)

*/

package microdata

import (
	"errors"
	"io"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/charset"
)

var (
	ErrInvalidPropertyType Error = errors.New("invalid property type")
)

type Error error

type Result struct {
	Items []*Item `json:"items"`
}

type Item struct {
	Type       []string               `json:"type"`
	Properties map[string]interface{} `json:"properties"`
	Id         string                 `json:"id,omitempty"`
}

// addProperty adds the key, property to the Item. It appends to any existing
// properties associated with the key.
func (item *Item) addProperty(key string, property interface{}) error {
	switch property.(type) {
	case *Item:
		if a, ok := item.Properties[key]; ok {
			item.Properties[key] = append((a).([]*Item), (property).(*Item))
		} else {
			item.Properties[key] = []*Item{(property).(*Item)}
		}
	case string:
		if a, ok := item.Properties[key]; ok {
			item.Properties[key] = append((a).([]string), (property).(string))
		} else {
			item.Properties[key] = []string{(property).(string)}
		}
	default:
		return ErrInvalidPropertyType
	}
	return nil
}

// NewItem returns a new Item with the given itemtype(s).
func NewItem(itemtype []string) *Item {
	props := make(map[string]interface{})
	return &Item{
		Type:       itemtype,
		Properties: props,
	}
}

// Parse parses the HTML document available in the given reader and returns the
// result. The given baseURL is used to complete incomplete URLs in src and href
// attributes. The given contentType is used convert the content of r to UTF-8.
func Parse(r io.Reader, baseURL string, contentType string) (Result, error) {
	result := Result{}
	u, err := url.Parse(baseURL)

	r, err = charset.NewReader(r, contentType)
	if err != nil {
		return result, err
	}

	doc, err := html.Parse(r)
	if err != nil {
		return result, err
	}

	var item *Item
	var f func(*html.Node, *Item) error
	f = func(n *html.Node, item *Item) error {
		switch n.Type {
		case html.ElementNode:
			var itemscope bool
			var itemtype, itemprop []string
			var value, itemid string

			for _, attr := range n.Attr {
				switch attr.Key {
				case "itemscope":
					itemscope = true
				case "itemtype":
					itemtype = strings.Split(attr.Val, " ")
				case "itemprop":
					itemprop = strings.Split(attr.Val, " ")
				case "src", "href":
					refURL, _ := u.Parse(attr.Val)
					value = refURL.String()
				case "content", "data", "datetime":
					value = attr.Val
				case "itemid":
					itemid = attr.Val
				}
			}

			// New Item
			if itemscope && len(itemtype) > 0 {
				i := NewItem(itemtype)
				switch {
				case len(itemid) > 0:
					i.Id = itemid
					result.Items = append(result.Items, i)
				case len(itemprop) == 0:
					result.Items = append(result.Items, i)
				case len(itemprop) > 0:
					// Might not be a valid spec
					for _, key := range itemprop {
						if err := item.addProperty(key, i); err != nil {
							return err
						}
					}
				}
				item = i
				break
			}

			// New Property
			if item != nil && len(itemprop) > 0 {
				for _, key := range itemprop {
					if len(value) < 1 {
						var s string
						var f func(*html.Node)
						f = func(n *html.Node) {
							if n.Type == html.TextNode {
								s += n.Data
							}
							for c := n.FirstChild; c != nil; c = c.NextSibling {
								f(c)
							}
						}
						f(n)
						if len(s) > 0 {
							value = s
						}
					}
					if err := item.addProperty(key, value); err != nil {
						return err
					}
				}
				break
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if err := f(c, item); err != nil {
				return err
			}
		}
		return nil
	}
	if err := f(doc, item); err != nil {
		return result, err
	}

	return result, nil
}

// ParseURL parses the HTML document available at the given URL and returns the
// result.
func ParseURL(urlStr string) (Result, error) {
	var result Result

	resp, err := http.DefaultClient.Get(urlStr)
	if err != nil {
		return result, err
	}
	contentType := resp.Header.Get("Content-Type")

	return Parse(resp.Body, urlStr, contentType)
}
<commit_msg>Reformulated introduction comment<commit_after>// Copyright 2015 Lars Wiegman. All rights reserved. Use of this source code is
// governed by a BSD-style license that can be found in the LICENSE file.

/*

	Package flag implements command-line flag parsing.
	Package microdata implements a HTML microdata parser. It depends on the
	golang.org/x/net/html HTML5-compliant parser.

	Usage:

	Pass a reader, baseURL and contentType to the Parse function.
		data, err := microdata.Parse(reader, baseURL, contentType)
		items := data.Items

	Pass an URL to the ParseURL function.
		data, _ := microdata.ParseURL("http://example.com/blogposting")
		items := data.Items
*/

package microdata

import (
	"errors"
	"io"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/charset"
)

var (
	ErrInvalidPropertyType Error = errors.New("invalid property type")
)

type Error error

type Result struct {
	Items []*Item `json:"items"`
}

type Item struct {
	Type       []string               `json:"type"`
	Properties map[string]interface{} `json:"properties"`
	Id         string                 `json:"id,omitempty"`
}

// addProperty adds the key, property to the Item. It appends to any existing
// properties associated with the key.
func (item *Item) addProperty(key string, property interface{}) error {
	switch property.(type) {
	case *Item:
		if a, ok := item.Properties[key]; ok {
			item.Properties[key] = append((a).([]*Item), (property).(*Item))
		} else {
			item.Properties[key] = []*Item{(property).(*Item)}
		}
	case string:
		if a, ok := item.Properties[key]; ok {
			item.Properties[key] = append((a).([]string), (property).(string))
		} else {
			item.Properties[key] = []string{(property).(string)}
		}
	default:
		return ErrInvalidPropertyType
	}
	return nil
}

// NewItem returns a new Item with the given itemtype(s).
func NewItem(itemtype []string) *Item {
	props := make(map[string]interface{})
	return &Item{
		Type:       itemtype,
		Properties: props,
	}
}

// Parse parses the HTML document available in the given reader and returns the
// result. The given baseURL is used to complete incomplete URLs in src and href
// attributes. The given contentType is used convert the content of r to UTF-8.
func Parse(r io.Reader, baseURL string, contentType string) (Result, error) {
	result := Result{}
	u, err := url.Parse(baseURL)

	r, err = charset.NewReader(r, contentType)
	if err != nil {
		return result, err
	}

	doc, err := html.Parse(r)
	if err != nil {
		return result, err
	}

	var item *Item
	var f func(*html.Node, *Item) error
	f = func(n *html.Node, item *Item) error {
		switch n.Type {
		case html.ElementNode:
			var itemscope bool
			var itemtype, itemprop []string
			var value, itemid string

			for _, attr := range n.Attr {
				switch attr.Key {
				case "itemscope":
					itemscope = true
				case "itemtype":
					itemtype = strings.Split(attr.Val, " ")
				case "itemprop":
					itemprop = strings.Split(attr.Val, " ")
				case "src", "href":
					refURL, _ := u.Parse(attr.Val)
					value = refURL.String()
				case "content", "data", "datetime":
					value = attr.Val
				case "itemid":
					itemid = attr.Val
				}
			}

			// New Item
			if itemscope && len(itemtype) > 0 {
				i := NewItem(itemtype)
				switch {
				case len(itemid) > 0:
					i.Id = itemid
					result.Items = append(result.Items, i)
				case len(itemprop) == 0:
					result.Items = append(result.Items, i)
				case len(itemprop) > 0:
					// Might not be a valid spec
					for _, key := range itemprop {
						if err := item.addProperty(key, i); err != nil {
							return err
						}
					}
				}
				item = i
				break
			}

			// New Property
			if item != nil && len(itemprop) > 0 {
				for _, key := range itemprop {
					if len(value) < 1 {
						var s string
						var f func(*html.Node)
						f = func(n *html.Node) {
							if n.Type == html.TextNode {
								s += n.Data
							}
							for c := n.FirstChild; c != nil; c = c.NextSibling {
								f(c)
							}
						}
						f(n)
						if len(s) > 0 {
							value = s
						}
					}
					if err := item.addProperty(key, value); err != nil {
						return err
					}
				}
				break
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if err := f(c, item); err != nil {
				return err
			}
		}
		return nil
	}
	if err := f(doc, item); err != nil {
		return result, err
	}

	return result, nil
}

// ParseURL parses the HTML document available at the given URL and returns the
// result.
func ParseURL(urlStr string) (Result, error) {
	var result Result

	resp, err := http.DefaultClient.Get(urlStr)
	if err != nil {
		return result, err
	}
	contentType := resp.Header.Get("Content-Type")

	return Parse(resp.Body, urlStr, contentType)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/boltdb/bolt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

// TopVersion is the current level of patches that the database knows of.
// This should always be the total amount of migrations.
//
// If you add a new migration, increase this number to make sure that your
// migration is actually applied.
const TopVersion = 1

var (
	errNoMigrationsYet = errors.New("no migrations have been added yet")
	levelKey           = []byte("level")
)

// migrations is a list of all the migration functions.
// Their indexes are used to determine when they are to be applied.
var migrations = []func(db *bolt.DB) error{
	InitialMigration,
}

// Migrate is the main migration entrypoint
//
// When called, it will check the database to see what migrations have already
// been applied. If that is lower than the const TopVersion, all migrations up
// to that point will sequentially be applied.
func Migrate(db *bolt.DB) error {
	version, err := getVersion(db)

	if err == errNoMigrationsYet {
		// No migrations have been done yet, so lets do the first one by adding
		// the migration bucket.
		version = 0
	} else if err != nil {
		// Something actually went wrong. Oh noes.
		return err
	}

	// If version is lower than the latest known version, it's time to apply the
	// migrations!
	if version < TopVersion {
		if err := applyMigrations(db, version); err != nil {
			log.Print("Error: Migration application failed ;'(")
			return err
		}
	} else {
		log.Print("Database up to date.")
	}

	return nil
}

func applyMigrations(db *bolt.DB, version int) error {
	log.Printf(" --- Migrating %d -> %d:", version, TopVersion)
	if err := backup(db, version, "db-migration-backup/"); err != nil {
		return err
	}

	// Run the new migrations and the new migrations only
	for x, migration := range migrations[version:] {
		log.Printf("  Applying migration %d", x)
		if err := migration(db); err != nil {
			log.Print("  Migration failure: ", err)
			return err
		}
	}

	log.Printf(" --- Migrations applied successfully. <3")
	return nil
}

func backup(db *bolt.DB, version int, path string) error {
	_ = os.Mkdir(path, 0755)

	fn := fmt.Sprintf(
		"%d_%d-%d.db",
		time.Now().UnixNano(),
		version,
		TopVersion,
	)
	dst := filepath.Join(path, fn)

	data, _ := ioutil.ReadFile(db.Path())
	if err := ioutil.WriteFile(dst, data, 0644); err != nil {
		return err
	}

	fmt.Printf(" Backed up into %s", fn)
	return nil
}

func getVersion(db *bolt.DB) (int, error) {
	var version int
	err := db.View(func(tx *bolt.Tx) error {
		var err error
		b := tx.Bucket(MigrationKey)
		if b == nil {
			return errNoMigrationsYet
		}

		x := b.Get(levelKey)
		version, err = strconv.Atoi(string(x))
		if err != nil {
			return err
		}

		return nil
	})

	return version, err
}

func setVersion(tx *bolt.Tx, version int) error {
	b, err := tx.CreateBucketIfNotExists(MigrationKey)
	if err != nil {
		return err
	}

	out, _ := json.Marshal(version)
	if err := b.Put(levelKey, out); err != nil {
		return err
	}
	return nil
}

// InitialMigration adds the db_version bucket with the sole entry
func InitialMigration(db *bolt.DB) error {
	return db.Update(func(tx *bolt.Tx) error {
		return setVersion(tx, 1)
	})
}
<commit_msg>Fix formatting of migration messages<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/boltdb/bolt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

// TopVersion is the current level of patches that the database knows of.
// This should always be the total amount of migrations.
//
// If you add a new migration, increase this number to make sure that your
// migration is actually applied.
const TopVersion = 1

var (
	errNoMigrationsYet = errors.New("no migrations have been added yet")
	levelKey           = []byte("level")
)

// migrations is a list of all the migration functions.
// Their indexes are used to determine when they are to be applied.
var migrations = []func(db *bolt.DB) error{
	InitialMigration,
}

// Migrate is the main migration entrypoint
//
// When called, it will check the database to see what migrations have already
// been applied. If that is lower than the const TopVersion, all migrations up
// to that point will sequentially be applied.
func Migrate(db *bolt.DB) error {
	version, err := getVersion(db)

	if err == errNoMigrationsYet {
		// No migrations have been done yet, so lets do the first one by adding
		// the migration bucket.
		version = 0
	} else if err != nil {
		// Something actually went wrong. Oh noes.
		return err
	}

	// If version is lower than the latest known version, it's time to apply the
	// migrations!
	if version < TopVersion {
		if err := applyMigrations(db, version); err != nil {
			log.Print("Error: Migration application failed ;'(")
			return err
		}
	} else {
		log.Print("Database up to date.")
	}

	return nil
}

func applyMigrations(db *bolt.DB, version int) error {
	log.Printf(" --- Migrating %d -> %d:\n", version, TopVersion)
	if err := backup(db, version, "db-migration-backup/"); err != nil {
		return err
	}

	// Run the new migrations and the new migrations only
	for x, migration := range migrations[version:] {
		log.Printf("     Applying migration %d\n", x)
		if err := migration(db); err != nil {
			log.Print("     Migration failure: ", err)
			return err
		}
	}

	log.Printf(" --- Migrations applied successfully. <3\n")
	return nil
}

func backup(db *bolt.DB, version int, path string) error {
	_ = os.Mkdir(path, 0755)

	fn := fmt.Sprintf(
		"%d_%d-%d.db",
		time.Now().UnixNano(),
		version,
		TopVersion,
	)
	dst := filepath.Join(path, fn)

	data, _ := ioutil.ReadFile(db.Path())
	if err := ioutil.WriteFile(dst, data, 0644); err != nil {
		return err
	}

	log.Printf("     Backed up into %s\n", fn)
	return nil
}

func getVersion(db *bolt.DB) (int, error) {
	var version int
	err := db.View(func(tx *bolt.Tx) error {
		var err error
		b := tx.Bucket(MigrationKey)
		if b == nil {
			return errNoMigrationsYet
		}

		x := b.Get(levelKey)
		version, err = strconv.Atoi(string(x))
		if err != nil {
			return err
		}

		return nil
	})

	return version, err
}

func setVersion(tx *bolt.Tx, version int) error {
	b, err := tx.CreateBucketIfNotExists(MigrationKey)
	if err != nil {
		return err
	}

	out, _ := json.Marshal(version)
	if err := b.Put(levelKey, out); err != nil {
		return err
	}
	return nil
}

// InitialMigration adds the db_version bucket with the sole entry
func InitialMigration(db *bolt.DB) error {
	return db.Update(func(tx *bolt.Tx) error {
		return setVersion(tx, 1)
	})
}
<|endoftext|>


<commit_before>// minicache is a work in progress in-memory caching system
// featuring a similar text based protocol to memcached
package main

// TODO:
// - TTL handling
// - casunique?
// - noreply?
// - Investigate how memcached handles concurrency/locking

import (
	"bufio"
	"fmt"
	"log"
	"net"
	// "os"
	"strconv"
	"strings"
	"time"
)

type Client struct {
	Id      string
	State   uint8
	Input   []string
	Command string
	Record  *Record
}

type Record struct {
	Key    string
	Value  string
	Flags  int64
	Ttl    int64
	Length int64
}

const STATE_DEFAULT uint8 = 1
const STATE_EXPECTING_VALUE uint8 = 2

const STATE_COMMAND_GET uint8 = 3
const STATE_COMMAND_SET uint8 = 4
const STATE_COMMAND_DELETE uint8 = 5
const STATE_COMMAND_QUIT uint8 = 6
const STATE_COMMAND_FLUSHALL uint8 = 7

var ticker = time.NewTicker(time.Second * 1)
var clients map[string]*Client
var datastore map[string]*Record

func main() {
	// Start the server on port 5268
	server, err := net.Listen("tcp", ":5268")

	if err != nil {
		log.Fatal(err)
	}

	// Ensure that the server closes
	defer server.Close()

	// Our datastore..
	datastore = make(map[string]*Record)

	// A list of active clients
	clients = make(map[string]*Client)

	// Print out the datastore contents every second for debug
	go func() {
		for range ticker.C {
			fmt.Println(datastore)
		}
	}()

	for {
		// Wait for a connection.
		connection, err := server.Accept()

		if err != nil {
			log.Fatal(err)
		}

		// Handle the connection in a new goroutine.
		// The loop then returns to accepting new connections,
		// so that multiple connections may be served concurrently.
		go func(connection net.Conn) {
			// Create the client and store it
			client := &Client{
				Id:      connection.RemoteAddr().String(),
				State:   STATE_DEFAULT,
				Input:   []string{},
				Command: "",
				Record:  &Record{},
			}
			clients[connection.RemoteAddr().String()] = client

			// Ensure the client is tidied up once they're done
			defer func(connection net.Conn, clients map[string]*Client, id string) {
				delete(clients, id)
				connection.Close()
			}(connection, clients, client.Id)

			// Create a new scanner for the client input
			scanner := bufio.NewScanner(connection)

			// Handle each line (command)
			for scanner.Scan() {
				// Split the client input up based on spaces
				client.Input = strings.Split(scanner.Text(), " ")

				// If we're in our default state then determine
				// what command we're running
				if client.State == STATE_DEFAULT {
					// Get the command
					client.Command = client.Input[0]

					switch client.Command {
					case "get":
						client.State = STATE_COMMAND_GET
					case "set":
						client.State = STATE_COMMAND_SET
					case "delete":
						client.State = STATE_COMMAND_DELETE
					case "flush_all":
						client.State = STATE_COMMAND_FLUSHALL
					}
				}

				// Switch on the type of command
				switch client.State {
				// Are we expecting a value from a set command?
				case STATE_EXPECTING_VALUE:
					// If the value isn't set then set it
					if client.Record.Value == "" {
						client.Record.Value = scanner.Text()
					// Otherwise append to it
					} else {
						client.Record.Value += scanner.Text()
					}

					client.Record.Value += "\r\n"

					// Count the length of the value minus the trailing \r\n
					valueLength := int64(len(client.Record.Value)) - 2

					// If the datastore is same or greater than the expected length
					// we are done with this op
					if valueLength >= client.Record.Length {
						// If it's the same length we can try and store it
						if valueLength == client.Record.Length {
							// Store the value
							datastore[client.Record.Key] = client.Record

							// Inform the client we have stored the value
							// TODO: error handling here
							fmt.Fprintln(connection, "STORED")
						// Otherwise the client has messed up
						} else {
							// Inform the client that they messed up
							fmt.Fprintln(connection, "CLIENT_ERROR")
							fmt.Fprintln(connection, valueLength)
							fmt.Fprintln(connection, "ERROR")
						}

						// Reset the clients state
						client.Reset()
					}
				// get [key1 ... keyn]
				// TODO: handling multiple key gets
				case STATE_COMMAND_GET:
					// Check if a key was passed, if so try and retrieve it
					if len(client.Input) == 2 {
						// Get the key
						key := client.Input[1]

						// Look up the record in our datastore
						record := datastore[key]

						// Did it exist?
						if record != nil {
							fmt.Fprintln(connection, fmt.Sprintf("VALUE %s %d %d", record.Key, record.Flags, record.Length))
							fmt.Fprint(connection, record.Value)
						}

						fmt.Fprintln(connection, "END")
					} else {
						fmt.Fprintln(connection, "CLIENT_ERROR")
					}

					// Reset the clients state regardless off success/failure
					client.Reset()
				// set [key] [flags] [exptime] [length] [casunique] [noreply]
				case STATE_COMMAND_SET:
					// Check the right number of arguments are passed
					if len(client.Input) == 7 {
						// Get the key name
						client.Record.Key = client.Input[1]

						// Get any flags
						client.Record.Flags, err = strconv.ParseInt(client.Input[2], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						// Get the key TTL
						client.Record.Ttl, err = strconv.ParseInt(client.Input[3], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						// Get the value length
						client.Record.Length, err = strconv.ParseInt(client.Input[4], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						// Set that we are expecting a value
						client.State = STATE_EXPECTING_VALUE
					} else {
						fmt.Fprintln(connection, "ERROR ", err)

						// Reset the clients state
						client.Reset()
					}
				// delete [key] [noreply]
				case STATE_COMMAND_DELETE:
					// Check if a key was passed, if so try and retrieve it
					if len(client.Input) == 2 {
						// Get the key
						key := client.Input[1]

						// Look up the record in our datastore
						record := datastore[key]

						// Did it exist? If so 'delete' it
						if record != nil {
							delete(datastore, key)
							fmt.Fprintln(connection, "DELETED")
						} else {
							fmt.Fprintln(connection, "NOT_FOUND")
						}

					} else {
						fmt.Fprintln(connection, "CLIENT_ERROR")
					}

					// Reset the clients state regardless off success/failure
					client.Reset()
				// quit
				case STATE_COMMAND_QUIT:
					// Not much to do here atm..
					// Eventually we will do logging etc
				// flushall [delay]
				case STATE_COMMAND_FLUSHALL:
					// Check if a delay was passed
					if len(client.Input) == 2 {
						delay, err := strconv.ParseInt(client.Input[1], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						time.Sleep(time.Duration(delay) * time.Second)
					}

					// Reset the datastore
					datastore = make(map[string]*Record)
					fmt.Fprintln(connection, "OK")
				}
			}

			// Print out errors to stderr
			if err := scanner.Err(); err != nil {
				fmt.Fprintln(connection, "ERROR ", err)
			}
		}(connection)
	}
}

// Reset a clients state to what it would be on first connection
func (client *Client) Reset() {
	client.State = STATE_DEFAULT
	client.Input = []string{}
	client.Command = ""
	client.Record =  &Record{}
}
<commit_msg>Handle unknown command<commit_after>// minicache is a work in progress in-memory caching system
// featuring a similar text based protocol to memcached
package main

// TODO:
// - TTL handling
// - casunique?
// - noreply?
// - Investigate how memcached handles concurrency/locking

import (
	"bufio"
	"fmt"
	"log"
	"net"
	// "os"
	"strconv"
	"strings"
	"time"
)

type Client struct {
	Id      string
	State   uint8
	Input   []string
	Command string
	Record  *Record
}

type Record struct {
	Key    string
	Value  string
	Flags  int64
	Ttl    int64
	Length int64
}

const STATE_DEFAULT uint8 = 1
const STATE_EXPECTING_VALUE uint8 = 2

const STATE_COMMAND_GET uint8 = 3
const STATE_COMMAND_SET uint8 = 4
const STATE_COMMAND_DELETE uint8 = 5
const STATE_COMMAND_QUIT uint8 = 6
const STATE_COMMAND_FLUSHALL uint8 = 7

var ticker = time.NewTicker(time.Second * 1)
var clients map[string]*Client
var datastore map[string]*Record

func main() {
	// Start the server on port 5268
	server, err := net.Listen("tcp", ":5268")

	if err != nil {
		log.Fatal(err)
	}

	// Ensure that the server closes
	defer server.Close()

	// Our datastore..
	datastore = make(map[string]*Record)

	// A list of active clients
	clients = make(map[string]*Client)

	// Print out the datastore contents every second for debug
	go func() {
		for range ticker.C {
			fmt.Println(datastore)
		}
	}()

	for {
		// Wait for a connection.
		connection, err := server.Accept()

		if err != nil {
			log.Fatal(err)
		}

		// Handle the connection in a new goroutine.
		// The loop then returns to accepting new connections,
		// so that multiple connections may be served concurrently.
		go func(connection net.Conn) {
			// Create the client and store it
			client := &Client{
				Id:      connection.RemoteAddr().String(),
				State:   STATE_DEFAULT,
				Input:   []string{},
				Command: "",
				Record:  &Record{},
			}
			clients[connection.RemoteAddr().String()] = client

			// Ensure the client is tidied up once they're done
			defer func(connection net.Conn, clients map[string]*Client, id string) {
				delete(clients, id)
				connection.Close()
			}(connection, clients, client.Id)

			// Create a new scanner for the client input
			scanner := bufio.NewScanner(connection)

			// Handle each line (command)
			for scanner.Scan() {
				// Split the client input up based on spaces
				client.Input = strings.Split(scanner.Text(), " ")

				// If we're in our default state then determine
				// what command we're running
				if client.State == STATE_DEFAULT {
					// Get the command
					client.Command = client.Input[0]

					switch client.Command {
					case "get":
						client.State = STATE_COMMAND_GET
					case "set":
						client.State = STATE_COMMAND_SET
					case "delete":
						client.State = STATE_COMMAND_DELETE
					case "flush_all":
						client.State = STATE_COMMAND_FLUSHALL
					default:
						fmt.Fprintln(connection, "ERROR")
						continue
					}
				}

				// Switch on the type of command
				switch client.State {
				// Are we expecting a value from a set command?
				case STATE_EXPECTING_VALUE:
					// If the value isn't set then set it
					if client.Record.Value == "" {
						client.Record.Value = scanner.Text()
					// Otherwise append to it
					} else {
						client.Record.Value += scanner.Text()
					}

					client.Record.Value += "\r\n"

					// Count the length of the value minus the trailing \r\n
					valueLength := int64(len(client.Record.Value)) - 2

					// If the datastore is same or greater than the expected length
					// we are done with this op
					if valueLength >= client.Record.Length {
						// If it's the same length we can try and store it
						if valueLength == client.Record.Length {
							// Store the value
							datastore[client.Record.Key] = client.Record

							// Inform the client we have stored the value
							// TODO: error handling here
							fmt.Fprintln(connection, "STORED")
						// Otherwise the client has messed up
						} else {
							// Inform the client that they messed up
							fmt.Fprintln(connection, "CLIENT_ERROR")
							fmt.Fprintln(connection, valueLength)
							fmt.Fprintln(connection, "ERROR")
						}

						// Reset the clients state
						client.Reset()
					}
				// get [key1 ... keyn]
				// TODO: handling multiple key gets
				case STATE_COMMAND_GET:
					// Check if a key was passed, if so try and retrieve it
					if len(client.Input) == 2 {
						// Get the key
						key := client.Input[1]

						// Look up the record in our datastore
						record := datastore[key]

						// Did it exist?
						if record != nil {
							fmt.Fprintln(connection, fmt.Sprintf("VALUE %s %d %d", record.Key, record.Flags, record.Length))
							fmt.Fprint(connection, record.Value)
						}

						fmt.Fprintln(connection, "END")
					} else {
						fmt.Fprintln(connection, "CLIENT_ERROR")
					}

					// Reset the clients state regardless off success/failure
					client.Reset()
				// set [key] [flags] [exptime] [length] [casunique] [noreply]
				case STATE_COMMAND_SET:
					// Check the right number of arguments are passed
					if len(client.Input) == 7 {
						// Get the key name
						client.Record.Key = client.Input[1]

						// Get any flags
						client.Record.Flags, err = strconv.ParseInt(client.Input[2], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						// Get the key TTL
						client.Record.Ttl, err = strconv.ParseInt(client.Input[3], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						// Get the value length
						client.Record.Length, err = strconv.ParseInt(client.Input[4], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						// Set that we are expecting a value
						client.State = STATE_EXPECTING_VALUE
					} else {
						fmt.Fprintln(connection, "ERROR ", err)

						// Reset the clients state
						client.Reset()
					}
				// delete [key] [noreply]
				case STATE_COMMAND_DELETE:
					// Check if a key was passed, if so try and retrieve it
					if len(client.Input) == 2 {
						// Get the key
						key := client.Input[1]

						// Look up the record in our datastore
						record := datastore[key]

						// Did it exist? If so 'delete' it
						if record != nil {
							delete(datastore, key)
							fmt.Fprintln(connection, "DELETED")
						} else {
							fmt.Fprintln(connection, "NOT_FOUND")
						}

					} else {
						fmt.Fprintln(connection, "CLIENT_ERROR")
					}

					// Reset the clients state regardless off success/failure
					client.Reset()
				// quit
				case STATE_COMMAND_QUIT:
					// Not much to do here atm..
					// Eventually we will do logging etc
				// flushall [delay]
				case STATE_COMMAND_FLUSHALL:
					// Check if a delay was passed
					if len(client.Input) == 2 {
						delay, err := strconv.ParseInt(client.Input[1], 10, 64)

						if err != nil {
							fmt.Fprintln(connection, "CLIENT_ERROR ", err)
							break
						}

						time.Sleep(time.Duration(delay) * time.Second)
					}

					// Reset the datastore
					datastore = make(map[string]*Record)
					fmt.Fprintln(connection, "OK")
				}
			}

			// Print out errors to stderr
			if err := scanner.Err(); err != nil {
				fmt.Fprintln(connection, "ERROR ", err)
			}
		}(connection)
	}
}

// Reset a clients state to what it would be on first connection
func (client *Client) Reset() {
	client.State = STATE_DEFAULT
	client.Input = []string{}
	client.Command = ""
	client.Record =  &Record{}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ml

import (
	"math"
	"testing"

	"github.com/cpmech/gosl/fun"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/num"
)

// LogReg implements a logistic regression model
type LogReg struct {

	// main
	name   string     // name of this "observer"
	data   *Data      // X-y data
	params *ParamsReg // parameters: θ, b, λ
	stat   *Stat      // statistics

	// workspace
	ybar la.Vector // bar{y}: yb[i] = (1 - y[i]) / m
	l    la.Vector // vector l = b⋅o + X⋅θ [nSamples]
	hmy  la.Vector // vector e = h(l) - y [nSamples]
}

// NewLogReg returns a new LogReg object
//   Input:
//     data   -- X,y data
//     params -- θ, b, λ
//     name   -- unique name of this (observer) object
func NewLogReg(data *Data, params *ParamsReg, name string) (o *LogReg) {
	o = new(LogReg)
	o.name = name
	o.data = data
	o.data.AddObserver(o) // need to recompute yb upon changes on y
	o.params = params
	o.stat = NewStat(data, "stat_"+name)
	o.stat.Update()
	o.ybar = la.NewVector(data.Nsamples)
	o.l = la.NewVector(data.Nsamples)
	o.hmy = la.NewVector(data.Nsamples)
	o.Update() // compute first ybar
	return
}

// Name returns the name of this "Observer"
func (o *LogReg) Name() string {
	return o.name
}

// Update perform updates after data has been changed (as an Observer)
func (o *LogReg) Update() {
	m := float64(o.data.Nsamples)
	for i := 0; i < o.data.Nsamples; i++ {
		o.ybar[i] = (1.0 - o.data.Y[i]) / m
	}
}

// Predict returns the model evaluation @ {x;θ,b}
//   Input:
//     x -- vector of features
//   Output:
//     y -- model prediction y(x)
func (o *LogReg) Predict(x la.Vector) (y float64) {
	θ := o.params.AccessThetas()
	b := o.params.GetBias()
	return fun.Logistic(b + la.VecDot(x, θ)) // g(b + xᵀθ) where g is logistic
}

// Cost returns the cost c(x;θ,b)
//   Input:
//     data -- X,y data
//     params -- θ and b
//     x -- vector of features
//   Output:
//     c -- total cost (model error)
func (o *LogReg) Cost() (c float64) {

	// auxiliary
	m := float64(o.data.Nsamples)
	λ := o.params.GetLambda()
	θ := o.params.AccessThetas()

	// cost
	o.calcl()
	sq := o.calcsumq()
	c = sq/m + la.VecDot(o.ybar, o.l)
	if λ > 0 {
		c += (0.5 * λ / m) * la.VecDot(θ, θ) // c += (0.5λ/m) θᵀθ
	}
	return c
}

// Gradients returns ∂C/∂θ and ∂C/∂b
//   Output:
//     dCdθ -- ∂C/∂θ
//     dCdb -- ∂C/∂b
func (o *LogReg) Gradients(dCdθ la.Vector) (dCdb float64) {

	// auxiliary
	m := float64(o.data.Nsamples)
	λ := o.params.GetLambda()
	θ := o.params.AccessThetas()
	X := o.data.X

	// dCdθ
	o.calcl()                             // l := b + X⋅θ
	o.calchmy()                           // hmy := h(l) - y
	la.MatTrVecMul(dCdθ, 1.0/m, X, o.hmy) // dCdθ := (1/m) Xᵀhmy
	if λ > 0 {
		la.VecAdd(dCdθ, 1, dCdθ, λ/m, θ) // dCdθ += (1/m) θ
	}

	// dCdb
	dCdb = (1.0 / m) * o.hmy.Accum() // dCdb = (1/m) oᵀhmy
	return
}

// AllocateHessian allocate objects to compute Hessian
func (o *LogReg) AllocateHessian() (d, v la.Vector, D, H *la.Matrix) {
	m := o.data.Nsamples
	n := o.data.Nfeatures
	d = la.NewVector(m)
	v = la.NewVector(n)
	D = la.NewMatrix(m, n)
	H = la.NewMatrix(n, n)
	return
}

// Hessian computes the Hessian matrix and other partial derivatives
//
//   Input, if d !=nil, otherwise allocate these four objects:
//     d -- [nSamples]  d[i] = g(l[i]) * [ 1 - g(l[i]) ]  auxiliary vector
//     v -- [nFeatures] v = ∂²C/∂θ∂b second order partial derivative
//     D -- [nSamples][nFeatures]  D[i][j] = d[i]*X[i][j]  auxiliary matrix
//     H -- [nFeatures][nFeatures]  H = ∂²C/∂θ² Hessian matrix
//
//   Output, either new objectos or pointers to the input ones:
//     dNew := d   (allocated here if d == nil)
//     vNew := v   (allocated here if v == nil)
//     Dnew := D   (allocated here if D == nil)
//     Hnew := H   (allocated here if H == nil)
//     w -- H = ∂²C/∂b²
//
func (o *LogReg) Hessian(d, v la.Vector, D, H *la.Matrix) (w float64) {

	// auxiliary
	m := o.data.Nsamples
	n := o.data.Nfeatures
	X := o.data.X
	λ := o.params.GetLambda()
	mm := float64(m)

	// calc d vector and D matrix
	o.calcl()
	for i := 0; i < m; i++ {
		d[i] = fun.LogisticD1(o.l[i]) // d vector
		for j := 0; j < n; j++ {
			D.Set(i, j, d[i]*X.Get(i, j)) // D matrix   (TODO: optimize this)
		}
	}

	// calc H matrix
	la.MatTrMatMul(H, 1.0/mm, X, D)
	if λ > 0 {
		for i := 0; i < n; i++ {
			H.Set(i, i, H.Get(i, i)+λ/mm) // D += (λ/m) I   (TODO: optimize here?)
		}
	}

	// calc v
	la.MatTrVecMul(v, 1.0/mm, X, d) // v := (1/m) Xᵀd

	// calc w
	w = d.Accum() / mm
	return
}

// Train finds θ and b using closed-form solution
//   Input:
//     data -- X,y data
//   Output:
//     params -- θ and b
func (o *LogReg) Train() {

	// auxiliary
	//m := o.data.Nsamples
	n := o.data.Nfeatures

	// allocate arrays
	//dCdθ := la.NewVector(o.data.Nfeatures)
	var w float64
	d, v, D, H := o.AllocateHessian()

	// objective function where z={θ,b} and fz={dCdθ,dCdb}
	ffcn := func(fz, z la.Vector) {
		o.params.Backup()
		o.params.SetThetas(z[:n])
		o.params.SetBias(z[n])
		dCdb := o.Gradients(fz[:n])
		fz[n] = dCdb
		o.params.Restore(false)
	}

	// Jacobian function
	Jfcn := func(dfdz *la.Matrix, z la.Vector) {
		o.params.Backup()
		o.params.SetThetas(z[:n])
		o.params.SetBias(z[n])
		w = o.Hessian(d, v, D, H)
		for j := 0; j < n; j++ { // TODO: optimize here
			for i := 0; i < n; i++ { //
				dfdz.Set(i, j, H.Get(i, j))
			}
			dfdz.Set(n, j, v[j])
			dfdz.Set(j, n, v[j])
		}
		dfdz.Set(n, n, w)
	}

	// initial values
	z := la.NewVector(n + 1) // {θ, b}
	copy(z, o.params.AccessThetas())
	z[n] = o.params.GetBias()

	// debug
	if true { // check Jacobian
		tolJac0 := 1e-4
		tst := new(testing.T)
		num.CompareJacDense(tst, ffcn, Jfcn, z, tolJac0)
	}

	// solver parameters
	solverParams := map[string]float64{
		"atol":    1e-4,
		"rtol":    1e-4,
		"ftol":    1e-4,
		"chkConv": 0,
	}

	// solve nonlinear problem
	silent := false
	useDenseJacobian := true
	numericalJacobian := false
	var solver num.NlSolver
	defer solver.Free()
	solver.Init(n+1, ffcn, nil, Jfcn, useDenseJacobian, numericalJacobian, solverParams)
	solver.Solve(z, silent)

	// results
	o.params.SetThetas(z[:n])
	o.params.SetBias(z[n])

	// debug
	if true { // check Jacobian
		tolJac0 := 1e-4
		tst := new(testing.T)
		num.CompareJacDense(tst, ffcn, Jfcn, z, tolJac0)
	}
}

// auxiliary ///////////////////////////////////////////////////////////////////////////////////////

// calce calculates l vector (saves into o.l) (linear model)
//  Output: l = b⋅o + X⋅θ
func (o *LogReg) calcl() {
	θ := o.params.AccessThetas()
	b := o.params.GetBias()
	X := o.data.X
	o.l.Fill(b)                   // l := b⋅o
	la.MatVecMulAdd(o.l, 1, X, θ) // l := b⋅o + X⋅θ
}

// calcsumq calculates Σq[i]
//  Input:
//    l -- precomputed o.l
//  Output:
//    sq -- sum(q)
func (o *LogReg) calcsumq() (sq float64) {
	for i := 0; i < o.data.Nsamples; i++ {
		sq += safeLog1pExp(o.l[i])
	}
	return
}

// calchmy calculates h(l) - y vector (saves into o.hmy)
//  Input:
//    l -- precomputed o.l
//  Output:
//    hmy -- computes hmy = h(l) - y
func (o *LogReg) calchmy() {
	for i := 0; i < o.data.Nsamples; i++ {
		o.hmy[i] = fun.Logistic(o.l[i]) - o.data.Y[i]
	}
}

// safeLog1pExp computes log(1+exp(-z)) safely by checking if exp(-z) is >> 1,
// thus returning -z. This is the case when z<0 and |z| is too large
func safeLog1pExp(z float64) float64 {
	if z < -500 {
		return -z
	}
	return math.Log(1.0 + math.Exp(-z))
}
<commit_msg>Add comment about LogReg being an Obsever of Data<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ml

import (
	"math"
	"testing"

	"github.com/cpmech/gosl/fun"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/num"
)

// LogReg implements a logistic regression model (Observer of Data)
type LogReg struct {

	// main
	name   string     // name of this "observer"
	data   *Data      // X-y data
	params *ParamsReg // parameters: θ, b, λ
	stat   *Stat      // statistics

	// workspace
	ybar la.Vector // bar{y}: yb[i] = (1 - y[i]) / m
	l    la.Vector // vector l = b⋅o + X⋅θ [nSamples]
	hmy  la.Vector // vector e = h(l) - y [nSamples]
}

// NewLogReg returns a new LogReg object
//   Input:
//     data   -- X,y data
//     params -- θ, b, λ
//     name   -- unique name of this (observer) object
func NewLogReg(data *Data, params *ParamsReg, name string) (o *LogReg) {
	o = new(LogReg)
	o.name = name
	o.data = data
	o.data.AddObserver(o) // need to recompute yb upon changes on y
	o.params = params
	o.stat = NewStat(data, "stat_"+name)
	o.stat.Update()
	o.ybar = la.NewVector(data.Nsamples)
	o.l = la.NewVector(data.Nsamples)
	o.hmy = la.NewVector(data.Nsamples)
	o.Update() // compute first ybar
	return
}

// Name returns the name of this "Observer"
func (o *LogReg) Name() string {
	return o.name
}

// Update perform updates after data has been changed (as an Observer)
func (o *LogReg) Update() {
	m := float64(o.data.Nsamples)
	for i := 0; i < o.data.Nsamples; i++ {
		o.ybar[i] = (1.0 - o.data.Y[i]) / m
	}
}

// Predict returns the model evaluation @ {x;θ,b}
//   Input:
//     x -- vector of features
//   Output:
//     y -- model prediction y(x)
func (o *LogReg) Predict(x la.Vector) (y float64) {
	θ := o.params.AccessThetas()
	b := o.params.GetBias()
	return fun.Logistic(b + la.VecDot(x, θ)) // g(b + xᵀθ) where g is logistic
}

// Cost returns the cost c(x;θ,b)
//   Input:
//     data -- X,y data
//     params -- θ and b
//     x -- vector of features
//   Output:
//     c -- total cost (model error)
func (o *LogReg) Cost() (c float64) {

	// auxiliary
	m := float64(o.data.Nsamples)
	λ := o.params.GetLambda()
	θ := o.params.AccessThetas()

	// cost
	o.calcl()
	sq := o.calcsumq()
	c = sq/m + la.VecDot(o.ybar, o.l)
	if λ > 0 {
		c += (0.5 * λ / m) * la.VecDot(θ, θ) // c += (0.5λ/m) θᵀθ
	}
	return c
}

// Gradients returns ∂C/∂θ and ∂C/∂b
//   Output:
//     dCdθ -- ∂C/∂θ
//     dCdb -- ∂C/∂b
func (o *LogReg) Gradients(dCdθ la.Vector) (dCdb float64) {

	// auxiliary
	m := float64(o.data.Nsamples)
	λ := o.params.GetLambda()
	θ := o.params.AccessThetas()
	X := o.data.X

	// dCdθ
	o.calcl()                             // l := b + X⋅θ
	o.calchmy()                           // hmy := h(l) - y
	la.MatTrVecMul(dCdθ, 1.0/m, X, o.hmy) // dCdθ := (1/m) Xᵀhmy
	if λ > 0 {
		la.VecAdd(dCdθ, 1, dCdθ, λ/m, θ) // dCdθ += (1/m) θ
	}

	// dCdb
	dCdb = (1.0 / m) * o.hmy.Accum() // dCdb = (1/m) oᵀhmy
	return
}

// AllocateHessian allocate objects to compute Hessian
func (o *LogReg) AllocateHessian() (d, v la.Vector, D, H *la.Matrix) {
	m := o.data.Nsamples
	n := o.data.Nfeatures
	d = la.NewVector(m)
	v = la.NewVector(n)
	D = la.NewMatrix(m, n)
	H = la.NewMatrix(n, n)
	return
}

// Hessian computes the Hessian matrix and other partial derivatives
//
//   Input, if d !=nil, otherwise allocate these four objects:
//     d -- [nSamples]  d[i] = g(l[i]) * [ 1 - g(l[i]) ]  auxiliary vector
//     v -- [nFeatures] v = ∂²C/∂θ∂b second order partial derivative
//     D -- [nSamples][nFeatures]  D[i][j] = d[i]*X[i][j]  auxiliary matrix
//     H -- [nFeatures][nFeatures]  H = ∂²C/∂θ² Hessian matrix
//
//   Output, either new objectos or pointers to the input ones:
//     dNew := d   (allocated here if d == nil)
//     vNew := v   (allocated here if v == nil)
//     Dnew := D   (allocated here if D == nil)
//     Hnew := H   (allocated here if H == nil)
//     w -- H = ∂²C/∂b²
//
func (o *LogReg) Hessian(d, v la.Vector, D, H *la.Matrix) (w float64) {

	// auxiliary
	m := o.data.Nsamples
	n := o.data.Nfeatures
	X := o.data.X
	λ := o.params.GetLambda()
	mm := float64(m)

	// calc d vector and D matrix
	o.calcl()
	for i := 0; i < m; i++ {
		d[i] = fun.LogisticD1(o.l[i]) // d vector
		for j := 0; j < n; j++ {
			D.Set(i, j, d[i]*X.Get(i, j)) // D matrix   (TODO: optimize this)
		}
	}

	// calc H matrix
	la.MatTrMatMul(H, 1.0/mm, X, D)
	if λ > 0 {
		for i := 0; i < n; i++ {
			H.Set(i, i, H.Get(i, i)+λ/mm) // D += (λ/m) I   (TODO: optimize here?)
		}
	}

	// calc v
	la.MatTrVecMul(v, 1.0/mm, X, d) // v := (1/m) Xᵀd

	// calc w
	w = d.Accum() / mm
	return
}

// Train finds θ and b using closed-form solution
//   Input:
//     data -- X,y data
//   Output:
//     params -- θ and b
func (o *LogReg) Train() {

	// auxiliary
	//m := o.data.Nsamples
	n := o.data.Nfeatures

	// allocate arrays
	//dCdθ := la.NewVector(o.data.Nfeatures)
	var w float64
	d, v, D, H := o.AllocateHessian()

	// objective function where z={θ,b} and fz={dCdθ,dCdb}
	ffcn := func(fz, z la.Vector) {
		o.params.Backup()
		o.params.SetThetas(z[:n])
		o.params.SetBias(z[n])
		dCdb := o.Gradients(fz[:n])
		fz[n] = dCdb
		o.params.Restore(false)
	}

	// Jacobian function
	Jfcn := func(dfdz *la.Matrix, z la.Vector) {
		o.params.Backup()
		o.params.SetThetas(z[:n])
		o.params.SetBias(z[n])
		w = o.Hessian(d, v, D, H)
		for j := 0; j < n; j++ { // TODO: optimize here
			for i := 0; i < n; i++ { //
				dfdz.Set(i, j, H.Get(i, j))
			}
			dfdz.Set(n, j, v[j])
			dfdz.Set(j, n, v[j])
		}
		dfdz.Set(n, n, w)
	}

	// initial values
	z := la.NewVector(n + 1) // {θ, b}
	copy(z, o.params.AccessThetas())
	z[n] = o.params.GetBias()

	// debug
	if true { // check Jacobian
		tolJac0 := 1e-4
		tst := new(testing.T)
		num.CompareJacDense(tst, ffcn, Jfcn, z, tolJac0)
	}

	// solver parameters
	solverParams := map[string]float64{
		"atol":    1e-4,
		"rtol":    1e-4,
		"ftol":    1e-4,
		"chkConv": 0,
	}

	// solve nonlinear problem
	silent := false
	useDenseJacobian := true
	numericalJacobian := false
	var solver num.NlSolver
	defer solver.Free()
	solver.Init(n+1, ffcn, nil, Jfcn, useDenseJacobian, numericalJacobian, solverParams)
	solver.Solve(z, silent)

	// results
	o.params.SetThetas(z[:n])
	o.params.SetBias(z[n])

	// debug
	if true { // check Jacobian
		tolJac0 := 1e-4
		tst := new(testing.T)
		num.CompareJacDense(tst, ffcn, Jfcn, z, tolJac0)
	}
}

// auxiliary ///////////////////////////////////////////////////////////////////////////////////////

// calce calculates l vector (saves into o.l) (linear model)
//  Output: l = b⋅o + X⋅θ
func (o *LogReg) calcl() {
	θ := o.params.AccessThetas()
	b := o.params.GetBias()
	X := o.data.X
	o.l.Fill(b)                   // l := b⋅o
	la.MatVecMulAdd(o.l, 1, X, θ) // l := b⋅o + X⋅θ
}

// calcsumq calculates Σq[i]
//  Input:
//    l -- precomputed o.l
//  Output:
//    sq -- sum(q)
func (o *LogReg) calcsumq() (sq float64) {
	for i := 0; i < o.data.Nsamples; i++ {
		sq += safeLog1pExp(o.l[i])
	}
	return
}

// calchmy calculates h(l) - y vector (saves into o.hmy)
//  Input:
//    l -- precomputed o.l
//  Output:
//    hmy -- computes hmy = h(l) - y
func (o *LogReg) calchmy() {
	for i := 0; i < o.data.Nsamples; i++ {
		o.hmy[i] = fun.Logistic(o.l[i]) - o.data.Y[i]
	}
}

// safeLog1pExp computes log(1+exp(-z)) safely by checking if exp(-z) is >> 1,
// thus returning -z. This is the case when z<0 and |z| is too large
func safeLog1pExp(z float64) float64 {
	if z < -500 {
		return -z
	}
	return math.Log(1.0 + math.Exp(-z))
}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"fmt"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/edsrzf/mmap-go"
)

func TestMultipleMMapsOnSameFile(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossMMap")
	defer os.RemoveAll(tmpDir)

	f, err := os.Create(tmpDir + string(os.PathSeparator) + "test.file")
	if err != nil {
		t.Errorf("expected open file to work, err: %v", err)
	}

	offset := 1024 * 1024 * 1024 // 1 GB.

	f.WriteAt([]byte("hello"), int64(offset))

	var mms []mmap.MMap

	for i := 0; i < 100; i++ { // Re-mmap the file.
		mm, err := mmap.Map(f, mmap.RDONLY, 0)
		if err != nil {
			t.Errorf("expected mmap to work, err: %v", err)
		}

		if string(mm[offset:offset+5]) != "hello" {
			t.Errorf("expected hello")
		}

		mms = append(mms, mm)
	}

	for _, mm := range mms {
		if string(mm[offset:offset+5]) != "hello" {
			t.Errorf("expected hello")
		}

		for j := 0; j < offset; j += 1024 * 1024 {
			if mm[j] != 0 {
				t.Errorf("expected 0")
			}
		}
	}

	for _, mm := range mms {
		mm.Unmap()
	}

	f.Close()
}

func TestMMapRef(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossStore")
	defer os.RemoveAll(tmpDir)

	var mu sync.Mutex
	counts := map[EventKind]int{}
	eventWaiters := map[EventKind]chan bool{}

	co := CollectionOptions{
		MergeOperator: &MergeOperatorStringAppend{Sep: ":"},
		OnEvent: func(event Event) {
			mu.Lock()
			counts[event.Kind]++
			eventWaiter := eventWaiters[event.Kind]
			mu.Unlock()
			if eventWaiter != nil {
				eventWaiter <- true
			}
		},
	}

	store, m, err := OpenStoreCollection(tmpDir, StoreOptions{
		CollectionOptions: co,
	}, StorePersistOptions{})
	if err != nil || m == nil || store == nil {
		t.Errorf("expected open empty store collection to work")
	}

	b, _ := m.NewBatch(0, 0)
	for i := 0; i < 1000; i++ {
		xs := fmt.Sprintf("%d", i)
		x := []byte(xs)
		b.Set(x, x)
	}
	err = m.ExecuteBatch(b, WriteOptions{})
	if err != nil {
		t.Errorf("expected exec batch to work")
	}
	b.Close()

	waitUntilClean := func() error {
		for {
			stats, err := m.Stats()
			if err != nil {
				return err
			}

			if stats.CurDirtyOps <= 0 &&
				stats.CurDirtyBytes <= 0 &&
				stats.CurDirtySegments <= 0 {
				break
			}

			time.Sleep(200 * time.Millisecond)
		}

		return nil
	}

	waitUntilClean()

	store.m.Lock()

	if store.refs != 1 {
		t.Errorf("expected 1 store ref, got: %d", store.refs)
	}

	footer := store.footer
	if footer == nil {
		t.Errorf("expected footer")
	}

	footer.m.Lock()
	if footer.refs != 2 {
		t.Errorf("expected 2 footer ref, : got: %d", footer.refs)
	}

	mref := footer.mref
	if mref == nil {
		t.Errorf("expected mref")
	}

	mrefsCheck := func(expected int) {
		mref.m.Lock()
		if mref.refs != expected {
			t.Errorf("expected mref.refs to be %d, got: %d", expected, mref.refs)
		}
		mref.m.Unlock()
	}

	mrefsCheck(1)

	rv := mref.AddRef()

	mrefsCheck(2)

	if rv != mref {
		t.Errorf("expected rv == mref")
	}

	if mref.DecRef() != nil {
		t.Errorf("expected mref.DecRef to be nil")
	}

	mrefsCheck(1)

	footer.m.Unlock()

	store.m.Unlock()

	m.Close()

	store.Close()

	footer.m.Lock()
	if footer.refs != 0 {
		t.Errorf("expected footer refs to be 0, got: %d", footer.refs)
	}
	footer.m.Unlock()

	store.m.Lock()
	if store.refs != 0 {
		t.Errorf("expected store refs to be 0, got: %d", footer.refs)
	}
	store.m.Unlock()

	mrefsCheck(0)
}

func TestRefCounting(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossStore")
	defer os.RemoveAll(tmpDir)

	var mu sync.Mutex
	counts := map[EventKind]int{}
	eventWaiters := map[EventKind]chan bool{}

	co := CollectionOptions{
		OnEvent: func(event Event) {
			mu.Lock()
			counts[event.Kind]++
			eventWaiter := eventWaiters[event.Kind]
			mu.Unlock()
			if eventWaiter != nil {
				eventWaiter <- true
			}
		},
	}

	store, m, err := OpenStoreCollection(tmpDir, StoreOptions{
		CollectionOptions:    co,
		CompactionPercentage: 10000.0,
	}, StorePersistOptions{
		CompactionConcern: CompactionAllow,
	})
	if err != nil || m == nil || store == nil {
		t.Errorf("expected open empty store collection to work")
	}

	// ---------------------------------------------

	checkRefs := func(f *Footer, frefs, mrefs int, cb func(), msg string) {
		f.m.Lock()

		if f.refs != frefs {
			t.Errorf("%s - expected footer.refs to be %d, got: %d",
				msg, frefs, f.refs)
		}

		if f.mref != nil {
			f.mref.m.Lock()

			if f.mref.refs != mrefs {
				t.Errorf("%s - expected mrefs to be %d, got: %d",
					msg, mrefs, f.mref.refs)
			}

			if cb != nil {
				cb()
			}

			f.mref.m.Unlock()
		} else if mrefs > 0 {
			t.Errorf("%s - expected footer.mref to be %d, but nil mref",
				msg, mrefs)
		}

		f.m.Unlock()
	}

	writeHello := func() {
		b, _ := m.NewBatch(0, 0)
		b.Set([]byte("hello"), []byte("world"))
		err = m.ExecuteBatch(b, WriteOptions{})
		if err != nil {
			t.Errorf("expected exec batch to work")
		}
		b.Close()
	}

	waitUntilClean := func() error {
		for {
			stats, err := m.Stats()
			if err != nil {
				return err
			}

			if stats.CurDirtyOps <= 0 &&
				stats.CurDirtyBytes <= 0 &&
				stats.CurDirtySegments <= 0 {
				break
			}

			time.Sleep(time.Millisecond)
		}

		return nil
	}

	countFooters := func() (*Footer, int) {
		fx := store.footer
		fxNum := 1
		for fx.prevFooter != nil {
			fx = fx.prevFooter
			fxNum++
		}
		return fx, fxNum
	}

	// ---------------------------------------------

	store.m.Lock()

	checkRefs(store.footer, 2, 0, nil, "new, empty store")

	_, numFooters := countFooters()
	if numFooters != 1 {
		t.Errorf("expected only 1 footer before 1st batch, got: %d",
			numFooters)
	}

	store.m.Unlock()

	// ---------------------------------------------

	writeHello()

	waitUntilClean()

	// ---------------------------------------------

	store.m.Lock()

	checkRefs(store.footer, 2, 1, nil, "after 1st batch persisted")

	_, numFooters = countFooters()
	if numFooters != 2 {
		// Two footers expected because the oldest, "empty" footer is
		// still in the chain at this point, since the mrefRefresh() /
		// splicing runs before the DecRef of the
		// initial-lower-level-snapshot.
		t.Errorf("expected 2 footers after 1st batch, got: %d",
			numFooters)
	}

	store.m.Unlock()

	// ---------------------------------------------

	ss0, err := store.Snapshot()
	if err != nil {
		t.Errorf("expected no err on snapshot")
	}

	f0, ok := ss0.(*Footer)
	if !ok {
		t.Errorf("expected Footer")
	}

	store.m.Lock()

	checkRefs(store.footer, 3, 1, nil, "after 1st batch persisted")

	oldestFooter, numFooters := countFooters()
	if numFooters != 2 {
		t.Errorf("expected only 2 footers after 1st ss")
	}
	if oldestFooter == f0 {
		// Expecting oldest footer to be the original "empty"
		// footer right now.
		t.Errorf("expected oldest footer to not be f0")
	}

	store.m.Unlock()

	for i := 0; i < 10; i++ {
		writeHello()
		waitUntilClean()
	}

	store.m.Lock()

	if f0 == store.footer {
		t.Errorf("expected curr footer to be different than f0")
	}

	checkRefs(store.footer, 2, 2, nil, "after nth batch persisted")

	oldestFooter, numFooters = countFooters()
	if oldestFooter != f0 {
		t.Errorf("expected fx to be same as oldest snapshot/footer")
	}
	if numFooters != 3 {
		t.Errorf("expected only 3 footers in chain after many batches,"+
			" got: %d", numFooters)
	}

	store.m.Unlock()

	var mref *mmapRef

	checkRefs(f0, 1, 2, func() {
		mref = f0.mref
		if f0.prevFooter != nil {
			t.Errorf("expected oldest footer to have nil prevFooter")
		}
	}, "oldest footer check")

	// ----------------------------------------

	ss0.Close() // Close the first, oldest snapshot.

	store.m.Lock()

	checkRefs(store.footer, 2, 1, nil, "after oldest snapshot closed")

	_, numFooters = countFooters()
	if numFooters != 3 {
		// The reason why is a persistence cycle hasn't run yet
		// that can cleanup the footer chain of unused footers.
		t.Errorf("expected 3 footers still in chain after ss0 closed,"+
			" got %d", numFooters)
	}

	store.m.Unlock()

	checkRefs(f0, 0, 0, nil, "oldest footer after ss0.Close()")

	mref.m.Lock()
	if mref.refs != 1 {
		t.Errorf("expected 1")
	}
	mref.m.Unlock()

	// ----------------------------------------

	m.Close()

	var fLast *Footer

	store.m.Lock()
	fLast = store.footer
	checkRefs(store.footer, 1, 1, nil, "after collection Close()'ed")
	store.m.Unlock()

	mref.m.Lock()
	if mref.refs != 1 {
		t.Errorf("expected 1")
	}
	mref.m.Unlock()

	// ----------------------------------------

	store.Close()

	mref.m.Lock()
	if mref.refs != 0 {
		t.Errorf("expected 0 mref.refs after everything closed")
	}
	mref.m.Unlock()

	checkRefs(fLast, 0, 0, nil, "last footer after store.Close()")

	checkRefs(f0, 0, 0, nil, "oldest footer after store.Close()")
}
<commit_msg>MB-20899 - TestAccessAfterUnmap (skipped)<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"fmt"
	"io/ioutil"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/edsrzf/mmap-go"
)

func TestMultipleMMapsOnSameFile(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossMMap")
	defer os.RemoveAll(tmpDir)

	f, err := os.Create(tmpDir + string(os.PathSeparator) + "test.file")
	if err != nil {
		t.Errorf("expected open file to work, err: %v", err)
	}

	offset := 1024 * 1024 * 1024 // 1 GB.

	f.WriteAt([]byte("hello"), int64(offset))

	var mms []mmap.MMap

	for i := 0; i < 100; i++ { // Re-mmap the file.
		mm, err := mmap.Map(f, mmap.RDONLY, 0)
		if err != nil {
			t.Errorf("expected mmap to work, err: %v", err)
		}

		if string(mm[offset:offset+5]) != "hello" {
			t.Errorf("expected hello")
		}

		mms = append(mms, mm)
	}

	for _, mm := range mms {
		if string(mm[offset:offset+5]) != "hello" {
			t.Errorf("expected hello")
		}

		for j := 0; j < offset; j += 1024 * 1024 {
			if mm[j] != 0 {
				t.Errorf("expected 0")
			}
		}
	}

	for _, mm := range mms {
		mm.Unmap()
	}

	f.Close()
}

func TestMMapRef(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossStore")
	defer os.RemoveAll(tmpDir)

	var mu sync.Mutex
	counts := map[EventKind]int{}
	eventWaiters := map[EventKind]chan bool{}

	co := CollectionOptions{
		MergeOperator: &MergeOperatorStringAppend{Sep: ":"},
		OnEvent: func(event Event) {
			mu.Lock()
			counts[event.Kind]++
			eventWaiter := eventWaiters[event.Kind]
			mu.Unlock()
			if eventWaiter != nil {
				eventWaiter <- true
			}
		},
	}

	store, m, err := OpenStoreCollection(tmpDir, StoreOptions{
		CollectionOptions: co,
	}, StorePersistOptions{})
	if err != nil || m == nil || store == nil {
		t.Errorf("expected open empty store collection to work")
	}

	b, _ := m.NewBatch(0, 0)
	for i := 0; i < 1000; i++ {
		xs := fmt.Sprintf("%d", i)
		x := []byte(xs)
		b.Set(x, x)
	}
	err = m.ExecuteBatch(b, WriteOptions{})
	if err != nil {
		t.Errorf("expected exec batch to work")
	}
	b.Close()

	waitUntilClean := func() error {
		for {
			stats, err := m.Stats()
			if err != nil {
				return err
			}

			if stats.CurDirtyOps <= 0 &&
				stats.CurDirtyBytes <= 0 &&
				stats.CurDirtySegments <= 0 {
				break
			}

			time.Sleep(200 * time.Millisecond)
		}

		return nil
	}

	waitUntilClean()

	store.m.Lock()

	if store.refs != 1 {
		t.Errorf("expected 1 store ref, got: %d", store.refs)
	}

	footer := store.footer
	if footer == nil {
		t.Errorf("expected footer")
	}

	footer.m.Lock()
	if footer.refs != 2 {
		t.Errorf("expected 2 footer ref, : got: %d", footer.refs)
	}

	mref := footer.mref
	if mref == nil {
		t.Errorf("expected mref")
	}

	mrefsCheck := func(expected int) {
		mref.m.Lock()
		if mref.refs != expected {
			t.Errorf("expected mref.refs to be %d, got: %d", expected, mref.refs)
		}
		mref.m.Unlock()
	}

	mrefsCheck(1)

	rv := mref.AddRef()

	mrefsCheck(2)

	if rv != mref {
		t.Errorf("expected rv == mref")
	}

	if mref.DecRef() != nil {
		t.Errorf("expected mref.DecRef to be nil")
	}

	mrefsCheck(1)

	footer.m.Unlock()

	store.m.Unlock()

	m.Close()

	store.Close()

	footer.m.Lock()
	if footer.refs != 0 {
		t.Errorf("expected footer refs to be 0, got: %d", footer.refs)
	}
	footer.m.Unlock()

	store.m.Lock()
	if store.refs != 0 {
		t.Errorf("expected store refs to be 0, got: %d", footer.refs)
	}
	store.m.Unlock()

	mrefsCheck(0)
}

func TestRefCounting(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossStore")
	defer os.RemoveAll(tmpDir)

	var mu sync.Mutex
	counts := map[EventKind]int{}
	eventWaiters := map[EventKind]chan bool{}

	co := CollectionOptions{
		OnEvent: func(event Event) {
			mu.Lock()
			counts[event.Kind]++
			eventWaiter := eventWaiters[event.Kind]
			mu.Unlock()
			if eventWaiter != nil {
				eventWaiter <- true
			}
		},
	}

	store, m, err := OpenStoreCollection(tmpDir, StoreOptions{
		CollectionOptions:    co,
		CompactionPercentage: 10000.0,
	}, StorePersistOptions{
		CompactionConcern: CompactionAllow,
	})
	if err != nil || m == nil || store == nil {
		t.Errorf("expected open empty store collection to work")
	}

	// ---------------------------------------------

	checkRefs := func(f *Footer, frefs, mrefs int, cb func(), msg string) {
		f.m.Lock()

		if f.refs != frefs {
			t.Errorf("%s - expected footer.refs to be %d, got: %d",
				msg, frefs, f.refs)
		}

		if f.mref != nil {
			f.mref.m.Lock()

			if f.mref.refs != mrefs {
				t.Errorf("%s - expected mrefs to be %d, got: %d",
					msg, mrefs, f.mref.refs)
			}

			if cb != nil {
				cb()
			}

			f.mref.m.Unlock()
		} else if mrefs > 0 {
			t.Errorf("%s - expected footer.mref to be %d, but nil mref",
				msg, mrefs)
		}

		f.m.Unlock()
	}

	writeHello := func() {
		b, _ := m.NewBatch(0, 0)
		b.Set([]byte("hello"), []byte("world"))
		err = m.ExecuteBatch(b, WriteOptions{})
		if err != nil {
			t.Errorf("expected exec batch to work")
		}
		b.Close()
	}

	waitUntilClean := func() error {
		for {
			stats, err := m.Stats()
			if err != nil {
				return err
			}

			if stats.CurDirtyOps <= 0 &&
				stats.CurDirtyBytes <= 0 &&
				stats.CurDirtySegments <= 0 {
				break
			}

			time.Sleep(time.Millisecond)
		}

		return nil
	}

	countFooters := func() (*Footer, int) {
		fx := store.footer
		fxNum := 1
		for fx.prevFooter != nil {
			fx = fx.prevFooter
			fxNum++
		}
		return fx, fxNum
	}

	// ---------------------------------------------

	store.m.Lock()

	checkRefs(store.footer, 2, 0, nil, "new, empty store")

	_, numFooters := countFooters()
	if numFooters != 1 {
		t.Errorf("expected only 1 footer before 1st batch, got: %d",
			numFooters)
	}

	store.m.Unlock()

	// ---------------------------------------------

	writeHello()

	waitUntilClean()

	// ---------------------------------------------

	store.m.Lock()

	checkRefs(store.footer, 2, 1, nil, "after 1st batch persisted")

	_, numFooters = countFooters()
	if numFooters != 2 {
		// Two footers expected because the oldest, "empty" footer is
		// still in the chain at this point, since the mrefRefresh() /
		// splicing runs before the DecRef of the
		// initial-lower-level-snapshot.
		t.Errorf("expected 2 footers after 1st batch, got: %d",
			numFooters)
	}

	store.m.Unlock()

	// ---------------------------------------------

	ss0, err := store.Snapshot()
	if err != nil {
		t.Errorf("expected no err on snapshot")
	}

	f0, ok := ss0.(*Footer)
	if !ok {
		t.Errorf("expected Footer")
	}

	store.m.Lock()

	checkRefs(store.footer, 3, 1, nil, "after 1st batch persisted")

	oldestFooter, numFooters := countFooters()
	if numFooters != 2 {
		t.Errorf("expected only 2 footers after 1st ss")
	}
	if oldestFooter == f0 {
		// Expecting oldest footer to be the original "empty"
		// footer right now.
		t.Errorf("expected oldest footer to not be f0")
	}

	store.m.Unlock()

	for i := 0; i < 10; i++ {
		writeHello()
		waitUntilClean()
	}

	store.m.Lock()

	if f0 == store.footer {
		t.Errorf("expected curr footer to be different than f0")
	}

	checkRefs(store.footer, 2, 2, nil, "after nth batch persisted")

	oldestFooter, numFooters = countFooters()
	if oldestFooter != f0 {
		t.Errorf("expected fx to be same as oldest snapshot/footer")
	}
	if numFooters != 3 {
		t.Errorf("expected only 3 footers in chain after many batches,"+
			" got: %d", numFooters)
	}

	store.m.Unlock()

	var mref *mmapRef

	checkRefs(f0, 1, 2, func() {
		mref = f0.mref
		if f0.prevFooter != nil {
			t.Errorf("expected oldest footer to have nil prevFooter")
		}
	}, "oldest footer check")

	// ----------------------------------------

	ss0.Close() // Close the first, oldest snapshot.

	store.m.Lock()

	checkRefs(store.footer, 2, 1, nil, "after oldest snapshot closed")

	_, numFooters = countFooters()
	if numFooters != 3 {
		// The reason why is a persistence cycle hasn't run yet
		// that can cleanup the footer chain of unused footers.
		t.Errorf("expected 3 footers still in chain after ss0 closed,"+
			" got %d", numFooters)
	}

	store.m.Unlock()

	checkRefs(f0, 0, 0, nil, "oldest footer after ss0.Close()")

	mref.m.Lock()
	if mref.refs != 1 {
		t.Errorf("expected 1")
	}
	mref.m.Unlock()

	// ----------------------------------------

	m.Close()

	var fLast *Footer

	store.m.Lock()
	fLast = store.footer
	checkRefs(store.footer, 1, 1, nil, "after collection Close()'ed")
	store.m.Unlock()

	mref.m.Lock()
	if mref.refs != 1 {
		t.Errorf("expected 1")
	}
	mref.m.Unlock()

	// ----------------------------------------

	store.Close()

	mref.m.Lock()
	if mref.refs != 0 {
		t.Errorf("expected 0 mref.refs after everything closed")
	}
	mref.m.Unlock()

	checkRefs(fLast, 0, 0, nil, "last footer after store.Close()")

	checkRefs(f0, 0, 0, nil, "oldest footer after store.Close()")
}

// ---------------------------------------------

// SKIPPED because segfault isn't caught by recover()
func SKIPPED_TestAccessAfterUnmap(t *testing.T) {
	tmpDir, _ := ioutil.TempDir("", "mossMMap")
	defer os.RemoveAll(tmpDir)

	f, err := os.Create(tmpDir + string(os.PathSeparator) + "test.file")
	if err != nil {
		t.Errorf("expected open file to work, err: %v", err)
	}

	defer f.Close()

	offset := 1024 * 1024 * 1024 // 1 GB.

	f.WriteAt([]byte("hello"), int64(offset))

	var mm mmap.MMap

	mm, err = mmap.Map(f, mmap.RDONLY, 0)
	if err != nil {
		t.Errorf("expected mmap to work, err: %v", err)
	}

	x := mm[offset : offset+5]

	if string(x) != "hello" {
		t.Errorf("expected hello")
	}

	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered in f", r)
		} else {
			t.Errorf("expected recover from panic")
		}
	}()

	mm.Unmap()

	/*
			The following access of x results in a segfault, like...

				unexpected fault address 0x4060c000
				fatal error: fault
				[signal 0xb code=0x1 addr=0x4060c000 pc=0xb193f]

		    The recover() machinery doesn't handle this situation, however,
		    as it's not a normal kind of panic()
	*/
	if x[0] != 'h' {
		t.Errorf("expected h, but actually expected a segfault")
	}

	t.Errorf("expected segfault, but instead unmmapped mem access worked")
}
<|endoftext|>