<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var heartFreq = flag.Duration("heartbeat", 10*time.Second,
	"Heartbeat frequency")
var reconcileFreq = flag.Duration("reconcile", 24*time.Hour,
	"Reconciliation frequency")

type AboutNode struct {
	Addr     string    `json:"addr"`
	Type     string    `json:"type"`
	Time     time.Time `json:"time"`
	BindAddr string    `json:"bindaddr"`
	Hash     string    `json:"hash"`
}

type PeriodicJob struct {
	period time.Duration
	f      func() error
}

var periodicJobs = map[string]PeriodicJob{
	"checkStaleNodes": PeriodicJob{
		time.Minute * 5,
		checkStaleNodes,
	},
}

func getNodeAddress(sid string) (string, error) {
	sidkey := "/" + sid
	aboutSid := AboutNode{}
	err := couchbase.Get(sidkey, &aboutSid)
	if err != nil {
		return "", err
	}
	if strings.HasPrefix(aboutSid.BindAddr, ":") {
		return aboutSid.Addr + aboutSid.BindAddr, nil
	}
	return aboutSid.BindAddr, nil
}

type JobMarker struct {
	Node    string    `json:"node"`
	Started time.Time `json:"started"`
	Type    string    `json:"type"`
}

// Run a named task if we know one hasn't in the last t seconds.
func runNamedGlobalTask(name string, t time.Duration, f func() error) bool {
	key := "/@" + name

	jm := JobMarker{
		Node:    serverId,
		Started: time.Now(),
		Type:    "job",
	}

	err := couchbase.Do(key, func(mc *memcached.Client, vb uint16) error {
		data, err := json.Marshal(&jm)
		if err != nil {
			log.Fatalf("Can't jsonify a JobMarker: %v", err)
		}
		resp, err := mc.Add(vb, key, 0, int(t.Seconds()), data)
		if err != nil {
			return err
		}
		if resp.Status != gomemcached.SUCCESS {
			return fmt.Errorf("Wanted success, got %v", resp.Status)
		}
		return nil
	})

	if err == nil {
		err = f()
		if err != nil {
			log.Printf("Error running periodic task %#v: %v", name, err)
		}
	}

	return true
}

func heartbeat() {
	for {
		u, err := url.Parse(*couchbaseServer)
		c, err := net.Dial("tcp", u.Host)
		localAddr := ""
		if err == nil {
			localAddr = strings.Split(c.LocalAddr().String(), ":")[0]
			c.Close()
		}

		aboutMe := AboutNode{
			Addr:     localAddr,
			Type:     "storage",
			Time:     time.Now().UTC(),
			BindAddr: *bindAddr,
			Hash:     *hashType,
		}

		err = couchbase.Set("/"+serverId, aboutMe)
		if err != nil {
			log.Printf("Failed to record a heartbeat: %v", err)
		}
		time.Sleep(*heartFreq)
	}
}

func reconcile() error {
	explen := getHash().Size() * 2
	return filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			// I can do way more efficient stuff than this.
			recordBlobOwnership(info.Name(), info.Size())
			return err
		}
		return nil
	})
}

func reconcileLoop() {
	if *reconcileFreq == 0 {
		return
	}
	for {
		err := reconcile()
		if err != nil {
			log.Printf("Error in reconciliation loop: %v", err)
		}
		time.Sleep(*reconcileFreq)
	}
}

func removeBlobOwnershipRecord(h, node string) {
	log.Printf("Cleaning up %v from %v", h, node)

	k := "/" + h
	err := couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				delete(ownership.Nodes, node)
			} else {
				return nil
			}

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
	if err != nil {
		log.Printf("Error cleaning %v from %v", node, h)
	}
}

func cleanupNode(node string) {
	log.Printf("Cleaning up node %v", node)
	vres, err := couchbase.View("cbfs", "node_blobs",
		map[string]interface{}{
			"key":    `"` + node + `"`,
			"limit":  1000,
			"reduce": false,
			"stale":  false,
		})
	if err != nil {
		log.Printf("Error executing node_blobs view: %v", err)
		return
	}
	foundRows := 0
	for _, r := range vres.Rows {
		removeBlobOwnershipRecord(r.ID[1:], node)
		foundRows++
	}
	if foundRows == 0 {
		log.Printf("Removing node record: %v", node)
		err = couchbase.Delete("/" + node)
		if err != nil {
			log.Printf("Error deleting %v node record: %v", node, err)
		}
	}
}

func checkStaleNodes() error {
	log.Printf("Checking stale nodes")
	vres, err := couchbase.View("cbfs", "nodes", map[string]interface{}{
		"stale": false})
	if err != nil {
		return err
	}
	for _, r := range vres.Rows {
		ks, ok := r.Key.(string)
		if !ok {
			log.Printf("Wrong key type returned from view: %#v", r)
			continue
		}
		t, err := time.Parse(time.RFC3339Nano, ks)
		if err != nil {
			log.Printf("Error parsing time from %v", r)
			continue
		}
		d := time.Since(t)

		if d > *heartFreq*10 {
			node := r.ID[1:]
			if node == serverId {
				log.Printf("Would've cleaned up myself after %v",
					d)
				continue
			}
			log.Printf("  Node %v missed heartbeat schedule: %v", node, d)
			go cleanupNode(node)
		}
	}
	return nil
}

func runPeriodicJob(name string, job PeriodicJob) {
	for {
		if runNamedGlobalTask(name, job.period, job.f) {
			log.Printf("Attempted job %v", name)
		} else {
			log.Printf("Didn't run job %v", name)
		}
		time.Sleep(job.period + time.Second)
	}
}

func runPeriodicJobs() {
	for n, j := range periodicJobs {
		go runPeriodicJob(n, j)
	}
}
<commit_msg>Don't delete nodes when there were view errors on blobs.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var heartFreq = flag.Duration("heartbeat", 10*time.Second,
	"Heartbeat frequency")
var reconcileFreq = flag.Duration("reconcile", 24*time.Hour,
	"Reconciliation frequency")

type AboutNode struct {
	Addr     string    `json:"addr"`
	Type     string    `json:"type"`
	Time     time.Time `json:"time"`
	BindAddr string    `json:"bindaddr"`
	Hash     string    `json:"hash"`
}

type PeriodicJob struct {
	period time.Duration
	f      func() error
}

var periodicJobs = map[string]PeriodicJob{
	"checkStaleNodes": PeriodicJob{
		time.Minute * 5,
		checkStaleNodes,
	},
}

func getNodeAddress(sid string) (string, error) {
	sidkey := "/" + sid
	aboutSid := AboutNode{}
	err := couchbase.Get(sidkey, &aboutSid)
	if err != nil {
		return "", err
	}
	if strings.HasPrefix(aboutSid.BindAddr, ":") {
		return aboutSid.Addr + aboutSid.BindAddr, nil
	}
	return aboutSid.BindAddr, nil
}

type JobMarker struct {
	Node    string    `json:"node"`
	Started time.Time `json:"started"`
	Type    string    `json:"type"`
}

// Run a named task if we know one hasn't in the last t seconds.
func runNamedGlobalTask(name string, t time.Duration, f func() error) bool {
	key := "/@" + name

	jm := JobMarker{
		Node:    serverId,
		Started: time.Now(),
		Type:    "job",
	}

	err := couchbase.Do(key, func(mc *memcached.Client, vb uint16) error {
		data, err := json.Marshal(&jm)
		if err != nil {
			log.Fatalf("Can't jsonify a JobMarker: %v", err)
		}
		resp, err := mc.Add(vb, key, 0, int(t.Seconds()), data)
		if err != nil {
			return err
		}
		if resp.Status != gomemcached.SUCCESS {
			return fmt.Errorf("Wanted success, got %v", resp.Status)
		}
		return nil
	})

	if err == nil {
		err = f()
		if err != nil {
			log.Printf("Error running periodic task %#v: %v", name, err)
		}
	}

	return true
}

func heartbeat() {
	for {
		u, err := url.Parse(*couchbaseServer)
		c, err := net.Dial("tcp", u.Host)
		localAddr := ""
		if err == nil {
			localAddr = strings.Split(c.LocalAddr().String(), ":")[0]
			c.Close()
		}

		aboutMe := AboutNode{
			Addr:     localAddr,
			Type:     "storage",
			Time:     time.Now().UTC(),
			BindAddr: *bindAddr,
			Hash:     *hashType,
		}

		err = couchbase.Set("/"+serverId, aboutMe)
		if err != nil {
			log.Printf("Failed to record a heartbeat: %v", err)
		}
		time.Sleep(*heartFreq)
	}
}

func reconcile() error {
	explen := getHash().Size() * 2
	return filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			// I can do way more efficient stuff than this.
			recordBlobOwnership(info.Name(), info.Size())
			return err
		}
		return nil
	})
}

func reconcileLoop() {
	if *reconcileFreq == 0 {
		return
	}
	for {
		err := reconcile()
		if err != nil {
			log.Printf("Error in reconciliation loop: %v", err)
		}
		time.Sleep(*reconcileFreq)
	}
}

func removeBlobOwnershipRecord(h, node string) {
	log.Printf("Cleaning up %v from %v", h, node)

	k := "/" + h
	err := couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				delete(ownership.Nodes, node)
			} else {
				return nil
			}

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
	if err != nil {
		log.Printf("Error cleaning %v from %v", node, h)
	}
}

func cleanupNode(node string) {
	log.Printf("Cleaning up node %v", node)
	vres, err := couchbase.View("cbfs", "node_blobs",
		map[string]interface{}{
			"key":    `"` + node + `"`,
			"limit":  1000,
			"reduce": false,
			"stale":  false,
		})
	if err != nil {
		log.Printf("Error executing node_blobs view: %v", err)
		return
	}
	foundRows := 0
	for _, r := range vres.Rows {
		removeBlobOwnershipRecord(r.ID[1:], node)
		foundRows++
	}
	if foundRows == 0 && len(vres.Errors) == 0 {
		log.Printf("Removing node record: %v", node)
		err = couchbase.Delete("/" + node)
		if err != nil {
			log.Printf("Error deleting %v node record: %v", node, err)
		}
	}
}

func checkStaleNodes() error {
	log.Printf("Checking stale nodes")
	vres, err := couchbase.View("cbfs", "nodes", map[string]interface{}{
		"stale": false})
	if err != nil {
		return err
	}
	for _, r := range vres.Rows {
		ks, ok := r.Key.(string)
		if !ok {
			log.Printf("Wrong key type returned from view: %#v", r)
			continue
		}
		t, err := time.Parse(time.RFC3339Nano, ks)
		if err != nil {
			log.Printf("Error parsing time from %v", r)
			continue
		}
		d := time.Since(t)

		if d > *heartFreq*10 {
			node := r.ID[1:]
			if node == serverId {
				log.Printf("Would've cleaned up myself after %v",
					d)
				continue
			}
			log.Printf("  Node %v missed heartbeat schedule: %v", node, d)
			go cleanupNode(node)
		}
	}
	return nil
}

func runPeriodicJob(name string, job PeriodicJob) {
	for {
		if runNamedGlobalTask(name, job.period, job.f) {
			log.Printf("Attempted job %v", name)
		} else {
			log.Printf("Didn't run job %v", name)
		}
		time.Sleep(job.period + time.Second)
	}
}

func runPeriodicJobs() {
	for n, j := range periodicJobs {
		go runPeriodicJob(n, j)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Stefan Nyman.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package hellosign

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"

	"io/ioutil"

	"github.com/ajg/form"
)

const (
	baseURL                  string = "https://api.hellosign.com/v3"
	contentType                     = "content-type"
	xRatelimitLimit                 = "x-Ratelimit-Limit"
	xRatelimitLimitRemaining        = "x-Ratelimit-Limit-Remaining"
	xRateLimitReset                 = "x-Ratelimit-Reset"
)

type ListInfo struct {
	Page       uint64 `json:"page"`
	NumPages   uint64 `json:"num_pages"`
	NumResults uint64 `json:"num_results"`
	PageSize   uint64 `json:"page_size"`
}

type ListParms struct {
	AccountId *string `form:"account_id,omitempty"`
	Page      *uint64 `form:"page,omitempty"`
	PageSize  *uint64 `form:"page_size,omitempty"`
	Query     *string `form:"query,omitempty"`
}

type FormField struct {
	ApiID    string `json:"api_id"`
	Name     string `json:"name"`
	Type     string `json:"type"`
	X        uint64 `json:"x"`
	Y        uint64 `json:"y"`
	Width    uint64 `json:"width"`
	Height   uint64 `json:"height"`
	Required bool   `json:"required"`
}

type APIErr struct {
	Code    int // HTTP response code
	Message string
	Name    string
}

type APIWarn struct {
	Code     int // HTTP response code
	Warnings []struct {
		Message string
		Name    string
	}
}

func (a APIErr) Error() string {
	return fmt.Sprintf("%s: %s", a.Name, a.Message)
}

func (a APIWarn) Error() string {
	outMsg := ""
	for _, w := range a.Warnings {
		outMsg += fmt.Sprintf("%s: %s\n", w.Name, w.Message)
	}
	return outMsg
}

type hellosign struct {
	apiKey             string
	baseURL            string
	httpClient         *http.Client
	RateLimit          uint64 // Number of requests allowed per hour
	RateLimitRemaining uint64 // Remaining number of requests this hour
	RateLimitReset     uint64 // When the limit will be reset. In seconds from epoch
	LastStatusCode     int
}

// Initializes a new Hellosign API client.
func newHellosign(apiKey string) *hellosign {
	return &hellosign{
		apiKey:     apiKey,
		baseURL:    baseURL,
		httpClient: &http.Client{},
	}
}

func (c *hellosign) perform(req *http.Request) (*http.Response, error) {
	req.Header.Add("accept", "application/json")
	req.SetBasicAuth(c.apiKey, "")
	d, err := httputil.DumpRequest(req, true)
	if err == nil {
		fmt.Println(string(d))
	}
	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	c.LastStatusCode = resp.StatusCode
	if resp.StatusCode >= 400 {
		return nil, c.parseResponseError(resp)
	}
	for _, hk := range []string{xRatelimitLimit, xRatelimitLimitRemaining, xRateLimitReset} {
		hv := resp.Header.Get(hk)
		if hv == "" {
			continue
		}
		hvui, pErr := strconv.ParseUint(hv, 10, 64)
		if pErr != nil {
			continue
		}
		switch hk {
		case xRatelimitLimit:
			c.RateLimit = hvui
		case xRatelimitLimitRemaining:
			c.RateLimitRemaining = hvui
		case xRateLimitReset:
			c.RateLimitReset = hvui
		}
	}
	return resp, err
}

func (c *hellosign) parseResponseError(resp *http.Response) error {
	e := &struct {
		Err struct {
			Msg  *string `json:"error_msg"`
			Name *string `json:"error_name"`
		} `json:"error"`
	}{}
	w := &struct {
		Warnings []struct {
			Msg  *string `json:"warning_msg"`
			Name *string `json:"warning_name"`
		} `json:"warnings"`
	}{}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	err = json.Unmarshal(b, e)
	if err != nil {
		return err
	}
	if e.Err.Name != nil {
		return APIErr{Code: resp.StatusCode, Message: *e.Err.Msg, Name: *e.Err.Name}
	}
	err = json.Unmarshal(b, w)
	if err != nil {
		return err
	}
	if len(w.Warnings) == 0 {
		return errors.New("Could not parse response error or warning")
	}
	retErr := APIWarn{}
	warns := []struct {
		Name    string
		Message string
	}{}
	for _, w := range w.Warnings {
		warns = append(warns, struct {
			Name    string
			Message string
		}{
			Name:    *w.Name,
			Message: *w.Msg,
		})
	}
	return retErr
}

func (c *hellosign) parseResponse(resp *http.Response, dst interface{}) error {
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		d := json.NewDecoder(resp.Body)
		d.UseNumber()
		return d.Decode(dst)
	}
	return errors.New("Status code invalid")
}

func (c *hellosign) postForm(ept string, o interface{}) (*http.Response, error) {
	v := ""
	if o != nil {
		encoded, err := form.EncodeToString(o)
		if err != nil {
			return nil, err
		}
		v = encoded
	}
	req, err := http.NewRequest(http.MethodPost, c.getEptUrl(ept), strings.NewReader(v))
	if err != nil {
		return nil, err
	}
	req.Header.Add(contentType, "application/x-www-form-urlencoded")
	return c.perform(req)
}

func (c *hellosign) postFormAndParse(ept string, inp, dst interface{}) error {
	resp, err := c.postForm(ept, inp)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return c.parseResponse(resp, dst)
}

func (c *hellosign) delete(ept string) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodDelete, c.getEptUrl(ept), nil)
	if err != nil {
		return nil, err
	}
	return c.perform(req)
}

func (c *hellosign) getEptUrl(ept string) string {
	return fmt.Sprintf("%s/%s", c.baseURL, ept)
}

func (c *hellosign) get(ept string, params *string) (*http.Response, error) {
	url := c.getEptUrl(ept)
	if params != nil && *params == "" {
		url = fmt.Sprintf("%s?%s", url, *params)
	}
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.perform(req)
	return resp, err
}

func (c *hellosign) getAndParse(ept string, params *string, dst interface{}) error {
	resp, err := c.get(ept, params)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return c.parseResponse(resp, dst)
}

func (c *hellosign) getFiles(ept, fileType string, getURL bool) ([]byte, *FileURL, error) {
	if fileType != "" && fileType != "pdf" && fileType != "zip" {
		return []byte{}, nil, errors.New("Invalid file type specified, pdf or zip")
	}
	parms, err := form.EncodeToString(&struct {
		FileType string `form:"file_type,omitempty"`
		GetUrl   bool   `form:"get_url,omitempty"`
	}{
		FileType: fileType,
		GetUrl:   getURL,
	})
	if err != nil {
		return []byte{}, nil, err
	}
	resp, err := c.get(ept, &parms)
	if err != nil {
		return []byte{}, nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return []byte{}, nil, errors.New(resp.Status)
	}
	if getURL {
		msg := &FileURL{}
		if err := c.parseResponse(resp, msg); err != nil {
			return []byte{}, nil, err
		}
		return []byte{}, msg, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, nil, err
	}
	return b, nil, nil
}
<commit_msg>hellosign: remove dump of all reqs, use default http client, remove unnecessary client props<commit_after>// Copyright 2016 Stefan Nyman.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package hellosign

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/ajg/form"
)

const (
	baseURL                  string = "https://api.hellosign.com/v3"
	contentType                     = "content-type"
	xRatelimitLimit                 = "x-Ratelimit-Limit"
	xRatelimitLimitRemaining        = "x-Ratelimit-Limit-Remaining"
	xRateLimitReset                 = "x-Ratelimit-Reset"
)

type ListInfo struct {
	Page       uint64 `json:"page"`
	NumPages   uint64 `json:"num_pages"`
	NumResults uint64 `json:"num_results"`
	PageSize   uint64 `json:"page_size"`
}

type ListParms struct {
	AccountId *string `form:"account_id,omitempty"`
	Page      *uint64 `form:"page,omitempty"`
	PageSize  *uint64 `form:"page_size,omitempty"`
	Query     *string `form:"query,omitempty"`
}

type FormField struct {
	ApiID    string `json:"api_id"`
	Name     string `json:"name"`
	Type     string `json:"type"`
	X        uint64 `json:"x"`
	Y        uint64 `json:"y"`
	Width    uint64 `json:"width"`
	Height   uint64 `json:"height"`
	Required bool   `json:"required"`
}

type APIErr struct {
	Code    int // HTTP response code
	Message string
	Name    string
}

type APIWarn struct {
	Code     int // HTTP response code
	Warnings []struct {
		Message string
		Name    string
	}
}

func (a APIErr) Error() string {
	return fmt.Sprintf("%s: %s", a.Name, a.Message)
}

func (a APIWarn) Error() string {
	outMsg := ""
	for _, w := range a.Warnings {
		outMsg += fmt.Sprintf("%s: %s\n", w.Name, w.Message)
	}
	return outMsg
}

type hellosign struct {
	apiKey             string
	RateLimit          uint64 // Number of requests allowed per hour
	RateLimitRemaining uint64 // Remaining number of requests this hour
	RateLimitReset     uint64 // When the limit will be reset. In seconds from epoch
	LastStatusCode     int
}

// Initializes a new Hellosign API client.
func newHellosign(apiKey string) *hellosign {
	return &hellosign{
		apiKey: apiKey,
	}
}

func (c *hellosign) perform(req *http.Request) (*http.Response, error) {
	req.Header.Add("accept", "application/json")
	req.SetBasicAuth(c.apiKey, "")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	c.LastStatusCode = resp.StatusCode
	if resp.StatusCode >= 400 {
		return nil, c.parseResponseError(resp)
	}
	for _, hk := range []string{xRatelimitLimit, xRatelimitLimitRemaining, xRateLimitReset} {
		hv := resp.Header.Get(hk)
		if hv == "" {
			continue
		}
		hvui, pErr := strconv.ParseUint(hv, 10, 64)
		if pErr != nil {
			continue
		}
		switch hk {
		case xRatelimitLimit:
			c.RateLimit = hvui
		case xRatelimitLimitRemaining:
			c.RateLimitRemaining = hvui
		case xRateLimitReset:
			c.RateLimitReset = hvui
		}
	}
	return resp, err
}

func (c *hellosign) parseResponseError(resp *http.Response) error {
	e := &struct {
		Err struct {
			Msg  *string `json:"error_msg"`
			Name *string `json:"error_name"`
		} `json:"error"`
	}{}
	w := &struct {
		Warnings []struct {
			Msg  *string `json:"warning_msg"`
			Name *string `json:"warning_name"`
		} `json:"warnings"`
	}{}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	err = json.Unmarshal(b, e)
	if err != nil {
		return err
	}
	if e.Err.Name != nil {
		return APIErr{Code: resp.StatusCode, Message: *e.Err.Msg, Name: *e.Err.Name}
	}
	err = json.Unmarshal(b, w)
	if err != nil {
		return err
	}
	if len(w.Warnings) == 0 {
		return errors.New("Could not parse response error or warning")
	}
	retErr := APIWarn{}
	warns := []struct {
		Name    string
		Message string
	}{}
	for _, w := range w.Warnings {
		warns = append(warns, struct {
			Name    string
			Message string
		}{
			Name:    *w.Name,
			Message: *w.Msg,
		})
	}
	return retErr
}

func (c *hellosign) parseResponse(resp *http.Response, dst interface{}) error {
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		d := json.NewDecoder(resp.Body)
		d.UseNumber()
		return d.Decode(dst)
	}
	return errors.New("Status code invalid")
}

func (c *hellosign) postForm(ept string, o interface{}) (*http.Response, error) {
	v := ""
	if o != nil {
		encoded, err := form.EncodeToString(o)
		if err != nil {
			return nil, err
		}
		v = encoded
	}
	req, err := http.NewRequest(http.MethodPost, c.getEptUrl(ept), strings.NewReader(v))
	if err != nil {
		return nil, err
	}
	req.Header.Add(contentType, "application/x-www-form-urlencoded")
	return c.perform(req)
}

func (c *hellosign) postFormAndParse(ept string, inp, dst interface{}) error {
	resp, err := c.postForm(ept, inp)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return c.parseResponse(resp, dst)
}

func (c *hellosign) delete(ept string) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodDelete, c.getEptUrl(ept), nil)
	if err != nil {
		return nil, err
	}
	return c.perform(req)
}

func GetEptURL(ept string) string {
	return fmt.Sprintf("%s/%s", baseURL, ept)
}

func (c *hellosign) getEptUrl(ept string) string {
	return GetEptURL(ept)
}

func (c *hellosign) get(ept string, params *string) (*http.Response, error) {
	url := c.getEptUrl(ept)
	if params != nil && *params == "" {
		url = fmt.Sprintf("%s?%s", url, *params)
	}
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.perform(req)
	return resp, err
}

func (c *hellosign) getAndParse(ept string, params *string, dst interface{}) error {
	resp, err := c.get(ept, params)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return c.parseResponse(resp, dst)
}

func (c *hellosign) getFiles(ept, fileType string, getURL bool) ([]byte, *FileURL, error) {
	if fileType != "" && fileType != "pdf" && fileType != "zip" {
		return []byte{}, nil, errors.New("Invalid file type specified, pdf or zip")
	}
	parms, err := form.EncodeToString(&struct {
		FileType string `form:"file_type,omitempty"`
		GetUrl   bool   `form:"get_url,omitempty"`
	}{
		FileType: fileType,
		GetUrl:   getURL,
	})
	if err != nil {
		return []byte{}, nil, err
	}
	resp, err := c.get(ept, &parms)
	if err != nil {
		return []byte{}, nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return []byte{}, nil, errors.New(resp.Status)
	}
	if getURL {
		msg := &FileURL{}
		if err := c.parseResponse(resp, msg); err != nil {
			return []byte{}, nil, err
		}
		return []byte{}, msg, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, nil, err
	}
	return b, nil, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Stefan Nyman.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package hellosign

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"

	"io/ioutil"

	"github.com/ajg/form"
)

const (
	baseURL                  string = "https://api.hellosign.com/v3"
	contentType                     = "content-type"
	xRatelimitLimit                 = "x-Ratelimit-Limit"
	xRatelimitLimitRemaining        = "x-Ratelimit-Limit-Remaining"
	xRateLimitReset                 = "x-Ratelimit-Reset"
)

type hellosign struct {
	apiKey             string
	baseURL            string
	httpClient         *http.Client
	RateLimit          uint64 // Number of requests allowed per hour
	RateLimitRemaining uint64 // Remaining number of requests this hour
	RateLimitReset     uint64 // When the limit will be reset. In seconds from epoch
	LastStatusCode     int
}

// Initializes a new Hellosign API client.
func newHellosign(apiKey string) *hellosign {
	return &hellosign{
		apiKey:     apiKey,
		baseURL:    baseURL,
		httpClient: &http.Client{},
	}
}

func (c *hellosign) perform(req *http.Request) (*http.Response, error) {
	req.Header.Add("accept", "application/json")
	req.SetBasicAuth(c.apiKey, "")
	d, err := httputil.DumpRequest(req, true)
	if err == nil {
		fmt.Println(string(d))
	}
	resp, err := c.httpClient.Do(req)
	for _, hk := range []string{xRatelimitLimit, xRatelimitLimitRemaining, xRateLimitReset} {
		hv := resp.Header.Get(hk)
		if hv == "" {
			continue
		}
		hvui, pErr := strconv.ParseUint(hv, 10, 64)
		if pErr != nil {
			continue
		}
		switch hk {
		case xRatelimitLimit:
			c.RateLimit = hvui
		case xRatelimitLimitRemaining:
			c.RateLimitRemaining = hvui
		case xRateLimitReset:
			c.RateLimitReset = hvui
		}
	}
	c.LastStatusCode = resp.StatusCode
	return resp, err
}

func (c *hellosign) parseResponse(resp *http.Response, dst interface{}) error {
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		d := json.NewDecoder(resp.Body)
		d.UseNumber()
		return d.Decode(dst)
	}
	return errors.New("Status code invalid")
}

func (c *hellosign) postForm(ept string, o interface{}) (*http.Response, error) {
	v := ""
	if o != nil {
		encoded, err := form.EncodeToString(o)
		if err != nil {
			return nil, err
		}
		v = encoded
	}
	req, err := http.NewRequest(http.MethodPost, c.getEptUrl(ept), strings.NewReader(v))
	if err != nil {
		return nil, err
	}
	req.Header.Add(contentType, "application/x-www-form-urlencoded")
	return c.perform(req)
}

func (c *hellosign) getEptUrl(ept string) string {
	return fmt.Sprintf("%s/%s", c.baseURL, ept)
}

func (c *hellosign) get(ept string, params *string) (*http.Response, error) {
	url := c.getEptUrl(ept)
	if params != nil && *params == "" {
		url = fmt.Sprintf("%s?%s", url, *params)
	}
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.perform(req)
	return resp, err
}

func (c *hellosign) getFiles(ept, fileType string, getURL bool) (*[]byte, *FileURL, error) {
	if fileType != "" && fileType != "pdf" && fileType != "zip" {
		return nil, nil, errors.New("Invalid file type specified, pdf or zip")
	}
	parms, err := form.EncodeToString(&struct {
		FileType string `form:"file_type,omitempty"`
		GetUrl   bool   `form:"get_url,omitempty"`
	}{
		FileType: fileType,
		GetUrl:   getURL,
	})
	if err != nil {
		return nil, nil, err
	}
	resp, err := c.get(ept, &parms)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, nil, errors.New(resp.Status)
	}
	if getURL {
		msg := &FileURL{}
		if err := c.parseResponse(resp, msg); err != nil {
			return nil, nil, err
		}
		return nil, msg, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, err
	}
	return &b, nil, nil
}
<commit_msg>hellosign: add convenience meths for get/post and parse response<commit_after>// Copyright 2016 Stefan Nyman.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package hellosign

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"

	"io/ioutil"

	"github.com/ajg/form"
)

const (
	baseURL                  string = "https://api.hellosign.com/v3"
	contentType                     = "content-type"
	xRatelimitLimit                 = "x-Ratelimit-Limit"
	xRatelimitLimitRemaining        = "x-Ratelimit-Limit-Remaining"
	xRateLimitReset                 = "x-Ratelimit-Reset"
)

type hellosign struct {
	apiKey             string
	baseURL            string
	httpClient         *http.Client
	RateLimit          uint64 // Number of requests allowed per hour
	RateLimitRemaining uint64 // Remaining number of requests this hour
	RateLimitReset     uint64 // When the limit will be reset. In seconds from epoch
	LastStatusCode     int
}

// Initializes a new Hellosign API client.
func newHellosign(apiKey string) *hellosign {
	return &hellosign{
		apiKey:     apiKey,
		baseURL:    baseURL,
		httpClient: &http.Client{},
	}
}

func (c *hellosign) perform(req *http.Request) (*http.Response, error) {
	req.Header.Add("accept", "application/json")
	req.SetBasicAuth(c.apiKey, "")
	d, err := httputil.DumpRequest(req, true)
	if err == nil {
		fmt.Println(string(d))
	}
	resp, err := c.httpClient.Do(req)
	for _, hk := range []string{xRatelimitLimit, xRatelimitLimitRemaining, xRateLimitReset} {
		hv := resp.Header.Get(hk)
		if hv == "" {
			continue
		}
		hvui, pErr := strconv.ParseUint(hv, 10, 64)
		if pErr != nil {
			continue
		}
		switch hk {
		case xRatelimitLimit:
			c.RateLimit = hvui
		case xRatelimitLimitRemaining:
			c.RateLimitRemaining = hvui
		case xRateLimitReset:
			c.RateLimitReset = hvui
		}
	}
	c.LastStatusCode = resp.StatusCode
	return resp, err
}

func (c *hellosign) parseResponse(resp *http.Response, dst interface{}) error {
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		d := json.NewDecoder(resp.Body)
		d.UseNumber()
		return d.Decode(dst)
	}
	return errors.New("Status code invalid")
}

func (c *hellosign) postForm(ept string, o interface{}) (*http.Response, error) {
	v := ""
	if o != nil {
		encoded, err := form.EncodeToString(o)
		if err != nil {
			return nil, err
		}
		v = encoded
	}
	req, err := http.NewRequest(http.MethodPost, c.getEptUrl(ept), strings.NewReader(v))
	if err != nil {
		return nil, err
	}
	req.Header.Add(contentType, "application/x-www-form-urlencoded")
	return c.perform(req)
}

func (c *hellosign) postFormAndParse(ept string, inp, dst interface{}) error {
	resp, err := c.postForm(ept, inp)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return c.parseResponse(resp, dst)
}

func (c *hellosign) getEptUrl(ept string) string {
	return fmt.Sprintf("%s/%s", c.baseURL, ept)
}

func (c *hellosign) get(ept string, params *string) (*http.Response, error) {
	url := c.getEptUrl(ept)
	if params != nil && *params == "" {
		url = fmt.Sprintf("%s?%s", url, *params)
	}
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.perform(req)
	return resp, err
}

func (c *hellosign) getAndParse(ept string, params *string, dst interface{}) error {
	resp, err := c.get(ept, params)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	return c.parseResponse(resp, dst)
}

func (c *hellosign) getFiles(ept, fileType string, getURL bool) (*[]byte, *FileURL, error) {
	if fileType != "" && fileType != "pdf" && fileType != "zip" {
		return nil, nil, errors.New("Invalid file type specified, pdf or zip")
	}
	parms, err := form.EncodeToString(&struct {
		FileType string `form:"file_type,omitempty"`
		GetUrl   bool   `form:"get_url,omitempty"`
	}{
		FileType: fileType,
		GetUrl:   getURL,
	})
	if err != nil {
		return nil, nil, err
	}
	resp, err := c.get(ept, &parms)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, nil, errors.New(resp.Status)
	}
	if getURL {
		msg := &FileURL{}
		if err := c.parseResponse(resp, msg); err != nil {
			return nil, nil, err
		}
		return nil, msg, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, err
	}
	return &b, nil, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package irc

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/goshuirc/irc-go/ircfmt"
	"github.com/goshuirc/irc-go/ircmatch"
	"github.com/goshuirc/irc-go/ircmsg"
	"github.com/oragono/oragono/irc/custime"
	"github.com/oragono/oragono/irc/sno"
	"github.com/tidwall/buntdb"
)

const (
	keyKlineEntry = "bans.kline %s"
)

// KLineInfo contains the address itself and expiration time for a given network.
type KLineInfo struct {
	// Mask that is blocked.
	Mask string
	// Matcher, to facilitate fast matching.
	Matcher ircmatch.Matcher
	// Info contains information on the ban.
	Info IPBanInfo
}

// KLineManager manages and klines.
type KLineManager struct {
	sync.RWMutex
	// kline'd entries
	entries map[string]*KLineInfo
}

// NewKLineManager returns a new KLineManager.
func NewKLineManager() *KLineManager {
	var km KLineManager
	km.entries = make(map[string]*KLineInfo)
	return &km
}

// AllBans returns all bans (for use with APIs, etc).
func (km *KLineManager) AllBans() map[string]IPBanInfo {
	allb := make(map[string]IPBanInfo)

	km.RLock()
	defer km.RUnlock()
	for name, info := range km.entries {
		allb[name] = info.Info
	}

	return allb
}

// AddMask adds to the blocked list.
func (km *KLineManager) AddMask(mask string, length *IPRestrictTime, reason string, operReason string) {
	kln := KLineInfo{
		Mask:    mask,
		Matcher: ircmatch.MakeMatch(mask),
		Info: IPBanInfo{
			Time:       length,
			Reason:     reason,
			OperReason: operReason,
		},
	}
	km.Lock()
	km.entries[mask] = &kln
	km.Unlock()
}

// RemoveMask removes a mask from the blocked list.
func (km *KLineManager) RemoveMask(mask string) {
	km.Lock()
	delete(km.entries, mask)
	km.Unlock()
}

// CheckMasks returns whether or not the hostmask(s) are banned, and how long they are banned for.
func (km *KLineManager) CheckMasks(masks ...string) (isBanned bool, info *IPBanInfo) {
	doCleanup := false
	defer func() {
		// asynchronously remove expired bans
		if doCleanup {
			go func() {
				km.Lock()
				defer km.Unlock()
				for key, entry := range km.entries {
					if entry.Info.Time.IsExpired() {
						delete(km.entries, key)
					}
				}
			}()
		}
	}()

	km.RLock()
	defer km.RUnlock()

	for _, entryInfo := range km.entries {
		if entryInfo.Info.Time != nil && entryInfo.Info.Time.IsExpired() {
			doCleanup = true
			continue
		}

		matches := false
		for _, mask := range masks {
			if entryInfo.Matcher.Match(mask) {
				matches = true
				break
			}
		}
		if matches {
			return true, &entryInfo.Info
		}
	}

	// no matches!
	return false, nil
}

// KLINE [ANDKILL] [MYSELF] [duration] <mask> [ON <server>] [reason [| oper reason]]
func klineHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	// check oper permissions
	if !client.class.Capabilities["oper:local_ban"] {
		client.Send(nil, server.name, ERR_NOPRIVS, client.nick, msg.Command, "Insufficient oper privs")
		return false
	}

	currentArg := 0

	// when setting a ban, if they say "ANDKILL" we should also kill all users who match it
	var andKill bool
	if len(msg.Params) > currentArg+1 && strings.ToLower(msg.Params[currentArg]) == "andkill" {
		andKill = true
		currentArg++
	}

	// when setting a ban that covers the oper's current connection, we require them to say
	// "KLINE MYSELF" so that we're sure they really mean it.
	var klineMyself bool
	if len(msg.Params) > currentArg+1 && strings.ToLower(msg.Params[currentArg]) == "myself" {
		klineMyself = true
		currentArg++
	}

	// duration
	duration, err := custime.ParseDuration(msg.Params[currentArg])
	durationIsUsed := err == nil
	if durationIsUsed {
		currentArg++
	}

	// get mask
	if len(msg.Params) < currentArg+1 {
		client.Send(nil, server.name, ERR_NEEDMOREPARAMS, client.nick, msg.Command, "Not enough parameters")
		return false
	}
	mask := strings.ToLower(msg.Params[currentArg])
	currentArg++

	// check mask
	if !strings.Contains(mask, "!") && !strings.Contains(mask, "@") {
		mask = mask + "!*@*"
	} else if !strings.Contains(mask, "@") {
		mask = mask + "@*"
	}

	matcher := ircmatch.MakeMatch(mask)

	for _, clientMask := range client.AllNickmasks() {
		if !klineMyself && matcher.Match(clientMask) {
			client.Send(nil, server.name, ERR_UNKNOWNERROR, client.nick, msg.Command, "This ban matches you. To KLINE yourself, you must use the command:  /KLINE MYSELF <arguments>")
			return false
		}
	}

	// check remote
	if len(msg.Params) > currentArg && msg.Params[currentArg] == "ON" {
		client.Send(nil, server.name, ERR_UNKNOWNERROR, client.nick, msg.Command, "Remote servers not yet supported")
		return false
	}

	// get comment(s)
	reason := "No reason given"
	operReason := "No reason given"
	if len(msg.Params) > currentArg {
		tempReason := strings.TrimSpace(msg.Params[currentArg])
		if len(tempReason) > 0 && tempReason != "|" {
			tempReasons := strings.SplitN(tempReason, "|", 2)
			if tempReasons[0] != "" {
				reason = tempReasons[0]
			}
			if len(tempReasons) > 1 && tempReasons[1] != "" {
				operReason = tempReasons[1]
			} else {
				operReason = reason
			}
		}
	}

	// assemble ban info
	var banTime *IPRestrictTime
	if durationIsUsed {
		banTime = &IPRestrictTime{
			Duration: duration,
			Expires:  time.Now().Add(duration),
		}
	}

	info := IPBanInfo{
		Reason:     reason,
		OperReason: operReason,
		Time:       banTime,
	}

	// save in datastore
	err = server.store.Update(func(tx *buntdb.Tx) error {
		klineKey := fmt.Sprintf(keyKlineEntry, mask)

		// assemble json from ban info
		b, err := json.Marshal(info)
		if err != nil {
			return err
		}

		tx.Set(klineKey, string(b), nil)

		return nil
	})

	if err != nil {
		client.Notice(fmt.Sprintf("Could not successfully save new K-LINE: %s", err.Error()))
		return false
	}

	server.klines.AddMask(mask, banTime, reason, operReason)

	var snoDescription string
	if durationIsUsed {
		client.Notice(fmt.Sprintf("Added temporary (%s) K-Line for %s", duration.String(), mask))
		snoDescription = fmt.Sprintf(ircfmt.Unescape("%s$r added temporary (%s) K-Line for %s"), client.nick, duration.String(), mask)
	} else {
		client.Notice(fmt.Sprintf("Added K-Line for %s", mask))
		snoDescription = fmt.Sprintf(ircfmt.Unescape("%s$r added K-Line for %s"), client.nick, mask)
	}
	server.snomasks.Send(sno.LocalXline, snoDescription)

	var killClient bool
	if andKill {
		var clientsToKill []*Client
		var killedClientNicks []string

		server.clients.ByNickMutex.RLock()
		for _, mcl := range server.clients.ByNick {
			for _, clientMask := range mcl.AllNickmasks() {
				if matcher.Match(clientMask) {
					clientsToKill = append(clientsToKill, mcl)
					killedClientNicks = append(killedClientNicks, mcl.nick)
				}
			}
		}
		server.clients.ByNickMutex.RUnlock()

		for _, mcl := range clientsToKill {
			mcl.exitedSnomaskSent = true
			mcl.Quit(fmt.Sprintf("You have been banned from this server (%s)", reason))
			if mcl == client {
				killClient = true
			} else {
				// if mcl == client, we kill them below
				mcl.destroy()
			}
		}

		// send snomask
		sort.Strings(killedClientNicks)
		server.snomasks.Send(sno.LocalKills, fmt.Sprintf(ircfmt.Unescape("%s killed %d clients with a KLINE $c[grey][$r%s$c[grey]]"), client.nick, len(killedClientNicks), strings.Join(killedClientNicks, ", ")))
	}

	return killClient
}

func unKLineHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	// check oper permissions
	if !client.class.Capabilities["oper:local_unban"] {
		client.Send(nil, server.name, ERR_NOPRIVS, client.nick, msg.Command, "Insufficient oper privs")
		return false
	}

	// get host
	mask := msg.Params[0]

	if !strings.Contains(mask, "!") && !strings.Contains(mask, "@") {
		mask = mask + "!*@*"
	} else if !strings.Contains(mask, "@") {
		mask = mask + "@*"
	}

	// save in datastore
	err := server.store.Update(func(tx *buntdb.Tx) error {
		klineKey := fmt.Sprintf(keyKlineEntry, mask)

		// check if it exists or not
		val, err := tx.Get(klineKey)
		if val == "" {
			return errNoExistingBan
		} else if err != nil {
			return err
		}

		tx.Delete(klineKey)
		return nil
	})

	if err != nil {
		client.Send(nil, server.name, ERR_UNKNOWNERROR, client.nick, msg.Command, fmt.Sprintf("Could not remove ban [%s]", err.Error()))
		return false
	}

	server.klines.RemoveMask(mask)

	client.Notice(fmt.Sprintf("Removed K-Line for %s", mask))
	server.snomasks.Send(sno.LocalXline, fmt.Sprintf(ircfmt.Unescape("%s$r removed K-Line for %s"), client.nick, mask))
	return false
}

func (s *Server) loadKLines() {
	s.klines = NewKLineManager()

	// load from datastore
	s.store.View(func(tx *buntdb.Tx) error {
		//TODO(dan): We could make this safer
		tx.AscendKeys("bans.kline *", func(key, value string) bool {
			// get address name
			key = key[len("bans.kline "):]
			mask := key

			// load ban info
			var info IPBanInfo
			json.Unmarshal([]byte(value), &info)

			// add to the server
			s.klines.AddMask(mask, info.Time, info.Reason, info.OperReason)

			return true // true to continue I guess?
		})
		return nil
	})
}
<commit_msg>kline: Gofmt -s<commit_after>// Copyright (c) 2016-2017 Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package irc

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/goshuirc/irc-go/ircfmt"
	"github.com/goshuirc/irc-go/ircmatch"
	"github.com/goshuirc/irc-go/ircmsg"
	"github.com/oragono/oragono/irc/custime"
	"github.com/oragono/oragono/irc/sno"
	"github.com/tidwall/buntdb"
)

const (
	keyKlineEntry = "bans.kline %s"
)

// KLineInfo contains the address itself and expiration time for a given network.
type KLineInfo struct {
	// Mask that is blocked.
	Mask string
	// Matcher, to facilitate fast matching.
	Matcher ircmatch.Matcher
	// Info contains information on the ban.
	Info IPBanInfo
}

// KLineManager manages and klines.
type KLineManager struct {
	sync.RWMutex
	// kline'd entries
	entries map[string]*KLineInfo
}

// NewKLineManager returns a new KLineManager.
func NewKLineManager() *KLineManager {
	var km KLineManager
	km.entries = make(map[string]*KLineInfo)
	return &km
}

// AllBans returns all bans (for use with APIs, etc).
func (km *KLineManager) AllBans() map[string]IPBanInfo {
	allb := make(map[string]IPBanInfo)

	km.RLock()
	defer km.RUnlock()
	for name, info := range km.entries {
		allb[name] = info.Info
	}

	return allb
}

// AddMask adds to the blocked list.
func (km *KLineManager) AddMask(mask string, length *IPRestrictTime, reason string, operReason string) {
	kln := KLineInfo{
		Mask:    mask,
		Matcher: ircmatch.MakeMatch(mask),
		Info: IPBanInfo{
			Time:       length,
			Reason:     reason,
			OperReason: operReason,
		},
	}
	km.Lock()
	km.entries[mask] = &kln
	km.Unlock()
}

// RemoveMask removes a mask from the blocked list.
func (km *KLineManager) RemoveMask(mask string) {
	km.Lock()
	delete(km.entries, mask)
	km.Unlock()
}

// CheckMasks returns whether or not the hostmask(s) are banned, and how long they are banned for.
func (km *KLineManager) CheckMasks(masks ...string) (isBanned bool, info *IPBanInfo) {
	doCleanup := false
	defer func() {
		// asynchronously remove expired bans
		if doCleanup {
			go func() {
				km.Lock()
				defer km.Unlock()
				for key, entry := range km.entries {
					if entry.Info.Time.IsExpired() {
						delete(km.entries, key)
					}
				}
			}()
		}
	}()

	km.RLock()
	defer km.RUnlock()

	for _, entryInfo := range km.entries {
		if entryInfo.Info.Time != nil && entryInfo.Info.Time.IsExpired() {
			doCleanup = true
			continue
		}

		matches := false
		for _, mask := range masks {
			if entryInfo.Matcher.Match(mask) {
				matches = true
				break
			}
		}
		if matches {
			return true, &entryInfo.Info
		}
	}

	// no matches!
	return false, nil
}

// KLINE [ANDKILL] [MYSELF] [duration] <mask> [ON <server>] [reason [| oper reason]]
func klineHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	// check oper permissions
	if !client.class.Capabilities["oper:local_ban"] {
		client.Send(nil, server.name, ERR_NOPRIVS, client.nick, msg.Command, "Insufficient oper privs")
		return false
	}

	currentArg := 0

	// when setting a ban, if they say "ANDKILL" we should also kill all users who match it
	var andKill bool
	if len(msg.Params) > currentArg+1 && strings.ToLower(msg.Params[currentArg]) == "andkill" {
		andKill = true
		currentArg++
	}

	// when setting a ban that covers the oper's current connection, we require them to say
	// "KLINE MYSELF" so that we're sure they really mean it.
	var klineMyself bool
	if len(msg.Params) > currentArg+1 && strings.ToLower(msg.Params[currentArg]) == "myself" {
		klineMyself = true
		currentArg++
	}

	// duration
	duration, err := custime.ParseDuration(msg.Params[currentArg])
	durationIsUsed := err == nil
	if durationIsUsed {
		currentArg++
	}

	// get mask
	if len(msg.Params) < currentArg+1 {
		client.Send(nil, server.name, ERR_NEEDMOREPARAMS, client.nick, msg.Command, "Not enough parameters")
		return false
	}
	mask := strings.ToLower(msg.Params[currentArg])
	currentArg++

	// check mask
	if !strings.Contains(mask, "!") && !strings.Contains(mask, "@") {
		mask = mask + "!*@*"
	} else if !strings.Contains(mask, "@") {
		mask = mask + "@*"
	}

	matcher := ircmatch.MakeMatch(mask)

	for _, clientMask := range client.AllNickmasks() {
		if !klineMyself && matcher.Match(clientMask) {
			client.Send(nil, server.name, ERR_UNKNOWNERROR, client.nick, msg.Command, "This ban matches you. To KLINE yourself, you must use the command:  /KLINE MYSELF <arguments>")
			return false
		}
	}

	// check remote
	if len(msg.Params) > currentArg && msg.Params[currentArg] == "ON" {
		client.Send(nil, server.name, ERR_UNKNOWNERROR, client.nick, msg.Command, "Remote servers not yet supported")
		return false
	}

	// get comment(s)
	reason := "No reason given"
	operReason := "No reason given"
	if len(msg.Params) > currentArg {
		tempReason := strings.TrimSpace(msg.Params[currentArg])
		if len(tempReason) > 0 && tempReason != "|" {
			tempReasons := strings.SplitN(tempReason, "|", 2)
			if tempReasons[0] != "" {
				reason = tempReasons[0]
			}
			if len(tempReasons) > 1 && tempReasons[1] != "" {
				operReason = tempReasons[1]
			} else {
				operReason = reason
			}
		}
	}

	// assemble ban info
	var banTime *IPRestrictTime
	if durationIsUsed {
		banTime = &IPRestrictTime{
			Duration: duration,
			Expires:  time.Now().Add(duration),
		}
	}

	info := IPBanInfo{
		Reason:     reason,
		OperReason: operReason,
		Time:       banTime,
	}

	// save in datastore
	err = server.store.Update(func(tx *buntdb.Tx) error {
		klineKey := fmt.Sprintf(keyKlineEntry, mask)

		// assemble json from ban info
		b, err := json.Marshal(info)
		if err != nil {
			return err
		}

		tx.Set(klineKey, string(b), nil)

		return nil
	})

	if err != nil {
		client.Notice(fmt.Sprintf("Could not successfully save new K-LINE: %s", err.Error()))
		return false
	}

	server.klines.AddMask(mask, banTime, reason, operReason)

	var snoDescription string
	if durationIsUsed {
		client.Notice(fmt.Sprintf("Added temporary (%s) K-Line for %s", duration.String(), mask))
		snoDescription = fmt.Sprintf(ircfmt.Unescape("%s$r added temporary (%s) K-Line for %s"), client.nick, duration.String(), mask)
	} else {
		client.Notice(fmt.Sprintf("Added K-Line for %s", mask))
		snoDescription = fmt.Sprintf(ircfmt.Unescape("%s$r added K-Line for %s"), client.nick, mask)
	}
	server.snomasks.Send(sno.LocalXline, snoDescription)

	var killClient bool
	if andKill {
		var clientsToKill []*Client
		var killedClientNicks []string

		server.clients.ByNickMutex.RLock()
		for _, mcl := range server.clients.ByNick {
			for _, clientMask := range mcl.AllNickmasks() {
				if matcher.Match(clientMask) {
					clientsToKill = append(clientsToKill, mcl)
					killedClientNicks = append(killedClientNicks, mcl.nick)
				}
			}
		}
		server.clients.ByNickMutex.RUnlock()

		for _, mcl := range clientsToKill {
			mcl.exitedSnomaskSent = true
			mcl.Quit(fmt.Sprintf("You have been banned from this server (%s)", reason))
			if mcl == client {
				killClient = true
			} else {
				// if mcl == client, we kill them below
				mcl.destroy()
			}
		}

		// send snomask
		sort.Strings(killedClientNicks)
		server.snomasks.Send(sno.LocalKills, fmt.Sprintf(ircfmt.Unescape("%s killed %d clients with a KLINE $c[grey][$r%s$c[grey]]"), client.nick, len(killedClientNicks), strings.Join(killedClientNicks, ", ")))
	}

	return killClient
}

func unKLineHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	// check oper permissions
	if !client.class.Capabilities["oper:local_unban"] {
		client.Send(nil, server.name, ERR_NOPRIVS, client.nick, msg.Command, "Insufficient oper privs")
		return false
	}

	// get host
	mask := msg.Params[0]

	if !strings.Contains(mask, "!") && !strings.Contains(mask, "@") {
		mask = mask + "!*@*"
	} else if !strings.Contains(mask, "@") {
		mask = mask + "@*"
	}

	// save in datastore
	err := server.store.Update(func(tx *buntdb.Tx) error {
		klineKey := fmt.Sprintf(keyKlineEntry, mask)

		// check if it exists or not
		val, err := tx.Get(klineKey)
		if val == "" {
			return errNoExistingBan
		} else if err != nil {
			return err
		}

		tx.Delete(klineKey)
		return nil
	})

	if err != nil {
		client.Send(nil, server.name, ERR_UNKNOWNERROR, client.nick, msg.Command, fmt.Sprintf("Could not remove ban [%s]", err.Error()))
		return false
	}

	server.klines.RemoveMask(mask)

	client.Notice(fmt.Sprintf("Removed K-Line for %s", mask))
	server.snomasks.Send(sno.LocalXline, fmt.Sprintf(ircfmt.Unescape("%s$r removed K-Line for %s"), client.nick, mask))
	return false
}

func (s *Server) loadKLines() {
	s.klines = NewKLineManager()

	// load from datastore
	s.store.View(func(tx *buntdb.Tx) error {
		//TODO(dan): We could make this safer
		tx.AscendKeys("bans.kline *", func(key, value string) bool {
			// get address name
			key = key[len("bans.kline "):]
			mask := key

			// load ban info
			var info IPBanInfo
			json.Unmarshal([]byte(value), &info)

			// add to the server
			s.klines.AddMask(mask, info.Time, info.Reason, info.OperReason)

			return true // true to continue I guess?
		})
		return nil
	})
}
<|endoftext|>


<commit_before>package drive

import (
    "fmt"
    "mime"
    "os"
    "io"
    "time"
    "path/filepath"
    "google.golang.org/api/googleapi"
    "google.golang.org/api/drive/v3"
)

type UploadArgs struct {
    Out io.Writer
    Progress io.Writer
    Path string
    Name string
    Parents []string
    Mime string
    Recursive bool
    Share bool
    ChunkSize int64
    SizeInBytes bool
}

func (self *Drive) Upload(args UploadArgs) error {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    return self.upload(args)
}

func (self *Drive) upload(args UploadArgs) error {
    f, err := os.Open(args.Path)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    info, err := f.Stat()
    if err != nil {
        return fmt.Errorf("Failed getting file metadata: %s", err)
    }

    if info.IsDir() && !args.Recursive {
        return fmt.Errorf("'%s' is a directory, use --recursive to upload directories", info.Name())
    } else if info.IsDir() {
        args.Name = ""
        return self.uploadDirectory(args)
    } else {
        return self.uploadFile(args)
    }
}

func (self *Drive) uploadDirectory(args UploadArgs) error {
    srcFile, srcFileInfo, err := openFile(args.Path)
    if err != nil {
        return err
    }

    // Make directory on drive
    f, err := self.mkdir(MkdirArgs{
        Out: args.Out,
        Name: srcFileInfo.Name(),
        Parents: args.Parents,
        Share: args.Share,
    })
    if err != nil {
        return err
    }

    // Read files from directory
    names, err := srcFile.Readdirnames(0)
    if err != nil && err != io.EOF {
        return fmt.Errorf("Failed reading directory: %s", err)
    }

    for _, name := range names {
        // Copy args and set new path and parents
        newArgs := args
        newArgs.Path = filepath.Join(args.Path, name)
        newArgs.Parents = []string{f.Id}

        // Upload
        err = self.upload(newArgs)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) uploadFile(args UploadArgs) error {
    srcFile, srcFileInfo, err := openFile(args.Path)
    if err != nil {
        return err
    }

    // Instantiate empty drive file
    dstFile := &drive.File{}

    // Use provided file name or use filename
    if args.Name == "" {
        dstFile.Name = filepath.Base(srcFileInfo.Name())
    } else {
        dstFile.Name = args.Name
    }

    // Set provided mime type or get type based on file extension
    if args.Mime == "" {
        dstFile.MimeType = mime.TypeByExtension(filepath.Ext(dstFile.Name))
    } else {
        dstFile.MimeType = args.Mime
    }

    // Set parent folders
    dstFile.Parents = args.Parents

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, srcFileInfo.Size())

    fmt.Fprintf(args.Out, "\nUploading %s...\n", args.Path)
    started := time.Now()

    f, err := self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
    if err != nil {
        return fmt.Errorf("Failed to upload file: %s", err)
    }

    // Calculate average upload rate
    rate := calcRate(f.Size, started, time.Now())

    fmt.Fprintf(args.Out, "[file] id: %s, md5: %s, name: %s\n", f.Id, f.Md5Checksum, f.Name)
    fmt.Fprintf(args.Out, "Uploaded '%s' at %s/s, total %s\n", f.Name, formatSize(rate, args.SizeInBytes), formatSize(f.Size, args.SizeInBytes))
    return nil
}

type UploadStreamArgs struct {
    Out io.Writer
    In io.Reader
    Name string
    Parents []string
    Mime string
    Share bool
    ChunkSize int64
}

func (self *Drive) UploadStream(args UploadStreamArgs) (err error) {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    // Instantiate empty drive file
    dstFile := &drive.File{Name: args.Name}

    // Set mime type if provided
    if args.Mime != "" {
        dstFile.MimeType = args.Mime
    }

    // Set parent folders
    dstFile.Parents = args.Parents

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    f, err := self.service.Files.Create(dstFile).Media(args.In, chunkSize).Do()
    if err != nil {
        return fmt.Errorf("Failed to upload file: %s", err)
    }

    fmt.Fprintf(args.Out, "Uploaded '%s' at %s, total %d\n", f.Name, "x/s", f.Size)
    //if args.Share {
    //    self.Share(TODO)
    //}
    return
}
<commit_msg>Add openFile function<commit_after>package drive

import (
    "fmt"
    "mime"
    "os"
    "io"
    "time"
    "path/filepath"
    "google.golang.org/api/googleapi"
    "google.golang.org/api/drive/v3"
)

type UploadArgs struct {
    Out io.Writer
    Progress io.Writer
    Path string
    Name string
    Parents []string
    Mime string
    Recursive bool
    Share bool
    ChunkSize int64
    SizeInBytes bool
}

func (self *Drive) Upload(args UploadArgs) error {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    return self.upload(args)
}

func (self *Drive) upload(args UploadArgs) error {
    f, err := os.Open(args.Path)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    info, err := f.Stat()
    if err != nil {
        return fmt.Errorf("Failed getting file metadata: %s", err)
    }

    if info.IsDir() && !args.Recursive {
        return fmt.Errorf("'%s' is a directory, use --recursive to upload directories", info.Name())
    } else if info.IsDir() {
        args.Name = ""
        return self.uploadDirectory(args)
    } else {
        return self.uploadFile(args)
    }
}

func (self *Drive) uploadDirectory(args UploadArgs) error {
    srcFile, srcFileInfo, err := openFile(args.Path)
    if err != nil {
        return err
    }

    // Make directory on drive
    f, err := self.mkdir(MkdirArgs{
        Out: args.Out,
        Name: srcFileInfo.Name(),
        Parents: args.Parents,
        Share: args.Share,
    })
    if err != nil {
        return err
    }

    // Read files from directory
    names, err := srcFile.Readdirnames(0)
    if err != nil && err != io.EOF {
        return fmt.Errorf("Failed reading directory: %s", err)
    }

    for _, name := range names {
        // Copy args and set new path and parents
        newArgs := args
        newArgs.Path = filepath.Join(args.Path, name)
        newArgs.Parents = []string{f.Id}

        // Upload
        err = self.upload(newArgs)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) uploadFile(args UploadArgs) error {
    srcFile, srcFileInfo, err := openFile(args.Path)
    if err != nil {
        return err
    }

    // Instantiate empty drive file
    dstFile := &drive.File{}

    // Use provided file name or use filename
    if args.Name == "" {
        dstFile.Name = filepath.Base(srcFileInfo.Name())
    } else {
        dstFile.Name = args.Name
    }

    // Set provided mime type or get type based on file extension
    if args.Mime == "" {
        dstFile.MimeType = mime.TypeByExtension(filepath.Ext(dstFile.Name))
    } else {
        dstFile.MimeType = args.Mime
    }

    // Set parent folders
    dstFile.Parents = args.Parents

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, srcFileInfo.Size())

    fmt.Fprintf(args.Out, "\nUploading %s...\n", args.Path)
    started := time.Now()

    f, err := self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
    if err != nil {
        return fmt.Errorf("Failed to upload file: %s", err)
    }

    // Calculate average upload rate
    rate := calcRate(f.Size, started, time.Now())

    fmt.Fprintf(args.Out, "[file] id: %s, md5: %s, name: %s\n", f.Id, f.Md5Checksum, f.Name)
    fmt.Fprintf(args.Out, "Uploaded '%s' at %s/s, total %s\n", f.Name, formatSize(rate, args.SizeInBytes), formatSize(f.Size, args.SizeInBytes))
    return nil
}

type UploadStreamArgs struct {
    Out io.Writer
    In io.Reader
    Name string
    Parents []string
    Mime string
    Share bool
    ChunkSize int64
}

func (self *Drive) UploadStream(args UploadStreamArgs) (err error) {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    // Instantiate empty drive file
    dstFile := &drive.File{Name: args.Name}

    // Set mime type if provided
    if args.Mime != "" {
        dstFile.MimeType = args.Mime
    }

    // Set parent folders
    dstFile.Parents = args.Parents

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    f, err := self.service.Files.Create(dstFile).Media(args.In, chunkSize).Do()
    if err != nil {
        return fmt.Errorf("Failed to upload file: %s", err)
    }

    fmt.Fprintf(args.Out, "Uploaded '%s' at %s, total %d\n", f.Name, "x/s", f.Size)
    //if args.Share {
    //    self.Share(TODO)
    //}
    return
}

func openFile(path string) (*os.File, os.FileInfo, error) {
    f, err := os.Open(path)
    if err != nil {
        return nil, nil, fmt.Errorf("Failed to open file: %s", err)
    }

    info, err := f.Stat()
    if err != nil {
        return nil, nil, fmt.Errorf("Failed getting file metadata: %s", err)
    }

    return f, info, nil
}
<|endoftext|>


<commit_before>//	Copyright 2013 slowfei And The Contributors All rights reserved.
//
//	Software Source Code License Agreement (BSD License)
//
//  Create on 2013-10-17
//  Update on 2013-10-22
//  Email  slowfei@foxmail.com
//  Home   http://www.slowfei.com

//	日志的配置文件加载操作
package SFLog

import (
	"encoding/json"
	"errors"
	"github.com/slowfei/gosfcore/utils/filemanager"
	"io/ioutil"
	"path/filepath"
	"sync"
)

const (
	//	separator
	LOG_SEPARATOR = "_"
	//	global group log config
	KEY_GLOBAL_GROUP_LOG_CONFIG = "globalGroup"
	// defalut pattern
	DEFAULT_PATTERN = "${yyyy}-${MM}-${dd} ${hh}:${mm}:${ss}${SSSSSS} [${TARGET}] ([${LOG_GROUP}][${LOG_TAG}][L${FILE_LINE} ${FUNC_NAME}])\n${MSG}"

	//	log target
	LOG_INFO  = "info"
	LOG_DEBUG = "debug"
	LOG_ERROR = "error"
	LOG_WARN  = "warn"
	LOG_FATAL = "fatal"
	LOG_PANIC = "panic"

	//	appender tag
	LOG_APPENDER = "appender"

	//	appender type
	VAL_APPENDER_CONSOLE = "console"
	VAL_APPENDER_FILE    = "file"
	VAL_APPENDER_MONGODB = "mongodb"
	VAL_APPENDER_EMAIL   = "email"
	VAL_APPENDER_HTML    = "html"
	VAL_APPENDER_NONE    = "none"
)

var (
	//	log target value
	TargetInfo  = LogTarget("info")
	TargetDebug = LogTarget("debug")
	TargetError = LogTarget("error")
	TargetWarn  = LogTarget("warn")
	TargetFatal = LogTarget("fatal")
	TargetPanic = LogTarget("panic")

	//	默认配置的json
	_defaultConfig = `
		{
			"InitAppenders":[
				"console"
			],
			"LogGroups" :{
				"` + KEY_GLOBAL_GROUP_LOG_CONFIG + `" :{
					"Appender":[
						"console"
					],
					"none":false,
					"ConsolePattern":"${yyyy}-${MM}-${dd} ${hh}:${mm}:${ss}${SSSSSS} [${TARGET}] ([${LOG_GROUP}][${LOG_TAG}][L${FILE_LINE} ${FUNC_NAME}])\n${MSG}"
				}
			}
		}
	`

	//	log config map
	_sharedLogConfig *MainLogConfig = nil
	_rwmutex         sync.RWMutex
)

//	log init
func init() {
	_sharedLogConfig = new(MainLogConfig)
	_sharedLogConfig.LogGroups = make(map[string]LogConfig)

	//	初始化的时候加载一次默认的配置
	if err := json.Unmarshal([]byte(_defaultConfig), _sharedLogConfig); nil != err {
		panic(err)
	}
}

//	reset load config
//	@configPath	相对或绝对路径，相对路径以执行文件目录开始
func LoadConfig(configPath string) error {
	if nil == _sharedLogConfig {
		return errors.New("not start log manager: SFLog.StartLogManager(...)")
	}

	if 0 != len(configPath) {
		_rwmutex.Lock()
		defer _rwmutex.Unlock()

		var path string
		if filepath.IsAbs(configPath) {
			path = configPath
		} else {
			path = filepath.Join(SFFileManager.GetExecDir(), configPath)
		}

		isExists, isDir, _ := SFFileManager.Exists(path)
		if !isExists || isDir {
			return errors.New("failed to load configuration file:" + configPath)
		}

		jsonData, e1 := ioutil.ReadFile(path)
		if nil != e1 {
			return e1
		}

		e2 := json.Unmarshal(jsonData, _sharedLogConfig)

		if nil != e2 {
			return e2
		}

		loadAppenders()
	}
	return nil
}

//	reset load config
//	@jsonData
func LoadConfigByJson(jsonData []byte) error {
	if nil == _sharedLogConfig {
		return errors.New("not start log manager: SFLog.StartLogManager(...)")
	}

	e2 := json.Unmarshal(jsonData, _sharedLogConfig)

	if nil != e2 {
		return e2
	}

	loadAppenders()

	return nil
}

//	main log config
type MainLogConfig struct {
	InitAppenders []string             // init appenders impl. console, file...
	TimeFormat    string               // time format
	LogGroups     map[string]LogConfig // log tags日志标识集合元素
}

//	log config
type LogConfig struct {
	//	target config
	Info  *TargetConfigInfo `json:"info"`
	Debug *TargetConfigInfo `json:"debug"`
	Error *TargetConfigInfo `json:"error"`
	Warn  *TargetConfigInfo `json:"warn"`
	Fatal *TargetConfigInfo `json:"fatal"`
	Panic *TargetConfigInfo `json:"panic"`

	//	global config
	*TargetConfigInfo
}

//	log target, info,debug,error,warn,fatal,panics
type LogTarget string

//	target config info,contain info,debug,error,warn,fatal,panics
type TargetConfigInfo struct {
	Appender []string
	*AppenderFileConfig
	*AppenderConsoleConfig
	*AppenderMongodbConfig
	*AppenderEmailConfig
	*AppenderHtmlConfig
	AppenderNoneConfig
}

//	appender none
type AppenderNoneConfig struct {
	None bool
}

//	appender mongodb
type AppenderMongodbConfig struct {
}
<commit_msg>AppenderNoneConfig针对json小写的struct tag<commit_after>//	Copyright 2013 slowfei And The Contributors All rights reserved.
//
//	Software Source Code License Agreement (BSD License)
//
//  Create on 2013-10-17
//  Update on 2014-09-23
//  Email  slowfei@foxmail.com
//  Home   http://www.slowfei.com

//	日志的配置文件加载操作
package SFLog

import (
	"encoding/json"
	"errors"
	"github.com/slowfei/gosfcore/utils/filemanager"
	"io/ioutil"
	"path/filepath"
	"sync"
)

const (
	//	separator
	LOG_SEPARATOR = "_"
	//	global group log config
	KEY_GLOBAL_GROUP_LOG_CONFIG = "globalGroup"
	// defalut pattern
	DEFAULT_PATTERN = "${yyyy}-${MM}-${dd} ${hh}:${mm}:${ss}${SSSSSS} [${TARGET}] ([${LOG_GROUP}][${LOG_TAG}][L${FILE_LINE} ${FUNC_NAME}])\n${MSG}"

	//	log target
	LOG_INFO  = "info"
	LOG_DEBUG = "debug"
	LOG_ERROR = "error"
	LOG_WARN  = "warn"
	LOG_FATAL = "fatal"
	LOG_PANIC = "panic"

	//	appender tag
	LOG_APPENDER = "appender"

	//	appender type
	VAL_APPENDER_CONSOLE = "console"
	VAL_APPENDER_FILE    = "file"
	VAL_APPENDER_MONGODB = "mongodb"
	VAL_APPENDER_EMAIL   = "email"
	VAL_APPENDER_HTML    = "html"
	VAL_APPENDER_NONE    = "none"
)

var (
	//	log target value
	TargetInfo  = LogTarget("info")
	TargetDebug = LogTarget("debug")
	TargetError = LogTarget("error")
	TargetWarn  = LogTarget("warn")
	TargetFatal = LogTarget("fatal")
	TargetPanic = LogTarget("panic")

	//	默认配置的json
	_defaultConfig = `
		{
			"InitAppenders":[
				"console"
			],
			"LogGroups" :{
				"` + KEY_GLOBAL_GROUP_LOG_CONFIG + `" :{
					"Appender":[
						"console"
					],
					"none":false,
					"ConsolePattern":"${yyyy}-${MM}-${dd} ${hh}:${mm}:${ss}${SSSSSS} [${TARGET}] ([${LOG_GROUP}][${LOG_TAG}][L${FILE_LINE} ${FUNC_NAME}])\n${MSG}"
				}
			}
		}
	`

	//	log config map
	_sharedLogConfig *MainLogConfig = nil
	_rwmutex         sync.RWMutex
)

//	log init
func init() {
	_sharedLogConfig = new(MainLogConfig)
	_sharedLogConfig.LogGroups = make(map[string]LogConfig)

	//	初始化的时候加载一次默认的配置
	if err := json.Unmarshal([]byte(_defaultConfig), _sharedLogConfig); nil != err {
		panic(err)
	}
}

//	reset load config
//	@configPath	相对或绝对路径，相对路径以执行文件目录开始
func LoadConfig(configPath string) error {
	if nil == _sharedLogConfig {
		return errors.New("not start log manager: SFLog.StartLogManager(...)")
	}

	if 0 != len(configPath) {
		_rwmutex.Lock()
		defer _rwmutex.Unlock()

		var path string
		if filepath.IsAbs(configPath) {
			path = configPath
		} else {
			path = filepath.Join(SFFileManager.GetExecDir(), configPath)
		}

		isExists, isDir, _ := SFFileManager.Exists(path)
		if !isExists || isDir {
			return errors.New("failed to load configuration file:" + configPath)
		}

		jsonData, e1 := ioutil.ReadFile(path)
		if nil != e1 {
			return e1
		}

		e2 := json.Unmarshal(jsonData, _sharedLogConfig)

		if nil != e2 {
			return e2
		}

		loadAppenders()
	}
	return nil
}

//	reset load config
//	@jsonData
func LoadConfigByJson(jsonData []byte) error {
	if nil == _sharedLogConfig {
		return errors.New("not start log manager: SFLog.StartLogManager(...)")
	}

	e2 := json.Unmarshal(jsonData, _sharedLogConfig)

	if nil != e2 {
		return e2
	}

	loadAppenders()

	return nil
}

//	main log config
type MainLogConfig struct {
	InitAppenders []string             // init appenders impl. console, file...
	TimeFormat    string               // time format
	LogGroups     map[string]LogConfig // log tags日志标识集合元素
}

//	log config
type LogConfig struct {
	//	target config
	Info  *TargetConfigInfo `json:"info"`
	Debug *TargetConfigInfo `json:"debug"`
	Error *TargetConfigInfo `json:"error"`
	Warn  *TargetConfigInfo `json:"warn"`
	Fatal *TargetConfigInfo `json:"fatal"`
	Panic *TargetConfigInfo `json:"panic"`

	//	global config
	*TargetConfigInfo
}

//	log target, info,debug,error,warn,fatal,panics
type LogTarget string

//	target config info,contain info,debug,error,warn,fatal,panics
type TargetConfigInfo struct {
	Appender []string
	*AppenderFileConfig
	*AppenderConsoleConfig
	*AppenderMongodbConfig
	*AppenderEmailConfig
	*AppenderHtmlConfig
	AppenderNoneConfig
}

//	appender none
type AppenderNoneConfig struct {
	None bool `json:none`
}

//	appender mongodb
type AppenderMongodbConfig struct {
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 SUSE LLC. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package securejoin

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

// TODO: These tests won't work on plan9 because it doesn't have symlinks, and
//       also we use '/' here explicitly which probably won't work on Windows.

func symlink(t *testing.T, oldname, newname string) {
	if err := os.Symlink(oldname, newname); err != nil {
		t.Fatal(err)
	}
}

// Test basic handling of symlink expansion.
func TestSymlink(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestSymlink")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	symlink(t, "somepath", filepath.Join(dir, "etc"))
	symlink(t, "../../../../../etc", filepath.Join(dir, "etclink"))
	symlink(t, "/../../../../../etc/passwd", filepath.Join(dir, "passwd"))

	for _, test := range []struct {
		root, unsafe string
		expected     string
	}{
		// Make sure that expansion with a root of '/' proceeds in the expected fashion.
		{"/", filepath.Join(dir, "passwd"), "/etc/passwd"},
		{"/", filepath.Join(dir, "etclink"), "/etc"},
		{"/", filepath.Join(dir, "etc"), filepath.Join(dir, "somepath")},
		// Now test scoped expansion.
		{dir, "passwd", filepath.Join(dir, "somepath", "passwd")},
		{dir, "etclink", filepath.Join(dir, "somepath")},
		{dir, "etc", filepath.Join(dir, "somepath")},
		{dir, "etc/test", filepath.Join(dir, "somepath", "test")},
		{dir, "etc/test/..", filepath.Join(dir, "somepath")},
	} {
		got, err := SecureJoin(test.root, test.unsafe)
		if err != nil {
			t.Errorf("securejoin(%q, %q): unexpected error: %v", test.root, test.unsafe, err)
			continue
		}
		if got != test.expected {
			t.Errorf("securejoin(%q, %q): expected %q, got %q", test.root, test.unsafe, test.expected, got)
			continue
		}
	}
}

// In a path without symlinks, SecureJoin is equivalent to Clean+Join.
func TestNoSymlink(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestNoSymlink")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for _, test := range []struct {
		root, unsafe string
	}{
		// TODO: Do we need to have some conditional FromSlash handling here?
		{dir, "somepath"},
		{dir, "even/more/path"},
		{dir, "/this/is/a/path"},
		{dir, "also/a/../path/././/with/some/./.././junk"},
		{dir, "yetanother/../path/././/with/some/./.././junk../../../../../../../../../../../../etc/passwd"},
		{dir, "/../../../../etc/passwd"},
		{dir, "../../../../somedir"},
		{dir, "../../../../"},
		{dir, "./../../.././././../../../etc passwd"},
	} {
		expected := filepath.Join(test.root, filepath.Clean(string(filepath.Separator)+test.unsafe))
		got, err := SecureJoin(test.root, test.unsafe)
		if err != nil {
			t.Errorf("securejoin(%q, %q): unexpected error: %v", test.root, test.unsafe, err)
			continue
		}
		if got != expected {
			t.Errorf("securejoin(%q, %q): expected %q, got %q", test.root, test.unsafe, expected, got)
			continue
		}
	}
}

// Make sure that .. is **not** expanded lexically.
func TestNonLexical(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestNonLexical")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	os.MkdirAll(filepath.Join(dir, "subdir"), 0755)
	os.MkdirAll(filepath.Join(dir, "cousinparent", "cousin"), 0755)
	symlink(t, "../cousinparent/cousin", filepath.Join(dir, "subdir", "link"))
	symlink(t, "/../cousinparent/cousin", filepath.Join(dir, "subdir", "link2"))
	symlink(t, "/../../../../../cousinparent/cousin", filepath.Join(dir, "subdir", "link3"))

	for _, test := range []struct {
		root, unsafe string
		expected     string
	}{
		{dir, "subdir", filepath.Join(dir, "subdir")},
		{dir, "subdir/link/test", filepath.Join(dir, "cousinparent", "cousin", "test")},
		{dir, "subdir/link2/test", filepath.Join(dir, "cousinparent", "cousin", "test")},
		{dir, "subdir/link3/test", filepath.Join(dir, "cousinparent", "cousin", "test")},
		{dir, "subdir/../test", filepath.Join(dir, "test")},
		// This is the divergence from a simple filepath.Clean implementation.
		{dir, "subdir/link/../test", filepath.Join(dir, "cousinparent", "test")},
		{dir, "subdir/link2/../test", filepath.Join(dir, "cousinparent", "test")},
		{dir, "subdir/link3/../test", filepath.Join(dir, "cousinparent", "test")},
	} {
		got, err := SecureJoin(test.root, test.unsafe)
		if err != nil {
			t.Errorf("securejoin(%q, %q): unexpected error: %v", test.root, test.unsafe, err)
			continue
		}
		if got != test.expected {
			t.Errorf("securejoin(%q, %q): expected %q, got %q", test.root, test.unsafe, test.expected, got)
			continue
		}
	}
}

// Make sure that symlink loops result in errors.
func TestSymlinkLoop(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestSymlinkLoop")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	os.MkdirAll(filepath.Join(dir, "subdir"), 0755)
	symlink(t, "../../../../../../../../../../../../path", filepath.Join(dir, "subdir", "link"))
	symlink(t, "/subdir/link", filepath.Join(dir, "path"))
	symlink(t, "/../../../../self", filepath.Join(dir, "self"))

	for _, test := range []struct {
		root, unsafe string
	}{
		{dir, "subdir/link"},
		{dir, "path"},
		{dir, "../../path"},
		{dir, "subdir/link/../.."},
		{dir, "../../../subdir/link/../.."},
		{dir, "self"},
		{dir, "self/.."},
		{dir, "/../../../self/.."},
		{dir, "/self/././.."},
	} {
		got, err := SecureJoin(test.root, test.unsafe)
		if err == nil {
			t.Errorf("securejoin(%q, %q): expected error, got %q", test.root, test.unsafe, got)
			continue
		}
	}
}
<commit_msg>join_test: fix OS X builds on travis<commit_after>// Copyright (C) 2017 SUSE LLC. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package securejoin

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

// TODO: These tests won't work on plan9 because it doesn't have symlinks, and
//       also we use '/' here explicitly which probably won't work on Windows.

func symlink(t *testing.T, oldname, newname string) {
	if err := os.Symlink(oldname, newname); err != nil {
		t.Fatal(err)
	}
}

// Test basic handling of symlink expansion.
func TestSymlink(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestSymlink")
	if err != nil {
		t.Fatal(err)
	}
	dir, err = filepath.EvalSymlinks(dir)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	symlink(t, "somepath", filepath.Join(dir, "etc"))
	symlink(t, "../../../../../../../../../../../../../etc", filepath.Join(dir, "etclink"))
	symlink(t, "/../../../../../../../../../../../../../etc/passwd", filepath.Join(dir, "passwd"))

	for _, test := range []struct {
		root, unsafe string
		expected     string
	}{
		// Make sure that expansion with a root of '/' proceeds in the expected fashion.
		{"/", filepath.Join(dir, "passwd"), "/etc/passwd"},
		{"/", filepath.Join(dir, "etclink"), "/etc"},
		{"/", filepath.Join(dir, "etc"), filepath.Join(dir, "somepath")},
		// Now test scoped expansion.
		{dir, "passwd", filepath.Join(dir, "somepath", "passwd")},
		{dir, "etclink", filepath.Join(dir, "somepath")},
		{dir, "etc", filepath.Join(dir, "somepath")},
		{dir, "etc/test", filepath.Join(dir, "somepath", "test")},
		{dir, "etc/test/..", filepath.Join(dir, "somepath")},
	} {
		got, err := SecureJoin(test.root, test.unsafe)
		if err != nil {
			t.Errorf("securejoin(%q, %q): unexpected error: %v", test.root, test.unsafe, err)
			continue
		}
		// This is only for OS X, where /etc is a symlink to /private/etc. In
		// principle, SecureJoin(/, pth) is the same as EvalSymlinks(pth) in
		// the case where the path exists.
		if test.root == "/" {
			if expected, err := filepath.EvalSymlinks(test.expected); err == nil {
				test.expected = expected
			}
		}
		if got != test.expected {
			t.Errorf("securejoin(%q, %q): expected %q, got %q", test.root, test.unsafe, test.expected, got)
			continue
		}
	}
}

// In a path without symlinks, SecureJoin is equivalent to Clean+Join.
func TestNoSymlink(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestNoSymlink")
	if err != nil {
		t.Fatal(err)
	}
	dir, err = filepath.EvalSymlinks(dir)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	for _, test := range []struct {
		root, unsafe string
	}{
		// TODO: Do we need to have some conditional FromSlash handling here?
		{dir, "somepath"},
		{dir, "even/more/path"},
		{dir, "/this/is/a/path"},
		{dir, "also/a/../path/././/with/some/./.././junk"},
		{dir, "yetanother/../path/././/with/some/./.././junk../../../../../../../../../../../../etc/passwd"},
		{dir, "/../../../../../../../../../../../../../../../../etc/passwd"},
		{dir, "../../../../../../../../../../../../../../../../somedir"},
		{dir, "../../../../../../../../../../../../../../../../"},
		{dir, "./../../.././././../../../../../../../../../../../../../../../../etc passwd"},
	} {
		expected := filepath.Join(test.root, filepath.Clean(string(filepath.Separator)+test.unsafe))
		got, err := SecureJoin(test.root, test.unsafe)
		if err != nil {
			t.Errorf("securejoin(%q, %q): unexpected error: %v", test.root, test.unsafe, err)
			continue
		}
		if got != expected {
			t.Errorf("securejoin(%q, %q): expected %q, got %q", test.root, test.unsafe, expected, got)
			continue
		}
	}
}

// Make sure that .. is **not** expanded lexically.
func TestNonLexical(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestNonLexical")
	if err != nil {
		t.Fatal(err)
	}
	dir, err = filepath.EvalSymlinks(dir)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	os.MkdirAll(filepath.Join(dir, "subdir"), 0755)
	os.MkdirAll(filepath.Join(dir, "cousinparent", "cousin"), 0755)
	symlink(t, "../cousinparent/cousin", filepath.Join(dir, "subdir", "link"))
	symlink(t, "/../cousinparent/cousin", filepath.Join(dir, "subdir", "link2"))
	symlink(t, "/../../../../../../../../../../../../../../../../cousinparent/cousin", filepath.Join(dir, "subdir", "link3"))

	for _, test := range []struct {
		root, unsafe string
		expected     string
	}{
		{dir, "subdir", filepath.Join(dir, "subdir")},
		{dir, "subdir/link/test", filepath.Join(dir, "cousinparent", "cousin", "test")},
		{dir, "subdir/link2/test", filepath.Join(dir, "cousinparent", "cousin", "test")},
		{dir, "subdir/link3/test", filepath.Join(dir, "cousinparent", "cousin", "test")},
		{dir, "subdir/../test", filepath.Join(dir, "test")},
		// This is the divergence from a simple filepath.Clean implementation.
		{dir, "subdir/link/../test", filepath.Join(dir, "cousinparent", "test")},
		{dir, "subdir/link2/../test", filepath.Join(dir, "cousinparent", "test")},
		{dir, "subdir/link3/../test", filepath.Join(dir, "cousinparent", "test")},
	} {
		got, err := SecureJoin(test.root, test.unsafe)
		if err != nil {
			t.Errorf("securejoin(%q, %q): unexpected error: %v", test.root, test.unsafe, err)
			continue
		}
		if got != test.expected {
			t.Errorf("securejoin(%q, %q): expected %q, got %q", test.root, test.unsafe, test.expected, got)
			continue
		}
	}
}

// Make sure that symlink loops result in errors.
func TestSymlinkLoop(t *testing.T) {
	dir, err := ioutil.TempDir("", "TestSymlinkLoop")
	if err != nil {
		t.Fatal(err)
	}
	dir, err = filepath.EvalSymlinks(dir)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	os.MkdirAll(filepath.Join(dir, "subdir"), 0755)
	symlink(t, "../../../../../../../../../../../../../../../../path", filepath.Join(dir, "subdir", "link"))
	symlink(t, "/subdir/link", filepath.Join(dir, "path"))
	symlink(t, "/../../../../../../../../../../../../../../../../self", filepath.Join(dir, "self"))

	for _, test := range []struct {
		root, unsafe string
	}{
		{dir, "subdir/link"},
		{dir, "path"},
		{dir, "../../path"},
		{dir, "subdir/link/../.."},
		{dir, "../../../../../../../../../../../../../../../../subdir/link/../../../../../../../../../../../../../../../.."},
		{dir, "self"},
		{dir, "self/.."},
		{dir, "/../../../../../../../../../../../../../../../../self/.."},
		{dir, "/self/././.."},
	} {
		got, err := SecureJoin(test.root, test.unsafe)
		if err == nil {
			t.Errorf("securejoin(%q, %q): expected error, got %q", test.root, test.unsafe, got)
			continue
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the TIFF file format, including
// an "off by one" algorithmic difference when compared to standard LZW.

package tiff

/*
This file was branched from src/pkg/compress/lzw/reader.go in the
standard library. Differences from the original are marked with "NOTE".

The tif_lzw.c file in the libtiff C library has this comment:

----
The 5.0 spec describes a different algorithm than Aldus
implements.  Specifically, Aldus does code length transitions
one code earlier than should be done (for real LZW).
Earlier versions of this library implemented the correct
LZW algorithm, but emitted codes in a bit order opposite
to the TIFF spec.  Thus, to maintain compatibility w/ Aldus
we interpret MSB-LSB ordered codes to be images written w/
old versions of this library, but otherwise adhere to the
Aldus "off by one" algorithm.
----

The Go code doesn't read (invalid) TIFF files written by old versions of
libtiff, but the LZW algorithm in this package still differs from the one in
Go's standard package library to accomodate this "off by one" in valid TIFFs.
*/

import (
	"bufio"
	"errors"
	"fmt"
	"io"
)

// lzwOrder specifies the bit ordering in an LZW data stream.
type lzwOrder int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	lzwLSB lzwOrder = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	lzwMSB
)

const (
	lzwMaxWidth           = 12
	lzwDecoderInvalidCode = 0xffff
	lzwFlushBuffer        = 1 << lzwMaxWidth
)

// lzwDecoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type lzwDecoder struct {
	r        io.ByteReader
	bits     uint32
	nBits    uint
	width    uint
	read     func(*lzwDecoder) (uint16, error) // readLSB or readMSB
	litWidth int                               // width in bits of literal codes
	err      error

	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	// overflow is the code at which hi overflows the code width. NOTE: TIFF's LZW is "off by one".
	// last is the most recently seen code, or lzwDecoderInvalidCode.
	clear, eof, hi, overflow, last uint16

	// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
	//   suffix[c] is the last of these bytes.
	//   prefix[c] is the code for all but the last byte.
	//   This code can either be a literal code or another code in [lo, c).
	// The c == hi case is a special case.
	suffix [1 << lzwMaxWidth]uint8
	prefix [1 << lzwMaxWidth]uint16

	// output is the temporary output buffer.
	// Literal codes are accumulated from the start of the buffer.
	// Non-literal codes decode to a sequence of suffixes that are first
	// written right-to-left from the end of the buffer before being copied
	// to the start of the buffer.
	// It is flushed when it contains >= 1<<lzwMaxWidth bytes,
	// so that there is always room to decode an entire code.
	output [2 * 1 << lzwMaxWidth]byte
	o      int    // write index into output
	toRead []byte // bytes to return from Read
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *lzwDecoder) readLSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *lzwDecoder) readMSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

func (d *lzwDecoder) Read(b []byte) (int, error) {
	for {
		if len(d.toRead) > 0 {
			n := copy(b, d.toRead)
			d.toRead = d.toRead[n:]
			return n, nil
		}
		if d.err != nil {
			return 0, d.err
		}
		d.decode()
	}
}

// decode decompresses bytes from r and leaves them in d.toRead.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func (d *lzwDecoder) decode() {
	// Loop over the code stream, converting codes into decompressed bytes.
	for {
		code, err := d.read(d)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			d.err = err
			return
		}
		switch {
		case code < d.clear:
			// We have a literal code.
			d.output[d.o] = uint8(code)
			d.o++
			if d.last != lzwDecoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(code)
				d.prefix[d.hi] = d.last
			}
		case code == d.clear:
			d.width = 1 + uint(d.litWidth)
			d.hi = d.eof
			d.overflow = 1 << d.width
			d.last = lzwDecoderInvalidCode
			continue
		case code == d.eof:
			d.flush()
			d.err = io.EOF
			return
		case code <= d.hi:
			c, i := code, len(d.output)-1
			if code == d.hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = d.last
				for c >= d.clear {
					c = d.prefix[c]
				}
				d.output[i] = uint8(c)
				i--
				c = d.last
			}
			// Copy the suffix chain into output and then write that to w.
			for c >= d.clear {
				d.output[i] = d.suffix[c]
				i--
				c = d.prefix[c]
			}
			d.output[i] = uint8(c)
			d.o += copy(d.output[d.o:], d.output[i:])
			if d.last != lzwDecoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(c)
				d.prefix[d.hi] = d.last
			}
		default:
			d.err = errors.New("lzw: invalid code")
			return
		}
		d.last, d.hi = code, d.hi+1
		if d.hi+1 >= d.overflow { // NOTE: the "+1" is where TIFF's LZW differs from the standard algorithm.
			if d.width == lzwMaxWidth {
				d.last = lzwDecoderInvalidCode
			} else {
				d.width++
				d.overflow <<= 1
			}
		}
		if d.o >= lzwFlushBuffer {
			d.flush()
			return
		}
	}
}

func (d *lzwDecoder) flush() {
	d.toRead = d.output[:d.o]
	d.o = 0
}

var errClosed = errors.New("compress/lzw: reader/writer is closed")

func (d *lzwDecoder) Close() error {
	d.err = errClosed // in case any Reads come along
	return nil
}

// newLzwReader creates a new io.ReadCloser.
// Reads from the returned io.ReadCloser read and decompress data from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8.
func newLzwReader(r io.Reader, order lzwOrder, litWidth int) io.ReadCloser {
	d := new(lzwDecoder)
	switch order {
	case lzwLSB:
		d.read = (*lzwDecoder).readLSB
	case lzwMSB:
		d.read = (*lzwDecoder).readMSB
	default:
		d.err = errors.New("lzw: unknown order")
		return d
	}
	if litWidth < 2 || 8 < litWidth {
		d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth)
		return d
	}
	if br, ok := r.(io.ByteReader); ok {
		d.r = br
	} else {
		d.r = bufio.NewReader(r)
	}
	d.litWidth = litWidth
	d.width = 1 + uint(litWidth)
	d.clear = uint16(1) << uint(litWidth)
	d.eof, d.hi = d.clear+1, d.clear+1
	d.overflow = uint16(1) << d.width
	d.last = lzwDecoderInvalidCode

	return d
}
<commit_msg>lzw: sync x/image/tiff/lzw<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lzw implements the Lempel-Ziv-Welch compressed data format,
// described in T. A. Welch, ``A Technique for High-Performance Data
// Compression'', Computer, 17(6) (June 1984), pp 8-19.
//
// In particular, it implements LZW as used by the TIFF file format, including
// an "off by one" algorithmic difference when compared to standard LZW.

package tiff

/*
This file was branched from src/pkg/compress/lzw/reader.go in the
standard library. Differences from the original are marked with "NOTE".

The tif_lzw.c file in the libtiff C library has this comment:

----
The 5.0 spec describes a different algorithm than Aldus
implements.  Specifically, Aldus does code length transitions
one code earlier than should be done (for real LZW).
Earlier versions of this library implemented the correct
LZW algorithm, but emitted codes in a bit order opposite
to the TIFF spec.  Thus, to maintain compatibility w/ Aldus
we interpret MSB-LSB ordered codes to be images written w/
old versions of this library, but otherwise adhere to the
Aldus "off by one" algorithm.
----

The Go code doesn't read (invalid) TIFF files written by old versions of
libtiff, but the LZW algorithm in this package still differs from the one in
Go's standard package library to accomodate this "off by one" in valid TIFFs.
*/

import (
	"bufio"
	"errors"
	"fmt"
	"io"
)

// lzwOrder specifies the bit ordering in an LZW data stream.
type lzwOrder int

const (
	// LSB means Least Significant Bits first, as used in the GIF file format.
	lzwLSB lzwOrder = iota
	// MSB means Most Significant Bits first, as used in the TIFF and PDF
	// file formats.
	lzwMSB
)

const (
	lzwMaxWidth           = 12
	lzwDecoderInvalidCode = 0xffff
	lzwFlushBuffer        = 1 << lzwMaxWidth
)

// lzwDecoder is the state from which the readXxx method converts a byte
// stream into a code stream.
type lzwDecoder struct {
	r        io.ByteReader
	bits     uint32
	nBits    uint
	width    uint
	read     func(*lzwDecoder) (uint16, error) // readLSB or readMSB
	litWidth int                               // width in bits of literal codes
	err      error

	// The first 1<<litWidth codes are literal codes.
	// The next two codes mean clear and EOF.
	// Other valid codes are in the range [lo, hi] where lo := clear + 2,
	// with the upper bound incrementing on each code seen.
	// overflow is the code at which hi overflows the code width. NOTE: TIFF's LZW is "off by one".
	// last is the most recently seen code, or lzwDecoderInvalidCode.
	clear, eof, hi, overflow, last uint16

	// Each code c in [lo, hi] expands to two or more bytes. For c != hi:
	//   suffix[c] is the last of these bytes.
	//   prefix[c] is the code for all but the last byte.
	//   This code can either be a literal code or another code in [lo, c).
	// The c == hi case is a special case.
	suffix [1 << lzwMaxWidth]uint8
	prefix [1 << lzwMaxWidth]uint16

	// output is the temporary output buffer.
	// Literal codes are accumulated from the start of the buffer.
	// Non-literal codes decode to a sequence of suffixes that are first
	// written right-to-left from the end of the buffer before being copied
	// to the start of the buffer.
	// It is flushed when it contains >= 1<<lzwMaxWidth bytes,
	// so that there is always room to decode an entire code.
	output [2 * 1 << lzwMaxWidth]byte
	o      int    // write index into output
	toRead []byte // bytes to return from Read
}

// readLSB returns the next code for "Least Significant Bits first" data.
func (d *lzwDecoder) readLSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << d.nBits
		d.nBits += 8
	}
	code := uint16(d.bits & (1<<d.width - 1))
	d.bits >>= d.width
	d.nBits -= d.width
	return code, nil
}

// readMSB returns the next code for "Most Significant Bits first" data.
func (d *lzwDecoder) readMSB() (uint16, error) {
	for d.nBits < d.width {
		x, err := d.r.ReadByte()
		if err != nil {
			return 0, err
		}
		d.bits |= uint32(x) << (24 - d.nBits)
		d.nBits += 8
	}
	code := uint16(d.bits >> (32 - d.width))
	d.bits <<= d.width
	d.nBits -= d.width
	return code, nil
}

func (d *lzwDecoder) Read(b []byte) (int, error) {
	for {
		if len(d.toRead) > 0 {
			n := copy(b, d.toRead)
			d.toRead = d.toRead[n:]
			return n, nil
		}
		if d.err != nil {
			return 0, d.err
		}
		d.decode()
	}
}

// decode decompresses bytes from r and leaves them in d.toRead.
// read specifies how to decode bytes into codes.
// litWidth is the width in bits of literal codes.
func (d *lzwDecoder) decode() {
	// Loop over the code stream, converting codes into decompressed bytes.
	for {
		code, err := d.read(d)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			d.err = err
			return
		}
		switch {
		case code < d.clear:
			// We have a literal code.
			d.output[d.o] = uint8(code)
			d.o++
			if d.last != lzwDecoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(code)
				d.prefix[d.hi] = d.last
			}
		case code == d.clear:
			d.width = 1 + uint(d.litWidth)
			d.hi = d.eof
			d.overflow = 1 << d.width
			d.last = lzwDecoderInvalidCode
			continue
		case code == d.eof:
			d.flush()
			d.err = io.EOF
			return
		case code <= d.hi:
			c, i := code, len(d.output)-1
			if code == d.hi {
				// code == hi is a special case which expands to the last expansion
				// followed by the head of the last expansion. To find the head, we walk
				// the prefix chain until we find a literal code.
				c = d.last
				for c >= d.clear {
					c = d.prefix[c]
				}
				d.output[i] = uint8(c)
				i--
				c = d.last
			}
			// Copy the suffix chain into output and then write that to w.
			for c >= d.clear {
				d.output[i] = d.suffix[c]
				i--
				c = d.prefix[c]
			}
			d.output[i] = uint8(c)
			d.o += copy(d.output[d.o:], d.output[i:])
			if d.last != lzwDecoderInvalidCode {
				// Save what the hi code expands to.
				d.suffix[d.hi] = uint8(c)
				d.prefix[d.hi] = d.last
			}
		default:
			d.err = errors.New("lzw: invalid code")
			return
		}
		d.last, d.hi = code, d.hi+1
		if d.hi+1 >= d.overflow { // NOTE: the "+1" is where TIFF's LZW differs from the standard algorithm.
			if d.width == lzwMaxWidth {
				d.last = lzwDecoderInvalidCode
			} else {
				d.width++
				d.overflow <<= 1
			}
		}
		if d.o >= lzwFlushBuffer {
			d.flush()
			return
		}
	}
}

func (d *lzwDecoder) flush() {
	d.toRead = d.output[:d.o]
	d.o = 0
}

var lzwErrClosed = errors.New("lzw: reader/writer is closed")

func (d *lzwDecoder) Close() error {
	d.err = lzwErrClosed // in case any Reads come along
	return nil
}

// newLzwReader creates a new io.ReadCloser.
// Reads from the returned io.ReadCloser read and decompress data from r.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser when
// finished reading.
// The number of bits to use for literal codes, litWidth, must be in the
// range [2,8] and is typically 8. It must equal the litWidth
// used during compression.
func newLzwReader(r io.Reader, order lzwOrder, litWidth int) io.ReadCloser {
	d := new(lzwDecoder)
	switch order {
	case lzwLSB:
		d.read = (*lzwDecoder).readLSB
	case lzwMSB:
		d.read = (*lzwDecoder).readMSB
	default:
		d.err = errors.New("lzw: unknown order")
		return d
	}
	if litWidth < 2 || 8 < litWidth {
		d.err = fmt.Errorf("lzw: litWidth %d out of range", litWidth)
		return d
	}
	if br, ok := r.(io.ByteReader); ok {
		d.r = br
	} else {
		d.r = bufio.NewReader(r)
	}
	d.litWidth = litWidth
	d.width = 1 + uint(litWidth)
	d.clear = uint16(1) << uint(litWidth)
	d.eof, d.hi = d.clear+1, d.clear+1
	d.overflow = uint16(1) << d.width
	d.last = lzwDecoderInvalidCode

	return d
}
<|endoftext|>