<commit_before>package toolbox

import (
	"strings"
	"net/url"
)

//ExtractURIParameters parses URIs to extract {<param>} defined in templateURI from requestURI, it returns extracted parameters and flag if requestURI matched templateURI
func ExtractURIParameters(templateURI, requestURI string) (map[string]string, bool) {
	var expectingValue, expectingName bool
	var name, value string
	var uriParameters = make(map[string]string)
	maxLength := len(templateURI) + len(requestURI)
	var requestURIIndex, templateURIIndex int

	questionMarkPosition := strings.Index(requestURI, "?")
	if questionMarkPosition != -1 {
		requestURI = string(requestURI[:questionMarkPosition])
	}

	for k := 0; k < maxLength; k++ {
		var requestChar, routingChar string

		if requestURIIndex < len(requestURI) {
			requestChar = requestURI[requestURIIndex: requestURIIndex+1]
		}

		if templateURIIndex < len(templateURI) {
			routingChar = templateURI[templateURIIndex: templateURIIndex+1]
		}
		if (!expectingValue && !expectingName) && requestChar == routingChar && routingChar != "" {
			requestURIIndex++
			templateURIIndex++
			continue
		}

		if routingChar == "}" {
			expectingName = false
			templateURIIndex++
		}

		if expectingValue && requestChar == "/" {
			expectingValue = false
		}

		if expectingName && templateURIIndex < len(templateURI) {
			name += routingChar
			templateURIIndex++
		}

		if routingChar == "{" {
			expectingValue = true
			expectingName = true
			templateURIIndex++

		}

		if expectingValue && requestURIIndex < len(requestURI) {
			value += requestChar
			requestURIIndex++
		}

		if !expectingValue && !expectingName && len(name) > 0 {
			uriParameters[name] = value
			name = ""
			value = ""
		}

	}

	if len(name) > 0 && len(value) > 0 {
		uriParameters[name] = value
	}
	matched := requestURIIndex == len(requestURI) && templateURIIndex == len(templateURI)
	return uriParameters, matched
}

func URLPathJoin(baseURL, path string) string {
	if strings.HasSuffix(baseURL, "/") {
		return baseURL + path
	}
	if ! strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	return baseURL + path
}

//URLBase returns base URL
func URLBase(URL string) string {
	parsedURL, err := url.Parse(URL)
	if err != nil || parsedURL.Path == "" {
		return URL
	}
	pathPosition := strings.Index(URL, parsedURL.Path)
	if pathPosition == -1 {
		return URL
	}
	return string(URL[:pathPosition])
}
<commit_msg>added URL split, patched URLBase<commit_after>package toolbox

import (
	"fmt"
	"net/url"
	"strings"
)

//ExtractURIParameters parses URIs to extract {<param>} defined in templateURI from requestURI, it returns extracted parameters and flag if requestURI matched templateURI
func ExtractURIParameters(templateURI, requestURI string) (map[string]string, bool) {
	var expectingValue, expectingName bool
	var name, value string
	var uriParameters = make(map[string]string)
	maxLength := len(templateURI) + len(requestURI)
	var requestURIIndex, templateURIIndex int

	questionMarkPosition := strings.Index(requestURI, "?")
	if questionMarkPosition != -1 {
		requestURI = string(requestURI[:questionMarkPosition])
	}

	for k := 0; k < maxLength; k++ {
		var requestChar, routingChar string

		if requestURIIndex < len(requestURI) {
			requestChar = requestURI[requestURIIndex : requestURIIndex+1]
		}

		if templateURIIndex < len(templateURI) {
			routingChar = templateURI[templateURIIndex : templateURIIndex+1]
		}
		if (!expectingValue && !expectingName) && requestChar == routingChar && routingChar != "" {
			requestURIIndex++
			templateURIIndex++
			continue
		}

		if routingChar == "}" {
			expectingName = false
			templateURIIndex++
		}

		if expectingValue && requestChar == "/" {
			expectingValue = false
		}

		if expectingName && templateURIIndex < len(templateURI) {
			name += routingChar
			templateURIIndex++
		}

		if routingChar == "{" {
			expectingValue = true
			expectingName = true
			templateURIIndex++

		}

		if expectingValue && requestURIIndex < len(requestURI) {
			value += requestChar
			requestURIIndex++
		}

		if !expectingValue && !expectingName && len(name) > 0 {
			uriParameters[name] = value
			name = ""
			value = ""
		}

	}

	if len(name) > 0 && len(value) > 0 {
		uriParameters[name] = value
	}
	matched := requestURIIndex == len(requestURI) && templateURIIndex == len(templateURI)
	return uriParameters, matched
}

func URLPathJoin(baseURL, path string) string {
	if strings.HasSuffix(baseURL, "/") {
		return baseURL + path
	}
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	return baseURL + path
}

//URLBase returns base URL
func URLBase(URL string) string {
	parsedURL, err := url.Parse(URL)
	if err != nil || parsedURL.Path == "" {
		return URL
	}
	pathPosition := strings.Index(URL, parsedURL.Path)
	if pathPosition == -1 {
		return URL
	}
	return string(URL[:pathPosition])
}

//URLSplit returns URL with parent path and resource name
func URLSplit(URL string) (string, string) {
	parsedURL, err := url.Parse(URL)
	if err != nil || parsedURL.Path == "" {
		return URL, ""
	}
	splitPosition := strings.LastIndex(parsedURL.Path, "/")
	if splitPosition == -1 {
		return URL, ""
	}
	return fmt.Sprintf("%v%v", URLBase(URL), string(parsedURL.Path[:splitPosition])), string(parsedURL.Path[splitPosition+1:])
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/smtp"
	"os"
	"strings"
)

var Trace *log.Logger

func getDomainFromAddress(address string) (domain string, err error) {
	at := strings.LastIndex(address, "@")
	if at < 0 {
		return "", fmt.Errorf("Invalid domain")
	}
	return address[at+1:], nil
}

func isDeliverable(host string, address string) (bool, error) {
	deliverable := false

	Trace.Printf("Connecting...")
	cli, err := smtp.Dial(host)
	if err != nil {
		log.Printf("Error on connect: %s\n", err)
		return false, err
	}
	defer cli.Close()
	Trace.Printf("Connected.")

	Trace.Printf("MAIL FROM:<%s>", address)
	err = cli.Mail(address)
	if err != nil {
		log.Printf("Error on MAIL: %s\n", err)
		return false, err
	}

	Trace.Printf("RCPT TO:<%s>", address)
	err = cli.Rcpt(address)
	if err != nil {
		log.Printf("Error on RCPT: %s\n", err)
		return false, err
	}

	// If RCPT succeeded, the server thinks the address is deliverable
	deliverable = true

	Trace.Printf("RSET")
	err = cli.Reset()
	if err != nil {
		log.Printf("Error on RSET: %s\n", err)
		return deliverable, err
	}

	Trace.Printf("QUIT")
	err = cli.Quit()
	if err != nil {
		log.Printf("Error on QUIT: %s\n", err)
		return deliverable, err
	}

	return deliverable, nil
}

func firstMxFromDomain(domain string) (mxhost string, err error) {
	mxs, err := net.LookupMX(domain)
	if err != nil {
		log.Printf("Error looking up MX record: %s\n", err)
		return "", err
	}
	if len(mxs) == 0 {
		return "", fmt.Errorf("No MX for domain")
	}

	// Return the first MX
	return mxs[0].Host, nil
}

func main() {
	// Parse command-line flags
	verbosePtr := flag.Bool("verbose", false, "Show verbose messages")
	quietPtr := flag.Bool("quiet", false, "Quiet (no output)")
	flag.Parse()

	if *verbosePtr && *quietPtr {
		log.Fatalf("Cannot be both quiet and verbose.")
	}

	// Set up verbose logging if required
	var traceDest io.Writer
	traceDest = ioutil.Discard
	if *verbosePtr {
		traceDest = os.Stderr
	}
	Trace = log.New(traceDest, "INFO: ", log.LstdFlags)

	log.SetOutput(os.Stderr)

	args := flag.Args()
	if len(args) != 1 {
		log.Fatalf("Usage: %s <address>\n", os.Args[0])
	}

	address := args[0]
	Trace.Printf("Address: %s\n", address)

	domain, err := getDomainFromAddress(address)
	if err != nil {
		log.Fatal("Error: cannot get domain from address.")
	}
	Trace.Printf("Domain: %s\n", domain)

	mxHost, err := firstMxFromDomain(domain)
	if err != nil {
		log.Fatal("Error: cannot get domain from address.")
	}
	Trace.Printf("MX host: %s\n", mxHost)

	host := fmt.Sprintf("%s:25", mxHost)

	deliverable, err := isDeliverable(host, address)
	if err != nil {
		log.Fatalf("Error checking deliverability: %s\n", err)
	}

	if deliverable {
		if !*quietPtr {
			fmt.Println(address, "is deliverable")
		}
		os.Exit(0)
	} else {
		if !*quietPtr {
			fmt.Println(address, "is not deliverable")
		}
		os.Exit(1)
	}
}
<commit_msg>distinguish "invalid recipient" from "wtf" and "ugh what"<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/smtp"
	"os"
	"regexp"
	"strings"
)

var Trace *log.Logger

func getDomainFromAddress(address string) (domain string, err error) {
	at := strings.LastIndex(address, "@")
	if at < 0 {
		return "", fmt.Errorf("Invalid domain")
	}
	return address[at+1:], nil
}

func isDeliverable(host string, address string) (bool, error) {
	deliverable := false

	Trace.Printf("Connecting...")
	cli, err := smtp.Dial(host)
	if err != nil {
		log.Printf("Error on connect: %s\n", err)
		return false, err
	}
	defer cli.Close()
	Trace.Printf("Connected.")

	Trace.Printf("MAIL FROM:<%s>", address)
	err = cli.Mail(address)
	if err != nil {
		log.Printf("Error on MAIL: %s\n", err)
		return false, err
	}

	Trace.Printf("RCPT TO:<%s>", address)
	err = cli.Rcpt(address)
	if err != nil {
		rx := regexp.MustCompile("^(451|550) [0-9]\\.1\\.1")

		// SMTP 550 X.1.1 means invalid address, but other errors mean other things
		if !rx.MatchString(err.Error()) {
			log.Printf("Error on RCPT: %s\n", err)
			return false, err
		}
		return false, nil
	}

	// If RCPT succeeded, the server thinks the address is deliverable
	deliverable = true

	Trace.Printf("RSET")
	err = cli.Reset()
	if err != nil {
		log.Printf("Error on RSET: %s\n", err)
		return deliverable, err
	}

	Trace.Printf("QUIT")
	err = cli.Quit()
	if err != nil {
		log.Printf("Error on QUIT: %s\n", err)
		return deliverable, err
	}

	return deliverable, nil
}

func firstMxFromDomain(domain string) (mxhost string, err error) {
	mxs, err := net.LookupMX(domain)
	if err != nil {
		log.Printf("Error looking up MX record: %s\n", err)
		return "", err
	}
	if len(mxs) == 0 {
		return "", fmt.Errorf("No MX for domain")
	}

	// Return the first MX
	return mxs[0].Host, nil
}

func main() {
	// Parse command-line flags
	verbosePtr := flag.Bool("verbose", false, "Show verbose messages")
	quietPtr := flag.Bool("quiet", false, "Quiet (no output)")
	flag.Parse()

	if *verbosePtr && *quietPtr {
		log.Fatalf("Cannot be both quiet and verbose.")
	}

	// Set up verbose logging if required
	var traceDest io.Writer
	traceDest = ioutil.Discard
	if *verbosePtr {
		traceDest = os.Stderr
	}
	Trace = log.New(traceDest, "INFO: ", log.LstdFlags)

	log.SetOutput(os.Stderr)

	args := flag.Args()
	if len(args) != 1 {
		log.Fatalf("Usage: %s <address>\n", os.Args[0])
	}

	address := args[0]
	Trace.Printf("Address: %s\n", address)

	domain, err := getDomainFromAddress(address)
	if err != nil {
		log.Fatal("Error: cannot get domain from address.")
	}
	Trace.Printf("Domain: %s\n", domain)

	mxHost, err := firstMxFromDomain(domain)
	if err != nil {
		log.Fatal("Error: cannot get domain from address.")
	}
	Trace.Printf("MX host: %s\n", mxHost)

	host := fmt.Sprintf("%s:25", mxHost)

	deliverable, err := isDeliverable(host, address)
	if err != nil {
		log.Fatalf("Error checking deliverability: %s\n", err)
	}

	if deliverable {
		if !*quietPtr {
			fmt.Println(address, "is deliverable")
		}
		os.Exit(0)
	} else {
		if !*quietPtr {
			fmt.Println(address, "is not deliverable")
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// # wallaby - Write Ahead Log
//
// This file contains all of the errors, constants and entry points for wallaby.
//
package wallaby

import (
    "bytes"
    "errors"
    "os"
    "sync"
    "time"

    xxhash "github.com/OneOfOne/xxhash/native"
    "github.com/eliquious/xbinary"
)

// ## **Possible Log Errors**

var (
    // - `ErrReadIndexHeader` occurs when the index header cannot be read
    ErrReadIndexHeader = errors.New("failed to read index header")

    // - `ErrWriteIndexHeader` occurs when the index header cannot be written
    ErrWriteIndexHeader = errors.New("failed to write index header")

    // - `ErrReadLogHeader` occurs when the log header cannot be read
    ErrReadLogHeader = errors.New("failed to read log header")

    // - `ErrWriteLogHeader` occurs when the log header cannot be written
    ErrWriteLogHeader = errors.New("failed to write log header")

    // - `ErrSliceOutOfBounds` occurs when index.Slice is called and the offset
    // is larger than the size of the index.
    ErrSliceOutOfBounds = errors.New("read offset out of bounds")

    // - `ErrReadIndex` occurs when index.Slice fails to read the records
    ErrReadIndex = errors.New("failed to read index records")

    // - `ErrConfigRequired` occurs when no log config is given when creating
    // or opening a log file.
    ErrConfigRequired = errors.New("log config required")

    // - `ErrInvalidFileVersion` occurs when the version in the file header
    // is unrecognized.
    ErrInvalidFileVersion = errors.New("invalid file version")

    // - `ErrInvalidFileSignature` occurs when the signature in the file header
    // is unrecognized.
    ErrInvalidFileSignature = errors.New("invalid file signature")

    // - `ErrWriteLogRecord` occurs when a record fails to be written to the log
    ErrWriteLogRecord = errors.New("failed to write record")

    // - `ErrLogAlreadyOpen` occurs when an open log tries to be opened again
    ErrLogAlreadyOpen = errors.New("log already open")

    // - `ErrLogClosed` occurs when `Append` is called after the log has been
    // closed.
    ErrLogClosed = errors.New("log has been closed")

    // `ErrRecordTooLarge` occurs when writing a record which exceed the max
    // record size for the log.
    ErrRecordTooLarge = errors.New("record is too large")

    // ErrInvalidRecordSize
    ErrInvalidRecordSize = errors.New("invalid record size")

    // ErrInvalidSnapshot occurs when a Snapshot cannot be decoded.
    ErrInvalidSnapshot = errors.New("invalid snapshot")

    // ## **Log Variables**

    // Log signature bytes - `LOG`
    LogFileSignature = []byte("LOG")

    // Index signature bytes - `IDX`
    IndexFileSignature = []byte("IDX")
)

// ## **Log Constants**

const (
    // - `DefaultIndexFlags` is the default boolean flags for an index file.
    DefaultIndexFlags = 0

    // - `DefaultRecordFlags` represents the default boolean flags for each log record.
    DefaultRecordFlags uint32 = 0

    // - `DefaultMaxRecordSize` is the default maximum size of a log record.
    DefaultMaxRecordSize = 0xffff

    // - `LogHeaderSize` is the size of the file header.
    LogHeaderSize = 8

    // - `VersionOne` is an integer denoting the first version
    VersionOne = 1

    // - `VersionOneIndexHeaderSize` is the size of the index file header.
    VersionOneIndexHeaderSize = 8

    // - `VersionOneIndexRecordSize` is the size of the index records.
    VersionOneIndexRecordSize = 32

    // - `VersionOneLogHeaderSize` is the header size of version 1 log files
    VersionOneLogHeaderSize = 8

    // - `VersionOneLogRecordHeaderSize` is the size of the log record headers.
    VersionOneLogRecordHeaderSize = 24

    // - `MaximumIndexSlice` is the maximum number of index records to be read at
    // one time
    MaximumIndexSlice = 32000
)

// ## **Metadata**
// Metadata simply contains descriptive information about the log
type Metadata struct {
    Size             int64
    LastModifiedTime int64
    FileName         string
    IndexFileName    string
}

// ## **Config**
// Config stores several log settings. This is used to describe how the log
// should be opened.
type Config struct {
    FileMode      os.FileMode
    MaxRecordSize int
    Flags         uint32
    Version       uint8
}

// > `DefaultConfig` can be used for sensible default log configuration.
var DefaultConfig Config = Config{
    FileMode:      0600,
    MaxRecordSize: DefaultMaxRecordSize,
    Flags:         DefaultRecordFlags,
    Version:       VersionOne,
}

// ## **Create a log file**
// Create returns a `WriteAheadLog` implementation if no errors occur. If the
// given filename already exists, the log file will try to be opened. If the
// file format can be verified, the existing log will be returned. If the
// file does not exist, a new log will be created with the given config.
//
// If the file already exists and the file version is different than the given
// `config.Version`, the file will remain the version in which it was created.
// In other words the file will not be updated to the newer version.

// ###### Implementation
func Create(filename string, config Config) (WriteAheadLog, error) {

    // Determine if the given config is valid. If the given config is `nil`,
    // a `ErrConfigRequired` error will be returned.
    if &config == nil {
        return nil, ErrConfigRequired
    }

    // Open the file name, creating the file if it does not already exist. The
    // file is opened with the `APPEND` flag, which means all writes are
    // appended to the file. Additional file modes can be given with the config.
    file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, config.FileMode)

    // If there was an error opening the file, the open error is returned.
    if err != nil {
        return nil, err
    }

    // Get the file stat. The file size is gotten from this call. This helps
    // to determine if the file already has a header or not.
    stat, err := file.Stat()

    // Return an error if the `os.FileStat` could not be retrieved. The file
    // is closed before returning.
    if err != nil {
        file.Close()
        return nil, err
    }

    // If the file size suggests the header exists, open an existing file.
    // Otherwise create a new file based on the given config.
    if stat.Size() >= LogHeaderSize {
        return openExisting(file, filename, config)
    }
    return createNew(file, filename, config)
}

// ## **Utility functions**
// These functions assist in opening both new and existing log files.

// ### **Creates a v1 log file**
// This function is used when creating a v1 log file.

// ###### *Implementation*
func createVersionOne(file *os.File, filename string, config Config) (WriteAheadLog, error) {

    // Create the file header structure from the log flags and version.
    header := BasicFileHeader{flags: config.Flags, version: config.Version}

    // Create the index file. If the file cannot be created, close the file
    // and return the error.
    index, err := VersionOneIndexFactory(filename+".idx", config.Version, config.Flags)
    if err != nil {
        file.Close()
        return nil, err
    }

    // Create the record factory. All records will be created using this
    // factory.
    recordFactory := BasicLogRecordFactory(config.MaxRecordSize)

    // Stat the file to get the size. If unsuccessful, close the file and return
    // the error.
    stat, err := file.Stat()
    if err != nil {
        file.Close()
        return nil, err
    }

    // Create a lock for the log and wrap the file in an atomic writer. The
    // atomic writer syncs to disk after each write.
    var lock sync.Mutex
    atomicWriter := NewAtomicWriter(file)

    // Finally, create the log file and return.
    return &versionOneLogFile{lock, file, atomicWriter, &header, index,
        recordFactory, config.Flags, stat.Size(), CLOSED, xxhash.New64(), time.Now().UnixNano()}, nil
}

// ### **Creates a new log file**
// A new log file is created with a file header consisting of a `LOG` signature
// followed by an 8-bit version. The file header is then synced to disk and
// a new log is created.

// ###### Implentation
func createNew(file *os.File, filename string, config Config) (WriteAheadLog, error) {

    // Create a buffer for the 4-byte file header.
    // The first 3 bytes contain the signature, `LOG`, followed by an 8-bit
    // version.
    buf := make([]byte, 8)

    // Write the `LOG` file signature to the first 3 bytes of the file.
    xbinary.LittleEndian.PutString(buf, 0, string(LogFileSignature))

    // Set file version to the given `config.Version`.
    buf[3] = byte(config.Version)

    // Write the config flags into the buffer
    xbinary.LittleEndian.PutUint32(buf, 4, config.Flags)

    // Write the file header buffer to the file.
    _, err := file.Write(buf)

    // If the header could not be written, close the file and return a
    // `ErrWriteLogHeader` error along with a `nil` log.
    if err != nil {
        file.Close()
        return nil, ErrWriteLogHeader
    }

    // If writing the file header succeeded, sync the file header to disk.
    err = file.Sync()

    // If the sync command failed, return a `ErrWriteLogHeader` error and a
    // `nil` log.
    if err != nil {
        return nil, ErrWriteLogHeader
    }

    // Returns the proper log parser based on the given `config.Version`.
    return selectVersion(file, filename, config)
}

// ### **Opens an existing log file**
// Opens an existing file and returns a log based on the file header. If the
// file contains a version which is not understood, the error
// `ErrInvalidFileVersion` is returned along with a `nil` log.
//
// If the file header cannot be read, an error is also returned.

// ###### Implementation
func openExisting(file *os.File, filename string, config Config) (WriteAheadLog, error) {
    // Create a buffer for the 8-byte file header.
    // The first 3 bytes are the signature `LOG` followed by an 8-bit version
    // and the boolean flags. Then read the file header into the buffer.
    buf := make([]byte, 8)
    _, err := file.ReadAt(buf, 0)

    // If there was an error reading the file header, close the file and return
    // a nil log and the read error.
    if err != nil {
        file.Close()
        return nil, err
    }

    // If the header was read sucessfully, verify the file signature matches
    // the expected "LOG" signature. If the first 3 bytes do not match `LOG`,
    // return a `nil` log and a `ErrInvalidFileSignature`.
    if !bytes.Equal(buf[0:3], LogFileSignature) {
        return nil, ErrInvalidFileSignature
    }

    // Read the boolean flags from the file header and overwrite the config
    // flags with the ones from the file.
    flags, err := xbinary.LittleEndian.Uint32(buf, 4)
    if err != nil {
        return nil, err
    }
    config.Flags = flags

    // The config version is updated to reflect the actual version of the file.
    // Then return the proper log parser based on the file version.
    config.Version = uint8(buf[3])
    return selectVersion(file, filename, config)
}

// ### **Select log version**
// `selectVersion` is only here to make the code a bit `DRY`er. It simple
// returns the proper log file based on the given version.

// ###### Implementation
// Open the log file based on the current version of the file.
// If the version is unrecognized, a `nil` log is returned as well as an
// `ErrInvalidFileVersion` error.
func selectVersion(file *os.File, filename string, config Config) (WriteAheadLog, error) {
    switch config.Version {
    case VersionOne:
        return createVersionOne(file, filename, config)
    default:
        return nil, ErrInvalidFileVersion
    }
}
<commit_msg>Add log truncate ability to config<commit_after>// # wallaby - Write Ahead Log
//
// This file contains all of the errors, constants and entry points for wallaby.
//
package wallaby

import (
    "bytes"
    "errors"
    "os"
    "sync"
    "time"

    xxhash "github.com/OneOfOne/xxhash/native"
    "github.com/eliquious/xbinary"
)

// ## **Possible Log Errors**

var (
    // - `ErrReadIndexHeader` occurs when the index header cannot be read
    ErrReadIndexHeader = errors.New("failed to read index header")

    // - `ErrWriteIndexHeader` occurs when the index header cannot be written
    ErrWriteIndexHeader = errors.New("failed to write index header")

    // - `ErrReadLogHeader` occurs when the log header cannot be read
    ErrReadLogHeader = errors.New("failed to read log header")

    // - `ErrWriteLogHeader` occurs when the log header cannot be written
    ErrWriteLogHeader = errors.New("failed to write log header")

    // - `ErrSliceOutOfBounds` occurs when index.Slice is called and the offset
    // is larger than the size of the index.
    ErrSliceOutOfBounds = errors.New("read offset out of bounds")

    // - `ErrReadIndex` occurs when index.Slice fails to read the records
    ErrReadIndex = errors.New("failed to read index records")

    // - `ErrConfigRequired` occurs when no log config is given when creating
    // or opening a log file.
    ErrConfigRequired = errors.New("log config required")

    // - `ErrInvalidFileVersion` occurs when the version in the file header
    // is unrecognized.
    ErrInvalidFileVersion = errors.New("invalid file version")

    // - `ErrInvalidFileSignature` occurs when the signature in the file header
    // is unrecognized.
    ErrInvalidFileSignature = errors.New("invalid file signature")

    // - `ErrWriteLogRecord` occurs when a record fails to be written to the log
    ErrWriteLogRecord = errors.New("failed to write record")

    // - `ErrLogAlreadyOpen` occurs when an open log tries to be opened again
    ErrLogAlreadyOpen = errors.New("log already open")

    // - `ErrLogClosed` occurs when `Append` is called after the log has been
    // closed.
    ErrLogClosed = errors.New("log has been closed")

    // `ErrRecordTooLarge` occurs when writing a record which exceed the max
    // record size for the log.
    ErrRecordTooLarge = errors.New("record is too large")

    // ErrInvalidRecordSize
    ErrInvalidRecordSize = errors.New("invalid record size")

    // ErrInvalidSnapshot occurs when a Snapshot cannot be decoded.
    ErrInvalidSnapshot = errors.New("invalid snapshot")

    // ## **Log Variables**

    // Log signature bytes - `LOG`
    LogFileSignature = []byte("LOG")

    // Index signature bytes - `IDX`
    IndexFileSignature = []byte("IDX")
)

// ## **Log Constants**

const (
    // - `DefaultIndexFlags` is the default boolean flags for an index file.
    DefaultIndexFlags = 0

    // - `DefaultRecordFlags` represents the default boolean flags for each log record.
    DefaultRecordFlags uint32 = 0

    // - `DefaultMaxRecordSize` is the default maximum size of a log record.
    DefaultMaxRecordSize = 0xffff

    // - `LogHeaderSize` is the size of the file header.
    LogHeaderSize = 8

    // - `VersionOne` is an integer denoting the first version
    VersionOne = 1

    // - `VersionOneIndexHeaderSize` is the size of the index file header.
    VersionOneIndexHeaderSize = 8

    // - `VersionOneIndexRecordSize` is the size of the index records.
    VersionOneIndexRecordSize = 32

    // - `VersionOneLogHeaderSize` is the header size of version 1 log files
    VersionOneLogHeaderSize = 8

    // - `VersionOneLogRecordHeaderSize` is the size of the log record headers.
    VersionOneLogRecordHeaderSize = 24

    // - `MaximumIndexSlice` is the maximum number of index records to be read at
    // one time
    MaximumIndexSlice = 32000
)

// ## **Metadata**
// Metadata simply contains descriptive information about the log
type Metadata struct {
    Size             int64
    LastModifiedTime int64
    FileName         string
    IndexFileName    string
}

// ## **Config**
// Config stores several log settings. This is used to describe how the log
// should be opened.
type Config struct {
    FileMode      os.FileMode
    MaxRecordSize int
    Flags         uint32
    Version       uint8
    Truncate      bool
}

// > `DefaultConfig` can be used for sensible default log configuration.
var DefaultConfig Config = Config{
    FileMode:      0600,
    MaxRecordSize: DefaultMaxRecordSize,
    Flags:         DefaultRecordFlags,
    Version:       VersionOne,
    Truncate:      false,
}

// ## **Create a log file**
// Create returns a `WriteAheadLog` implementation if no errors occur. If the
// given filename already exists, the log file will try to be opened. If the
// file format can be verified, the existing log will be returned. If the
// file does not exist, a new log will be created with the given config.
//
// If the file already exists and the file version is different than the given
// `config.Version`, the file will remain the version in which it was created.
// In other words the file will not be updated to the newer version.

// ###### Implementation
func Create(filename string, config Config) (WriteAheadLog, error) {

    // Determine if the given config is valid. If the given config is `nil`,
    // a `ErrConfigRequired` error will be returned.
    if &config == nil {
        return nil, ErrConfigRequired
    }

    // Open the file name, creating the file if it does not already exist. The
    // file is opened with the `APPEND` flag, which means all writes are
    // appended to the file. Additional file modes can be given with the config.
    file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, config.FileMode)

    // If there was an error opening the file, the open error is returned.
    if err != nil {
        return nil, err
    }

    // Truncate the log file if requested in the given config
    if config.Truncate {
        err = file.Truncate(0)
        if err != nil {
            file.Close()
            return nil, err
        }
    }

    // Get the file stat. The file size is gotten from this call. This helps
    // to determine if the file already has a header or not.
    stat, err := file.Stat()

    // Return an error if the `os.FileStat` could not be retrieved. The file
    // is closed before returning.
    if err != nil {
        file.Close()
        return nil, err
    }

    // If the file size suggests the header exists, open an existing file.
    // Otherwise create a new file based on the given config.
    if stat.Size() >= LogHeaderSize {
        return openExisting(file, filename, config)
    }
    return createNew(file, filename, config)
}

// ## **Utility functions**
// These functions assist in opening both new and existing log files.

// ### **Creates a v1 log file**
// This function is used when creating a v1 log file.

// ###### *Implementation*
func createVersionOne(file *os.File, filename string, config Config) (WriteAheadLog, error) {

    // Create the file header structure from the log flags and version.
    header := BasicFileHeader{flags: config.Flags, version: config.Version}

    // Create the index file. If the file cannot be created, close the file
    // and return the error.
    index, err := VersionOneIndexFactory(filename+".idx", config.Version, config.Flags)
    if err != nil {
        file.Close()
        return nil, err
    }

    // Create the record factory. All records will be created using this
    // factory.
    recordFactory := BasicLogRecordFactory(config.MaxRecordSize)

    // Stat the file to get the size. If unsuccessful, close the file and return
    // the error.
    stat, err := file.Stat()
    if err != nil {
        file.Close()
        return nil, err
    }

    // Create a lock for the log and wrap the file in an atomic writer. The
    // atomic writer syncs to disk after each write.
    var lock sync.Mutex
    atomicWriter := NewAtomicWriter(file)

    // Finally, create the log file and return.
    return &versionOneLogFile{lock, file, atomicWriter, &header, index,
        recordFactory, config.Flags, stat.Size(), CLOSED, xxhash.New64(), time.Now().UnixNano()}, nil
}

// ### **Creates a new log file**
// A new log file is created with a file header consisting of a `LOG` signature
// followed by an 8-bit version. The file header is then synced to disk and
// a new log is created.

// ###### Implentation
func createNew(file *os.File, filename string, config Config) (WriteAheadLog, error) {

    // Create a buffer for the 4-byte file header.
    // The first 3 bytes contain the signature, `LOG`, followed by an 8-bit
    // version.
    buf := make([]byte, 8)

    // Write the `LOG` file signature to the first 3 bytes of the file.
    xbinary.LittleEndian.PutString(buf, 0, string(LogFileSignature))

    // Set file version to the given `config.Version`.
    buf[3] = byte(config.Version)

    // Write the config flags into the buffer
    xbinary.LittleEndian.PutUint32(buf, 4, config.Flags)

    // Write the file header buffer to the file.
    _, err := file.Write(buf)

    // If the header could not be written, close the file and return a
    // `ErrWriteLogHeader` error along with a `nil` log.
    if err != nil {
        file.Close()
        return nil, ErrWriteLogHeader
    }

    // If writing the file header succeeded, sync the file header to disk.
    err = file.Sync()

    // If the sync command failed, return a `ErrWriteLogHeader` error and a
    // `nil` log.
    if err != nil {
        return nil, ErrWriteLogHeader
    }

    // Returns the proper log parser based on the given `config.Version`.
    return selectVersion(file, filename, config)
}

// ### **Opens an existing log file**
// Opens an existing file and returns a log based on the file header. If the
// file contains a version which is not understood, the error
// `ErrInvalidFileVersion` is returned along with a `nil` log.
//
// If the file header cannot be read, an error is also returned.

// ###### Implementation
func openExisting(file *os.File, filename string, config Config) (WriteAheadLog, error) {
    // Create a buffer for the 8-byte file header.
    // The first 3 bytes are the signature `LOG` followed by an 8-bit version
    // and the boolean flags. Then read the file header into the buffer.
    buf := make([]byte, 8)
    _, err := file.ReadAt(buf, 0)

    // If there was an error reading the file header, close the file and return
    // a nil log and the read error.
    if err != nil {
        file.Close()
        return nil, err
    }

    // If the header was read sucessfully, verify the file signature matches
    // the expected "LOG" signature. If the first 3 bytes do not match `LOG`,
    // return a `nil` log and a `ErrInvalidFileSignature`.
    if !bytes.Equal(buf[0:3], LogFileSignature) {
        return nil, ErrInvalidFileSignature
    }

    // Read the boolean flags from the file header and overwrite the config
    // flags with the ones from the file.
    flags, err := xbinary.LittleEndian.Uint32(buf, 4)
    if err != nil {
        return nil, err
    }
    config.Flags = flags

    // The config version is updated to reflect the actual version of the file.
    // Then return the proper log parser based on the file version.
    config.Version = uint8(buf[3])
    return selectVersion(file, filename, config)
}

// ### **Select log version**
// `selectVersion` is only here to make the code a bit `DRY`er. It simple
// returns the proper log file based on the given version.

// ###### Implementation
// Open the log file based on the current version of the file.
// If the version is unrecognized, a `nil` log is returned as well as an
// `ErrInvalidFileVersion` error.
func selectVersion(file *os.File, filename string, config Config) (WriteAheadLog, error) {
    switch config.Version {
    case VersionOne:
        return createVersionOne(file, filename, config)
    default:
        return nil, ErrInvalidFileVersion
    }
}
<|endoftext|>


<commit_before>// Package zlmgo provides Go bindings to the Zen License Manager (ZLM).
package zlmgo

/*
#cgo CFLAGS: -I/usr/local/include
#cgo LDFLAGS: /usr/local/lib/libzlm.a

#include <zlm.h>

char zlmgo_errbuf_array[ZLM_ERRBUF];
char* zlmgo_errbuf = zlmgo_errbuf_array; // hack around cgo warning
*/
import "C"

import (
	"errors"
	"runtime"
)

type License struct {
	l *C.ZlmLicense
}

// LicenseNew returns a new license object (panics if not enough memory is available).
func LicenseNew() *License {
	license := &License{C.zlm_license_new(C.zlmgo_errbuf)}
	if license.l == nil {
		panic(C.GoString(C.zlmgo_errbuf))
	}
	runtime.SetFinalizer(license, (*License).free)
	return license
}

func (license *License) Get(product, version, argv0, path, license_string string) error {
	var c_product, c_version, c_argv0, c_path, c_license_string *C.char
	// convert argument to C strings
	if product != "" {
		c_product = C.CString(product)
	}
	if version != "" {
		c_version = C.CString(version)
	}
	if argv0 != "" {
		c_argv0 = C.CString(argv0)
	}
	if path != "" {
		c_path = C.CString(path)
	}
	if license_string != "" {
		c_license_string = C.CString(license_string)
	}
	// call actual method
	if C.zlm_license_get(license.l, c_product, c_version, c_argv0, c_path, c_license_string, C.zlmgo_errbuf) != C.ZLM_OK {
		return errors.New(C.GoString(C.zlmgo_errbuf))
	}
	return nil
}

func (license *License) free() {
	C.zlm_license_free(license.l)
}
<commit_msg>use Go naming conventions<commit_after>// Package zlmgo provides Go bindings to the Zen License Manager (ZLM).
package zlmgo

/*
#cgo CFLAGS: -I/usr/local/include
#cgo LDFLAGS: /usr/local/lib/libzlm.a

#include <zlm.h>

char zlmgo_errbuf_array[ZLM_ERRBUF];
char* zlmgo_errbuf = zlmgo_errbuf_array; // hack around cgo warning
*/
import "C"

import (
	"errors"
	"runtime"
)

type License struct {
	l *C.ZlmLicense
}

// LicenseNew returns a new license object (panics if not enough memory is available).
func LicenseNew() *License {
	license := &License{C.zlm_license_new(C.zlmgo_errbuf)}
	if license.l == nil {
		panic(C.GoString(C.zlmgo_errbuf))
	}
	runtime.SetFinalizer(license, (*License).free)
	return license
}

func (license *License) Get(product, version, argv0, path, licenseString string) error {
	var c_product, c_version, c_argv0, c_path, c_license_string *C.char
	// convert argument to C strings
	if product != "" {
		c_product = C.CString(product)
	}
	if version != "" {
		c_version = C.CString(version)
	}
	if argv0 != "" {
		c_argv0 = C.CString(argv0)
	}
	if path != "" {
		c_path = C.CString(path)
	}
	if licenseString != "" {
		c_license_string = C.CString(licenseString)
	}
	// call actual method
	if C.zlm_license_get(license.l, c_product, c_version, c_argv0, c_path, c_license_string, C.zlmgo_errbuf) != C.ZLM_OK {
		return errors.New(C.GoString(C.zlmgo_errbuf))
	}
	return nil
}

func (license *License) free() {
	C.zlm_license_free(license.l)
}
<|endoftext|>


<commit_before>package main_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"testing"
	"time"

	"github.com/influxdb/influxdb"
	"github.com/influxdb/influxdb/client"
	"github.com/influxdb/influxdb/influxql"
	"github.com/influxdb/influxdb/messaging"

	main "github.com/influxdb/influxdb/cmd/influxd"
)

type node struct {
	broker *messaging.Broker
	server *influxdb.Server
}

// createCombinedNodeCluster creates a cluster of nServers nodes, each of which
// runs as both a Broker and Data node. If any part cluster creation fails,
// the testing is marked as failed.
//
// This function returns a slice of nodes, the first of which will be the leader.
func createCombinedNodeCluster(t *testing.T, testName string, nNodes, basePort int) []node {
	t.Logf("Creating cluster of %d nodes for test %s", nNodes, testName)
	if nNodes < 1 {
		t.Fatalf("Test %s: asked to create nonsense cluster", testName)
	}

	nodes := make([]node, 0)

	tmpDir := os.TempDir()
	tmpBrokerDir := filepath.Join(tmpDir, "broker-integration-test")
	tmpDataDir := filepath.Join(tmpDir, "data-integration-test")
	t.Logf("Test %s: using tmp directory %q for brokers\n", testName, tmpBrokerDir)
	t.Logf("Test %s: using tmp directory %q for data nodes\n", testName, tmpDataDir)
	// Sometimes if a test fails, it's because of a log.Fatal() in the program.
	// This prevents the defer from cleaning up directories.
	// To be safe, nuke them always before starting
	_ = os.RemoveAll(tmpBrokerDir)
	_ = os.RemoveAll(tmpDataDir)

	// Create the first node, special case.
	c := main.NewConfig()
	c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(basePort))
	c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(basePort))
	c.Broker.Port = basePort
	c.Data.Port = basePort

	b, s := main.Run(c, "", "x.x", os.Stderr)
	if b == nil {
		t.Fatalf("Test %s: failed to create broker on port %d", testName, basePort)
	}
	if s == nil {
		t.Fatalf("Test %s: failed to create leader data node on port %d", testName, basePort)
	}
	nodes = append(nodes, node{broker: b, server: s})

	// Create subsequent nodes, which join to first node.
	for i := 1; i < nNodes; i++ {
		nextPort := basePort + i
		c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(nextPort))
		c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(nextPort))
		c.Broker.Port = nextPort
		c.Data.Port = nextPort

		b, s := main.Run(c, "http://localhost:"+strconv.Itoa(basePort), "x.x", os.Stderr)
		if b == nil {
			t.Fatalf("Test %s: failed to create following broker on port %d", testName, basePort)
		}
		if s == nil {
			t.Fatalf("Test %s: failed to create following data node on port %d", testName, basePort)
		}
		nodes = append(nodes, node{broker: b, server: s})
	}

	return nodes
}

// simpleWriteAndQuery creates a simple database, retention policy, and replicates
// the data across all nodes. It then ensures a series of writes and queries are OK.
func simpleWriteAndQuery(t *testing.T, testname string, serverURL *url.URL, nNodes int) {
	now := time.Now().UTC()

	// Create a database
	t.Log("Creating database")
	u := urlFor(serverURL, "query", url.Values{"q": []string{"CREATE DATABASE foo"}})
	resp, err := http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create database: %s", err)
	}
	defer resp.Body.Close()

	var results client.Results
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create database failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}

	// Query the database exists
	u = urlFor(serverURL, "query", url.Values{"q": []string{"SHOW DATABASES"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("show databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults := client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				influxql.Row{
					Columns: []string{"name"},
					Values:  [][]interface{}{{"foo"}},
				},
			}},
		},
	}
	if !reflect.DeepEqual(results, expectedResults) {
		t.Fatalf("show databases failed.  Unexpected results.  expected: %+v, actual %+v", expectedResults, results)
	}

	// Create a retention policy
	t.Log("Creating retention policy")
	replication := fmt.Sprintf("CREATE RETENTION POLICY bar ON foo DURATION 1h REPLICATION %d DEFAULT", nNodes)
	u = urlFor(serverURL, "query", url.Values{"q": []string{replication}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create retention policy: %s", err)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create retention policy failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create retention policy failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}

	// TODO corylanou: Query the retention policy exists

	// Write Data
	t.Log("Write data")
	u = urlFor(serverURL, "write", url.Values{})

	buf := []byte(fmt.Sprintf(`{"database" : "foo", "retentionPolicy" : "bar", "points": [{"name": "cpu", "tags": {"host": "server01"},"timestamp": %d, "precision":"n","values": {"value": 100}}]}`, now.UnixNano()))
	t.Logf("Writing raw data: %s", string(buf))
	resp, err = http.Post(u.String(), "application/json", bytes.NewReader(buf))
	if err != nil {
		t.Fatalf("Couldn't write data: %s", err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Write to database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	// Need some time for server to get consensus and write data
	// TODO corylanou query the status endpoint for the server and wait for the index to update to know the write was applied
	time.Sleep(time.Duration(nNodes) * time.Second)

	// Query the data exists
	t.Log("Query data")
	u = urlFor(serverURL, "query", url.Values{"q": []string{`select value from "foo"."bar".cpu`}, "db": []string{"foo"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Couldn't read body of response: %s", err)
	}
	t.Logf("resp.Body: %s\n", string(body))

	dec := json.NewDecoder(bytes.NewReader(body))
	dec.UseNumber()
	err = dec.Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("query databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults = client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				{
					Name:    "cpu",
					Columns: []string{"time", "value"},
					Values: [][]interface{}{
						[]interface{}{now.Format(time.RFC3339Nano), json.Number("100")},
					},
				}}},
		},
	}

	if !reflect.DeepEqual(results, expectedResults) {
		t.Logf("Expected:\n")
		t.Logf("%#v\n", expectedResults)
		t.Logf("Actual:\n")
		t.Logf("%#v\n", results)
		t.Fatalf("query databases failed.  Unexpected results.")
	}
}

func Test_ServerSingleIntegration(t *testing.T) {
	nNodes := 1
	createCombinedNodeCluster(t, "single node", nNodes, 8090)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:8090",
	}
	simpleWriteAndQuery(t, "single node", serverURL, nNodes)
}

func Test_Server3NodeIntegration(t *testing.T) {
	nNodes := 3
	createCombinedNodeCluster(t, "3 node", nNodes, 8090)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:8090",
	}
	simpleWriteAndQuery(t, "3 node", serverURL, nNodes)
}

func Test_Server5NodeIntegration(t *testing.T) {
	nNodes := 5
	createCombinedNodeCluster(t, "5 node", nNodes, 8090)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:8090",
	}
	simpleWriteAndQuery(t, "5 node", serverURL, nNodes)
}

func urlFor(u *url.URL, path string, params url.Values) *url.URL {
	u.Path = path
	u.RawQuery = params.Encode()
	return u
}
<commit_msg>Use different port ranges for each cluster<commit_after>package main_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"testing"
	"time"

	"github.com/influxdb/influxdb"
	"github.com/influxdb/influxdb/client"
	"github.com/influxdb/influxdb/influxql"
	"github.com/influxdb/influxdb/messaging"

	main "github.com/influxdb/influxdb/cmd/influxd"
)

type node struct {
	broker *messaging.Broker
	server *influxdb.Server
}

// createCombinedNodeCluster creates a cluster of nServers nodes, each of which
// runs as both a Broker and Data node. If any part cluster creation fails,
// the testing is marked as failed.
//
// This function returns a slice of nodes, the first of which will be the leader.
func createCombinedNodeCluster(t *testing.T, testName string, nNodes, basePort int) []node {
	t.Logf("Creating cluster of %d nodes for test %s", nNodes, testName)
	if nNodes < 1 {
		t.Fatalf("Test %s: asked to create nonsense cluster", testName)
	}

	nodes := make([]node, 0)

	tmpDir := os.TempDir()
	tmpBrokerDir := filepath.Join(tmpDir, "broker-integration-test")
	tmpDataDir := filepath.Join(tmpDir, "data-integration-test")
	t.Logf("Test %s: using tmp directory %q for brokers\n", testName, tmpBrokerDir)
	t.Logf("Test %s: using tmp directory %q for data nodes\n", testName, tmpDataDir)
	// Sometimes if a test fails, it's because of a log.Fatal() in the program.
	// This prevents the defer from cleaning up directories.
	// To be safe, nuke them always before starting
	_ = os.RemoveAll(tmpBrokerDir)
	_ = os.RemoveAll(tmpDataDir)

	// Create the first node, special case.
	c := main.NewConfig()
	c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(basePort))
	c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(basePort))
	c.Broker.Port = basePort
	c.Data.Port = basePort

	b, s := main.Run(c, "", "x.x", os.Stderr)
	if b == nil {
		t.Fatalf("Test %s: failed to create broker on port %d", testName, basePort)
	}
	if s == nil {
		t.Fatalf("Test %s: failed to create leader data node on port %d", testName, basePort)
	}
	nodes = append(nodes, node{broker: b, server: s})

	// Create subsequent nodes, which join to first node.
	for i := 1; i < nNodes; i++ {
		nextPort := basePort + i
		c.Broker.Dir = filepath.Join(tmpBrokerDir, strconv.Itoa(nextPort))
		c.Data.Dir = filepath.Join(tmpDataDir, strconv.Itoa(nextPort))
		c.Broker.Port = nextPort
		c.Data.Port = nextPort

		b, s := main.Run(c, "http://localhost:"+strconv.Itoa(basePort), "x.x", os.Stderr)
		if b == nil {
			t.Fatalf("Test %s: failed to create following broker on port %d", testName, basePort)
		}
		if s == nil {
			t.Fatalf("Test %s: failed to create following data node on port %d", testName, basePort)
		}
		nodes = append(nodes, node{broker: b, server: s})
	}

	return nodes
}

// simpleWriteAndQuery creates a simple database, retention policy, and replicates
// the data across all nodes. It then ensures a series of writes and queries are OK.
func simpleWriteAndQuery(t *testing.T, testname string, serverURL *url.URL, nNodes int) {
	now := time.Now().UTC()

	// Create a database
	t.Log("Creating database")
	u := urlFor(serverURL, "query", url.Values{"q": []string{"CREATE DATABASE foo"}})
	resp, err := http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create database: %s", err)
	}
	defer resp.Body.Close()

	var results client.Results
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create database failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}

	// Query the database exists
	u = urlFor(serverURL, "query", url.Values{"q": []string{"SHOW DATABASES"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("show databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults := client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				influxql.Row{
					Columns: []string{"name"},
					Values:  [][]interface{}{{"foo"}},
				},
			}},
		},
	}
	if !reflect.DeepEqual(results, expectedResults) {
		t.Fatalf("show databases failed.  Unexpected results.  expected: %+v, actual %+v", expectedResults, results)
	}

	// Create a retention policy
	t.Log("Creating retention policy")
	replication := fmt.Sprintf("CREATE RETENTION POLICY bar ON foo DURATION 1h REPLICATION %d DEFAULT", nNodes)
	u = urlFor(serverURL, "query", url.Values{"q": []string{replication}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't create retention policy: %s", err)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Create retention policy failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	if len(results.Results) != 1 {
		t.Fatalf("Create retention policy failed.  Unexpected results length.  expected: %d, actual %d", 1, len(results.Results))
	}

	// TODO corylanou: Query the retention policy exists

	// Write Data
	t.Log("Write data")
	u = urlFor(serverURL, "write", url.Values{})

	buf := []byte(fmt.Sprintf(`{"database" : "foo", "retentionPolicy" : "bar", "points": [{"name": "cpu", "tags": {"host": "server01"},"timestamp": %d, "precision":"n","values": {"value": 100}}]}`, now.UnixNano()))
	t.Logf("Writing raw data: %s", string(buf))
	resp, err = http.Post(u.String(), "application/json", bytes.NewReader(buf))
	if err != nil {
		t.Fatalf("Couldn't write data: %s", err)
	}
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("Write to database failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	// Need some time for server to get consensus and write data
	// TODO corylanou query the status endpoint for the server and wait for the index to update to know the write was applied
	time.Sleep(time.Duration(nNodes) * time.Second)

	// Query the data exists
	t.Log("Query data")
	u = urlFor(serverURL, "query", url.Values{"q": []string{`select value from "foo"."bar".cpu`}, "db": []string{"foo"}})
	resp, err = http.Get(u.String())
	if err != nil {
		t.Fatalf("Couldn't query databases: %s", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatalf("Couldn't read body of response: %s", err)
	}
	t.Logf("resp.Body: %s\n", string(body))

	dec := json.NewDecoder(bytes.NewReader(body))
	dec.UseNumber()
	err = dec.Decode(&results)
	if err != nil {
		t.Fatalf("Couldn't decode results: %v", err)
	}

	if results.Error() != nil {
		t.Logf("results.Error(): %q", results.Error().Error())
	}

	if resp.StatusCode != http.StatusOK {
		t.Fatalf("query databases failed.  Unexpected status code.  expected: %d, actual %d", http.StatusOK, resp.StatusCode)
	}

	expectedResults = client.Results{
		Results: []client.Result{
			{Rows: []influxql.Row{
				{
					Name:    "cpu",
					Columns: []string{"time", "value"},
					Values: [][]interface{}{
						[]interface{}{now.Format(time.RFC3339Nano), json.Number("100")},
					},
				}}},
		},
	}

	if !reflect.DeepEqual(results, expectedResults) {
		t.Logf("Expected:\n")
		t.Logf("%#v\n", expectedResults)
		t.Logf("Actual:\n")
		t.Logf("%#v\n", results)
		t.Fatalf("query databases failed.  Unexpected results.")
	}
}

func Test_ServerSingleIntegration(t *testing.T) {
	nNodes := 1
	basePort := 8090
	createCombinedNodeCluster(t, "single node", nNodes, basePort)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:" + strconv.Itoa(basePort),
	}
	simpleWriteAndQuery(t, "single node", serverURL, nNodes)
}

func Test_Server3NodeIntegration(t *testing.T) {
	nNodes := 3
	basePort := 8190
	createCombinedNodeCluster(t, "3 node", nNodes, basePort)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:" + strconv.Itoa(basePort),
	}
	simpleWriteAndQuery(t, "3 node", serverURL, nNodes)
}

func Test_Server5NodeIntegration(t *testing.T) {
	nNodes := 5
	basePort := 8290
	createCombinedNodeCluster(t, "5 node", nNodes, basePort)

	serverURL := &url.URL{
		Scheme: "http",
		Host:   "localhost:" + strconv.Itoa(basePort),
	}
	simpleWriteAndQuery(t, "5 node", serverURL, nNodes)
}

func urlFor(u *url.URL, path string, params url.Values) *url.URL {
	u.Path = path
	u.RawQuery = params.Encode()
	return u
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package service

import (
	"io"
	stdLog "log"
	"os"
	"path/filepath"

	"github.com/webx-top/com"

	"github.com/admpub/log"
)

func Run(options *Options, action string) error {
	conf := &Config{}
	conf.CopyFromOptions(options)
	conf.Dir = com.SelfDir()
	conf.Exec = os.Args[0]
	if len(os.Args) > 3 {
		conf.Args = os.Args[3:]
	}
	if err := initServiceLog(conf); err != nil {
		return err
	}
	return New(conf, action)
}

func FileWriter(file string) (io.WriteCloser, error) {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0777)
	return f, err
}

func initServiceLog(conf *Config) error {
	logDir := filepath.Join(com.SelfDir(), `data`, `logs`)
	err := com.MkdirAll(logDir, os.ModePerm)
	if err != nil {
		return err
	}
	serviceLog := log.New()
	serviceLog.SetFormatter(func(l *log.Logger, e *log.Entry) string {
		return e.Message
	})
	fileTarget := log.NewFileTarget()
	fileTarget.FileName = filepath.Join(logDir, `app_{date:20060102}.log`) //按天分割日志
	fileTarget.MaxBytes = 100 * 1024 * 1024
	serviceLog.SetTarget(fileTarget)
	conf.Stderr = serviceLog.Writer(log.LevelError)
	conf.Stdout = serviceLog.Writer(log.LevelInfo)

	w, err := FileWriter(filepath.Join(logDir, `service.log`))
	if err != nil {
		return err
	}
	conf.OnExited = func() error {
		if w != nil {
			return w.Close()
		}
		return nil
	}
	stdLog.SetOutput(w)
	stdLog.SetFlags(stdLog.Lshortfile)
	conf.logger = newLogger(w)
	return err
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package service

import (
	"io"
	stdLog "log"
	"os"
	"path/filepath"

	"github.com/webx-top/com"

	"github.com/admpub/log"
)

func Run(options *Options, action string) error {
	conf := &Config{}
	conf.CopyFromOptions(options)
	conf.Dir = com.SelfDir()
	conf.Exec = os.Args[0]
	if len(os.Args) > 3 {
		conf.Args = os.Args[3:]
	}
	if err := initServiceLog(conf); err != nil {
		return err
	}
	return New(conf, action)
}

func FileWriter(file string) (io.WriteCloser, error) {
	f, err := os.OpenFile(file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0777)
	return f, err
}

func initServiceLog(conf *Config) error {
	logDir := filepath.Join(com.SelfDir(), `data`, `logs`)
	err := com.MkdirAll(logDir, os.ModePerm)
	if err != nil {
		return err
	}
	// 保存子进程输出到文件
	serviceLog := log.New()
	serviceLog.SetFormatter(func(l *log.Logger, e *log.Entry) string {
		return e.Message
	})
	fileTarget := log.NewFileTarget()
	fileTarget.FileName = filepath.Join(logDir, `app_{date:20060102}.log`) //按天分割日志
	fileTarget.MaxBytes = 100 * 1024 * 1024
	serviceLog.SetTarget(fileTarget)
	conf.Stderr = serviceLog.Writer(log.LevelError)
	conf.Stdout = serviceLog.Writer(log.LevelInfo)

	// 保存service程序中输出的日志
	w, err := FileWriter(filepath.Join(logDir, `service.log`))
	if err != nil {
		return err
	}
	conf.OnExited = func() error {
		serviceLog.Close()
		if w != nil {
			return w.Close()
		}
		return nil
	}
	stdLog.SetOutput(w)
	stdLog.SetFlags(stdLog.Lshortfile)
	conf.logger = newLogger(w)
	return err
}
<|endoftext|>


<commit_before>package finder

import (
	"errors"
	"os"
	"path/filepath"
	"regexp"

	"github.com/gobwas/glob"
)

type itemType uint8

const (
	typeAll itemType = iota
	typeFile
	typeDir
)

// Finder contains options for a file/directory search.
type Finder struct {
	dirs        []string
	names       []Matcher
	paths       []Matcher
	notPaths    []Matcher
	notNames    []Matcher
	setupErrors []error
	itype       itemType
}

// Matcher checks whether an Item matches.
type Matcher func(Item) bool

// New returns a new finder.
//
// By default it will search for both files and directories.
func New() *Finder {
	return &Finder{}
}

// In searches in the given list of directories.
func (f *Finder) In(directories ...string) *Finder {
	f.dirs = append(f.dirs, directories...)
	return f
}

// Path narrows down the folders to be searched using gobwas/glob
//
// p is matched against the item's RelPath()
// See https://github.com/gobwas/glob
func (f *Finder) Path(p string) *Finder {
	matcher := f.path(p)
	if matcher != nil {
		f.paths = append(f.paths, matcher)
	}
	return f
}

// NotPath excludes folders from the search using gobwas/glob
//
// p is matched against the item's RelPath()
// See https://github.com/gobwas/glob
func (f *Finder) NotPath(p string) *Finder {
	matcher := f.path(p)
	if matcher != nil {
		f.notPaths = append(f.notPaths, matcher)
	}
	return f
}

func (f *Finder) path(p string) Matcher {
	g, err := glob.Compile(p, os.PathSeparator)
	if err != nil {
		f.setupErrors = append(f.setupErrors, err)
		return nil
	}
	return func(i Item) bool {
		if i.IsDir() {
			return g.Match(i.RelPath())
		}
		return g.Match(filepath.Dir(i.RelPath()))
	}
}

// Name matches a file or directory name using gobwas/glob
//
// See https://github.com/gobwas/glob
func (f *Finder) Name(n string) *Finder {
	matcher := f.name(n)
	if matcher != nil {
		f.names = append(f.names, matcher)
	}
	return f
}

func (f *Finder) name(n string) Matcher {
	g, err := glob.Compile(n, os.PathSeparator)
	if err != nil {
		f.setupErrors = append(f.setupErrors, err)
		return nil
	}
	return func(i Item) bool {
		return g.Match(i.Name())
	}
}

// NameRegex matches a file or directory name using package regexp.
func (f *Finder) NameRegex(n string) *Finder {
	matcher := f.nameRegex(n)
	if matcher != nil {
		f.names = append(f.names, matcher)
	}
	return f
}

func (f *Finder) nameRegex(n string) Matcher {
	re, err := regexp.Compile(n)
	if err != nil {
		f.setupErrors = append(f.setupErrors, err)
		return nil
	}
	return func(i Item) bool {
		return re.MatchString(i.Name())
	}
}

// NotName excludes a file or directory name using gobwas/glob
//
// See https://github.com/gobwas/glob
func (f *Finder) NotName(n string) *Finder {
	matcher := f.name(n)
	if matcher != nil {
		f.notNames = append(f.notNames, matcher)
	}
	return f
}

// NotNameRegex excludes a file or directory name using package regexp.
func (f *Finder) NotNameRegex(n string) *Finder {
	matcher := f.nameRegex(n)
	if matcher != nil {
		f.notNames = append(f.notNames, matcher)
	}
	return f
}

// Files makes the finder return files only.
func (f *Finder) Files() *Finder {
	f.itype = typeFile
	return f
}

// Dirs makes the finder return directories only.
func (f *Finder) Dirs() *Finder {
	f.itype = typeDir
	return f
}

var errNoMatch = errors.New("Item did not match")
var errSkipDir = filepath.SkipDir

func (f *Finder) match(i Item) error {
	if (f.itype == typeDir && !i.IsDir()) || (f.itype == typeFile && i.IsDir()) {
		return errNoMatch
	}
	var match error
	if len(f.paths) > 0 {
		if i.IsDir() {
			for _, p := range f.paths {
				if !p(i) {
					return errSkipDir
				}
			}
		} else {
			match = errNoMatch
			for _, p := range f.paths {
				if p(i) {
					match = nil
				}
			}
			if match == errNoMatch {
				return match
			}
		}
	}
	if len(f.notPaths) > 0 {
		if i.IsDir() {
			for _, p := range f.notPaths {
				if p(i) {
					return errSkipDir
				}
			}
		} else {
			for _, p := range f.notPaths {
				if p(i) {
					return errNoMatch
				}
			}
		}
	}
	if len(f.names) > 0 {
		match = errNoMatch
		for _, n := range f.names {
			if n(i) {
				match = nil
				break
			}
		}
	}
	if match == errNoMatch {
		return match
	}
	if len(f.notNames) > 0 {
		for _, matcher := range f.notNames {
			if matcher(i) {
				match = errNoMatch
				continue
			}
		}
	}
	return match
}

// Each calls func fn with each found item.
func (f *Finder) Each(fn func(Item)) []error {
	var errs []error
	var dir string
	walker := func(path string, info os.FileInfo, err error) error {
		if dir == path {
			return nil
		}
		if err != nil {
			return err
		}
		relDir, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}
		item := newItem(info, dir, relDir)
		match := f.match(item)
		switch match {
		case nil:
			fn(item)
			return nil
		case errNoMatch:
			return nil
		case errSkipDir:
			return errSkipDir
		}
		return nil
	}
	for _, dir = range f.dirs {
		err := filepath.Walk(dir, walker)
		if err != nil && err != errSkipDir {
			errs = append(errs, err)
		}
	}
	return append(f.setupErrors, errs...)
}

// ToSlice returns a slice of all found items.
func (f *Finder) ToSlice() (ItemSlice, []error) {
	var l []Item
	errs := f.Each(func(file Item) {
		l = append(l, file)
	})
	return l, errs
}
<commit_msg>Extract match* methods from Finder.match<commit_after>package finder

import (
	"errors"
	"os"
	"path/filepath"
	"regexp"

	"github.com/gobwas/glob"
)

type itemType uint8

const (
	typeAll itemType = iota
	typeFile
	typeDir
)

// Finder contains options for a file/directory search.
type Finder struct {
	dirs        []string
	names       []Matcher
	paths       []Matcher
	notPaths    []Matcher
	notNames    []Matcher
	setupErrors []error
	itype       itemType
}

// Matcher checks whether an Item matches.
type Matcher func(Item) bool

// New returns a new finder.
//
// By default it will search for both files and directories.
func New() *Finder {
	return &Finder{}
}

// In searches in the given list of directories.
func (f *Finder) In(directories ...string) *Finder {
	f.dirs = append(f.dirs, directories...)
	return f
}

// Path narrows down the folders to be searched using gobwas/glob
//
// p is matched against the item's RelPath()
// See https://github.com/gobwas/glob
func (f *Finder) Path(p string) *Finder {
	matcher := f.path(p)
	if matcher != nil {
		f.paths = append(f.paths, matcher)
	}
	return f
}

// NotPath excludes folders from the search using gobwas/glob
//
// p is matched against the item's RelPath()
// See https://github.com/gobwas/glob
func (f *Finder) NotPath(p string) *Finder {
	matcher := f.path(p)
	if matcher != nil {
		f.notPaths = append(f.notPaths, matcher)
	}
	return f
}

func (f *Finder) path(p string) Matcher {
	g, err := glob.Compile(p, os.PathSeparator)
	if err != nil {
		f.setupErrors = append(f.setupErrors, err)
		return nil
	}
	return func(i Item) bool {
		if i.IsDir() {
			return g.Match(i.RelPath())
		}
		return g.Match(filepath.Dir(i.RelPath()))
	}
}

// Name matches a file or directory name using gobwas/glob
//
// See https://github.com/gobwas/glob
func (f *Finder) Name(n string) *Finder {
	matcher := f.name(n)
	if matcher != nil {
		f.names = append(f.names, matcher)
	}
	return f
}

func (f *Finder) name(n string) Matcher {
	g, err := glob.Compile(n, os.PathSeparator)
	if err != nil {
		f.setupErrors = append(f.setupErrors, err)
		return nil
	}
	return func(i Item) bool {
		return g.Match(i.Name())
	}
}

// NameRegex matches a file or directory name using package regexp.
func (f *Finder) NameRegex(n string) *Finder {
	matcher := f.nameRegex(n)
	if matcher != nil {
		f.names = append(f.names, matcher)
	}
	return f
}

func (f *Finder) nameRegex(n string) Matcher {
	re, err := regexp.Compile(n)
	if err != nil {
		f.setupErrors = append(f.setupErrors, err)
		return nil
	}
	return func(i Item) bool {
		return re.MatchString(i.Name())
	}
}

// NotName excludes a file or directory name using gobwas/glob
//
// See https://github.com/gobwas/glob
func (f *Finder) NotName(n string) *Finder {
	matcher := f.name(n)
	if matcher != nil {
		f.notNames = append(f.notNames, matcher)
	}
	return f
}

// NotNameRegex excludes a file or directory name using package regexp.
func (f *Finder) NotNameRegex(n string) *Finder {
	matcher := f.nameRegex(n)
	if matcher != nil {
		f.notNames = append(f.notNames, matcher)
	}
	return f
}

// Files makes the finder return files only.
func (f *Finder) Files() *Finder {
	f.itype = typeFile
	return f
}

// Dirs makes the finder return directories only.
func (f *Finder) Dirs() *Finder {
	f.itype = typeDir
	return f
}

var errNoMatch = errors.New("Item did not match")
var errSkipDir = filepath.SkipDir

func (f *Finder) match(i Item) error {
	if (f.itype == typeDir && !i.IsDir()) || (f.itype == typeFile && i.IsDir()) {
		return errNoMatch
	}
	var match error
	match = f.matchPaths(i)
	if match != nil {
		return match
	}
	match = f.matchNotPaths(i)
	if match != nil {
		return match
	}
	match = f.matchNames(i)
	if match != nil {
		return match
	}
	return f.matchNotNames(i)
}

func (f *Finder) matchPaths(i Item) error {
	if len(f.paths) == 0 {
		return nil
	}
	if i.IsDir() {
		for _, p := range f.paths {
			if !p(i) {
				return errSkipDir
			}
		}
		return nil
	}
	match := errNoMatch
	for _, p := range f.paths {
		if p(i) {
			match = nil
		}
	}
	return match
}

func (f *Finder) matchNotPaths(i Item) error {
	if len(f.notPaths) == 0 {
		return nil
	}
	if i.IsDir() {
		for _, p := range f.notPaths {
			if p(i) {
				return errSkipDir
			}
		}
		return nil
	}
	for _, p := range f.notPaths {
		if p(i) {
			return errNoMatch
		}
	}
	return nil
}

func (f *Finder) matchNames(i Item) error {
	if len(f.names) == 0 {
		return nil
	}
	for _, n := range f.names {
		if n(i) {
			return nil
		}
	}
	return errNoMatch
}

func (f *Finder) matchNotNames(i Item) error {
	if len(f.notNames) == 0 {
		return nil
	}
	for _, n := range f.notNames {
		if n(i) {
			return errNoMatch
		}
	}
	return nil
}

// Each calls func fn with each found item.
func (f *Finder) Each(fn func(Item)) []error {
	var errs []error
	var dir string
	walker := func(path string, info os.FileInfo, err error) error {
		if dir == path {
			return nil
		}
		if err != nil {
			return err
		}
		relDir, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}
		item := newItem(info, dir, relDir)
		match := f.match(item)
		switch match {
		case nil:
			fn(item)
			return nil
		case errNoMatch:
			return nil
		case errSkipDir:
			return errSkipDir
		}
		return nil
	}
	for _, dir = range f.dirs {
		err := filepath.Walk(dir, walker)
		if err != nil && err != errSkipDir {
			errs = append(errs, err)
		}
	}
	return append(f.setupErrors, errs...)
}

// ToSlice returns a slice of all found items.
func (f *Finder) ToSlice() (ItemSlice, []error) {
	var l []Item
	errs := f.Each(func(file Item) {
		l = append(l, file)
	})
	return l, errs
}
<|endoftext|>


<commit_before>package flache

import (
	"sync"
	"time"
)

const (
	shardsLog  = uint64(10)
	shardsNum  = 1 << shardsLog
	shardsMask = shardsNum - 1
)

// entry internal struct to store value in the `cache`
type entry struct {
	Expiration int64
	Value      interface{}
}

// Cache a concurrent, sharded and flexible cache
type Cache struct {
	buckets     [shardsNum]map[string]entry
	mutexes     [shardsNum]sync.Mutex
	expiration  int64
	autocleanup int64
}

// NewCache creates empty Cache
func NewCache(expiration, autocleanup time.Duration) *Cache {
	f := NewStaticCache()
	f.expiration = int64(expiration)
	f.autocleanup = int64(autocleanup)

	if autocleanup != 0 {
		go func() {
			time.Sleep(time.Duration(f.autocleanup))
			f.Purge()
		}()
	}
	return f
}

// NewStaticCache creates empty Cache without any time setups
func NewStaticCache() *Cache {
	f := Cache{}
	for i := uint64(0); i < shardsNum; i++ {
		f.buckets[i] = make(map[string]entry)
		f.mutexes[i] = sync.Mutex{}
	}
	return &f
}

// Add adds key-value in to the cache for a predefined time
func (f *Cache) Add(key string, value interface{}) {
	f.AddExt(key, value, time.Duration(f.expiration))
}

// AddExt adds key-value in to the cache for a specified time
func (f *Cache) AddExt(key string, value interface{}, duration time.Duration) {
	now := time.Now().UnixNano()
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	f.buckets[index][key] = entry{Expiration: int64(duration) + now, Value: value}
	//
	f.mutexes[index].Unlock()
}

// Check returns state of an object for a given key
func (f *Cache) Check(key string) (exists, alive bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, exists := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	alive = exists && (f.expiration == 0 || value.Expiration > time.Now().UnixNano())
	return exists, alive
}

// Has returns true if value for key is in the cache and not expired
func (f *Cache) Has(key string) bool {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	return ok && (f.expiration == 0 || value.Expiration > time.Now().UnixNano())
}

// HasExt returns true(and duration) if value for key is in the cache and not expired
func (f *Cache) HasExt(key string) (time.Duration, bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	if ok && (f.expiration == 0 || value.Expiration > time.Now().UnixNano()) {
		return time.Duration(value.Expiration), true
	}
	return 0, false
}

// Get returns value for a `key` if it's not expired yet
func (f *Cache) Get(key string) (interface{}, time.Duration, bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	if ok && (f.expiration == 0 || value.Expiration > time.Now().UnixNano()) {
		return value.Value, time.Duration(value.Expiration), true
	}
	var zero interface{}
	return zero, 0, false
}

// GetUpd returns value for a `key` if it's not expired yet and updates expiration time for the `key`
func (f *Cache) GetUpd(key string) (interface{}, time.Duration, bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	if ok {
		value.Expiration = f.expiration + time.Now().UnixNano()
		f.buckets[index][key] = value
	}
	//
	f.mutexes[index].Unlock()

	if ok {
		return value.Value, time.Duration(value.Expiration), true
	}
	var zero interface{}
	return zero, 0, false
}

// Gets returns values for a `keys` if it's not expired yet
func (f *Cache) Gets(keys ...string) ([]interface{}) {
	size := len(keys)
	res := make([]interface{}, size)
	now := time.Now().UnixNano()

	for i := 0; i < size; i++ {
		index := f.hash(keys[i]) & shardsMask

		f.mutexes[index].Lock()
		//
		value, ok := f.buckets[index][keys[i]]
		//
		f.mutexes[index].Unlock()

		if ok && value.Expiration >= now {
			res[i] = value.Value
		} else {
			res[i] = nil
		}
	}

	return res
}

// Del removes `key` from the cache
func (f *Cache) Del(key string) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	delete(f.buckets[index], key)
	//
	f.mutexes[index].Unlock()
}

// Dels removes `keys` from the cache
func (f *Cache) Dels(keys ...string) {
	size := len(keys)

	for i := 0; i < size; i++ {
		index := f.hash(keys[i]) & shardsMask

		f.mutexes[index].Lock()
		//
		delete(f.buckets[index], keys[i])
		//
		f.mutexes[index].Unlock()
	}
}

// Set updates value for a `key` but do not touches expiration time
func (f *Cache) Set(key string, value interface{}) {
	expiration := f.expiration
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	oldValue, ok := f.buckets[index][key]
	if ok {
		expiration = oldValue.Expiration
	}
	f.buckets[index][key] = entry{Expiration: expiration, Value: value}
	//
	f.mutexes[index].Unlock()
}

// Touch updates expiration time for the `key`
func (f *Cache) Touch(key string) {
	if f.expiration == 0 {
		return
	}
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	if ok {
		value.Expiration = f.expiration + time.Now().UnixNano()
		f.buckets[index][key] = value
	}
	//
	f.mutexes[index].Unlock()
}

// Touches updates expiration time for the `key`
func (f *Cache) Touches(keys ...string) {
	if f.expiration == 0 {
		return
	}
	size := len(keys)
	now := time.Now().UnixNano()

	for i := 0; i < size; i++ {
		index := f.hash(keys[i]) & shardsMask

		f.mutexes[index].Lock()
		//
		value, ok := f.buckets[index][keys[i]]
		if ok {
			value.Expiration = f.expiration + now
			f.buckets[index][keys[i]] = value
		}
		//
		f.mutexes[index].Unlock()
	}
}

// Keys returns slice of all non-expired keys
func (f *Cache) Keys() []string {
	var res []string
	for i := uint64(0); i < shardsNum; i++ {
		now := time.Now().UnixNano()

		f.mutexes[i].Lock()
		//
		for k, v := range f.buckets[i] {
			if f.expiration == 0 || v.Expiration > now {
				res = append(res, k)
			}
		}
		//
		f.mutexes[i].Unlock()
	}
	return res
}

// Values returns slice of all non-expired values
func (f *Cache) Values() []interface{} {
	var res []interface{}
	for i := uint64(0); i < shardsNum; i++ {
		now := time.Now().UnixNano()

		f.mutexes[i].Lock()
		//
		for _, v := range f.buckets[i] {
			if f.expiration == 0 || v.Expiration > now {
				res = append(res, v.Value)
			}
		}
		//
		f.mutexes[i].Unlock()
	}
	return res
}

// Purge removes expired entries from shard
func (f *Cache) Purge() {
	if f.expiration == 0 {
		return
	}
	for i := uint64(0); i < shardsNum; i++ {
		go func(i uint64) {
			now := time.Now().UnixNano()

			f.mutexes[i].Lock()
			//
			for k, v := range f.buckets[i] {
				if v.Expiration < now {
					delete(f.buckets[i], k)
				}
			}
			//
			f.mutexes[i].Unlock()
		}(i)
	}
}

// Clear removes all values from a cache
func (f *Cache) Clear() {
	for i := uint64(0); i < shardsNum; i++ {
		f.mutexes[i].Lock()
		//
		f.buckets[i] = make(map[string]entry)
		//
		f.mutexes[i].Unlock()
	}
}

// Size returns amount of not expired objects
func (f *Cache) Size() int64 {
	var res int64
	ch := make(chan int64, shardsNum)

	for i := uint64(0); i < shardsNum; i++ {
		go func(i uint64, c chan int64) {
			var alive int64

			f.mutexes[i].Lock()
			//
			if f.expiration == 0 {
				alive = int64(len(f.buckets[i]))
			} else {
				now := time.Now().UnixNano()
				for _, v := range f.buckets[i] {
					if v.Expiration > now {
						alive++
					}
				}
			}
			//
			f.mutexes[i].Unlock()

			ch <- alive
		}(i, ch)
	}
	for i := uint64(0); i < shardsNum; i++ {
		res += <-ch
	}
	return res
}

// hash return hash of key, FNV-1 algorithm
func (f *Cache) hash(key string) uint64 {
	hash := uint64(14695981039346656037)
	for _, c := range key {
		hash *= 1099511628211
		hash ^= uint64(c)
	}
	return hash
}
<commit_msg>Added Cacher interface<commit_after>package flache

import (
	"sync"
	"time"
)

const (
	shardsLog  = uint64(10)
	shardsNum  = 1 << shardsLog
	shardsMask = shardsNum - 1
)

// entry internal struct to store value in the `cache`
type entry struct {
	Expiration int64
	Value      interface{}
}

type Cacher interface {
	Add(string, interface{})
	AddExt(string, interface{}, time.Duration)
	Check(string) (bool, bool)
	Has(string)
	HasExt(string) (interface{}, time.Duration, bool)
	Get(string) (interface{})
	GetExt(string) (interface{}, time.Duration, bool)
	Gets(...string) []interface{}
	Del(string)
	Dels(...string)
	Set(string, interface{})
	Touch(string)
	Touchs(...string)
	Keys() []string
	Values() []interface{}
	Purge()
	Clear()
	Size() int64
}

// Cache a concurrent, sharded and flexible cache
type Cache struct {
	buckets     [shardsNum]map[string]entry
	mutexes     [shardsNum]sync.Mutex
	expiration  int64
	autocleanup int64
}

// NewCache creates empty Cache
func NewCache(expiration, autocleanup time.Duration) *Cache {
	f := NewStaticCache()
	f.expiration = int64(expiration)
	f.autocleanup = int64(autocleanup)

	if autocleanup != 0 {
		go func() {
			time.Sleep(time.Duration(f.autocleanup))
			f.Purge()
		}()
	}
	return f
}

// NewStaticCache creates empty Cache without any time setups
func NewStaticCache() *Cache {
	f := Cache{}
	for i := uint64(0); i < shardsNum; i++ {
		f.buckets[i] = make(map[string]entry)
		f.mutexes[i] = sync.Mutex{}
	}
	return &f
}

// Add adds key-value in to the cache for a predefined time
func (f *Cache) Add(key string, value interface{}) {
	f.AddExt(key, value, time.Duration(f.expiration))
}

// AddExt adds key-value in to the cache for a specified time
func (f *Cache) AddExt(key string, value interface{}, duration time.Duration) {
	now := time.Now().UnixNano()
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	f.buckets[index][key] = entry{Expiration: int64(duration) + now, Value: value}
	//
	f.mutexes[index].Unlock()
}

// Check returns state of an object for a given key
func (f *Cache) Check(key string) (exists, alive bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, exists := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	alive = exists && (f.expiration == 0 || value.Expiration > time.Now().UnixNano())
	return exists, alive
}

// Has returns true if value for key is in the cache and not expired
func (f *Cache) Has(key string) bool {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	return ok && (f.expiration == 0 || value.Expiration > time.Now().UnixNano())
}

// HasExt returns true(and duration) if value for key is in the cache and not expired
func (f *Cache) HasExt(key string) (time.Duration, bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	if ok && (f.expiration == 0 || value.Expiration > time.Now().UnixNano()) {
		return time.Duration(value.Expiration), true
	}
	return 0, false
}

// Get returns value for a `key` if it's not expired yet
func (f *Cache) Get(key string) (interface{}, time.Duration, bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	//
	f.mutexes[index].Unlock()

	if ok && (f.expiration == 0 || value.Expiration > time.Now().UnixNano()) {
		return value.Value, time.Duration(value.Expiration), true
	}
	var zero interface{}
	return zero, 0, false
}

// GetUpd returns value for a `key` if it's not expired yet and updates expiration time for the `key`
func (f *Cache) GetUpd(key string) (interface{}, time.Duration, bool) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	if ok {
		value.Expiration = f.expiration + time.Now().UnixNano()
		f.buckets[index][key] = value
	}
	//
	f.mutexes[index].Unlock()

	if ok {
		return value.Value, time.Duration(value.Expiration), true
	}
	var zero interface{}
	return zero, 0, false
}

// Gets returns values for a `keys` if it's not expired yet
func (f *Cache) Gets(keys ...string) ([]interface{}) {
	size := len(keys)
	res := make([]interface{}, size)
	now := time.Now().UnixNano()

	for i := 0; i < size; i++ {
		index := f.hash(keys[i]) & shardsMask

		f.mutexes[index].Lock()
		//
		value, ok := f.buckets[index][keys[i]]
		//
		f.mutexes[index].Unlock()

		if ok && value.Expiration >= now {
			res[i] = value.Value
		} else {
			res[i] = nil
		}
	}

	return res
}

// Del removes `key` from the cache
func (f *Cache) Del(key string) {
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	delete(f.buckets[index], key)
	//
	f.mutexes[index].Unlock()
}

// Dels removes `keys` from the cache
func (f *Cache) Dels(keys ...string) {
	size := len(keys)

	for i := 0; i < size; i++ {
		index := f.hash(keys[i]) & shardsMask

		f.mutexes[index].Lock()
		//
		delete(f.buckets[index], keys[i])
		//
		f.mutexes[index].Unlock()
	}
}

// Set updates value for a `key` but do not touches expiration time
func (f *Cache) Set(key string, value interface{}) {
	expiration := f.expiration
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	oldValue, ok := f.buckets[index][key]
	if ok {
		expiration = oldValue.Expiration
	}
	f.buckets[index][key] = entry{Expiration: expiration, Value: value}
	//
	f.mutexes[index].Unlock()
}

// Touch updates expiration time for the `key`
func (f *Cache) Touch(key string) {
	if f.expiration == 0 {
		return
	}
	index := f.hash(key) & shardsMask

	f.mutexes[index].Lock()
	//
	value, ok := f.buckets[index][key]
	if ok {
		value.Expiration = f.expiration + time.Now().UnixNano()
		f.buckets[index][key] = value
	}
	//
	f.mutexes[index].Unlock()
}

// Touches updates expiration time for the `key`
func (f *Cache) Touches(keys ...string) {
	if f.expiration == 0 {
		return
	}
	size := len(keys)
	now := time.Now().UnixNano()

	for i := 0; i < size; i++ {
		index := f.hash(keys[i]) & shardsMask

		f.mutexes[index].Lock()
		//
		value, ok := f.buckets[index][keys[i]]
		if ok {
			value.Expiration = f.expiration + now
			f.buckets[index][keys[i]] = value
		}
		//
		f.mutexes[index].Unlock()
	}
}

// Keys returns slice of all non-expired keys
func (f *Cache) Keys() []string {
	var res []string
	for i := uint64(0); i < shardsNum; i++ {
		now := time.Now().UnixNano()

		f.mutexes[i].Lock()
		//
		for k, v := range f.buckets[i] {
			if f.expiration == 0 || v.Expiration > now {
				res = append(res, k)
			}
		}
		//
		f.mutexes[i].Unlock()
	}
	return res
}

// Values returns slice of all non-expired values
func (f *Cache) Values() []interface{} {
	var res []interface{}
	for i := uint64(0); i < shardsNum; i++ {
		now := time.Now().UnixNano()

		f.mutexes[i].Lock()
		//
		for _, v := range f.buckets[i] {
			if f.expiration == 0 || v.Expiration > now {
				res = append(res, v.Value)
			}
		}
		//
		f.mutexes[i].Unlock()
	}
	return res
}

// Purge removes expired entries from shard
func (f *Cache) Purge() {
	if f.expiration == 0 {
		return
	}
	for i := uint64(0); i < shardsNum; i++ {
		go func(i uint64) {
			now := time.Now().UnixNano()

			f.mutexes[i].Lock()
			//
			for k, v := range f.buckets[i] {
				if v.Expiration < now {
					delete(f.buckets[i], k)
				}
			}
			//
			f.mutexes[i].Unlock()
		}(i)
	}
}

// Clear removes all values from a cache
func (f *Cache) Clear() {
	for i := uint64(0); i < shardsNum; i++ {
		f.mutexes[i].Lock()
		//
		f.buckets[i] = make(map[string]entry)
		//
		f.mutexes[i].Unlock()
	}
}

// Size returns amount of not expired objects
func (f *Cache) Size() int64 {
	var res int64
	ch := make(chan int64, shardsNum)

	for i := uint64(0); i < shardsNum; i++ {
		go func(i uint64, c chan int64) {
			var alive int64

			f.mutexes[i].Lock()
			//
			if f.expiration == 0 {
				alive = int64(len(f.buckets[i]))
			} else {
				now := time.Now().UnixNano()
				for _, v := range f.buckets[i] {
					if v.Expiration > now {
						alive++
					}
				}
			}
			//
			f.mutexes[i].Unlock()

			ch <- alive
		}(i, ch)
	}
	for i := uint64(0); i < shardsNum; i++ {
		res += <-ch
	}
	return res
}

// hash return hash of key, FNV-1 algorithm
func (f *Cache) hash(key string) uint64 {
	hash := uint64(14695981039346656037)
	for _, c := range key {
		hash *= 1099511628211
		hash ^= uint64(c)
	}
	return hash
}
<|endoftext|>


<commit_before>package gapi

import (
	"bytes"
	"encoding/json"
	"fmt"
)

// Folder represents a Grafana folder.
type Folder struct {
	ID    int64  `json:"id"`
	UID   string `json:"uid"`
	Title string `json:"title"`
}

// Folders fetches and returns Grafana folders.
func (c *Client) Folders() ([]Folder, error) {
	folders := make([]Folder, 0)
	err := c.request("GET", "/api/folders/", nil, nil, &folders)
	if err != nil {
		return folders, err
	}

	return folders, err
}

// Folder fetches and returns the Grafana folder whose ID it's passed.
func (c *Client) Folder(id int64) (*Folder, error) {
	folder := &Folder{}
	err := c.request("GET", fmt.Sprintf("/api/folders/id/%d", id), nil, nil, folder)
	if err != nil {
		return folder, err
	}

	return folder, err
}

// Folder fetches and returns the Grafana folder whose UID it's passed.
func (c *Client) FolderByUID(uid string) (*Folder, error) {
	folder := &Folder{}
	err := c.request("GET", fmt.Sprintf("/api/folders/%s", uid), nil, nil, folder)
	if err != nil {
		return folder, err
	}

	return folder, err
}

// NewFolder creates a new Grafana folder.
func (c *Client) NewFolder(title string) (Folder, error) {
	folder := Folder{}
	dataMap := map[string]string{
		"title": title,
	}
	data, err := json.Marshal(dataMap)
	if err != nil {
		return folder, err
	}

	err = c.request("POST", "/api/folders", nil, bytes.NewBuffer(data), &folder)
	if err != nil {
		return folder, err
	}

	return folder, err
}

// UpdateFolder updates the folder whose ID it's passed.
func (c *Client) UpdateFolder(id string, name string) error {
	dataMap := map[string]string{
		"name": name,
	}
	data, err := json.Marshal(dataMap)
	if err != nil {
		return err
	}

	return c.request("PUT", fmt.Sprintf("/api/folders/%s", id), nil, bytes.NewBuffer(data), nil)
}

// DeleteFolder deletes the folder whose ID it's passed.
func (c *Client) DeleteFolder(id string) error {
	return c.request("DELETE", fmt.Sprintf("/api/folders/%s", id), nil, nil, nil)
}
<commit_msg>Add folder UID option (#67)<commit_after>package gapi

import (
	"bytes"
	"encoding/json"
	"fmt"
)

// Folder represents a Grafana folder.
type Folder struct {
	ID    int64  `json:"id"`
	UID   string `json:"uid"`
	Title string `json:"title"`
}

// Folders fetches and returns Grafana folders.
func (c *Client) Folders() ([]Folder, error) {
	folders := make([]Folder, 0)
	err := c.request("GET", "/api/folders/", nil, nil, &folders)
	if err != nil {
		return folders, err
	}

	return folders, err
}

// Folder fetches and returns the Grafana folder whose ID it's passed.
func (c *Client) Folder(id int64) (*Folder, error) {
	folder := &Folder{}
	err := c.request("GET", fmt.Sprintf("/api/folders/id/%d", id), nil, nil, folder)
	if err != nil {
		return folder, err
	}

	return folder, err
}

// Folder fetches and returns the Grafana folder whose UID it's passed.
func (c *Client) FolderByUID(uid string) (*Folder, error) {
	folder := &Folder{}
	err := c.request("GET", fmt.Sprintf("/api/folders/%s", uid), nil, nil, folder)
	if err != nil {
		return folder, err
	}

	return folder, err
}

// NewFolder creates a new Grafana folder.
func (c *Client) NewFolder(title string, uid ...string) (Folder, error) {
	if len(uid) > 1 {
		return Folder{}, fmt.Errorf("too many arguments. Expected 1 or 2")
	}

	folder := Folder{}
	dataMap := map[string]string{
		"title": title,
	}
	if len(uid) == 1 {
		dataMap["uid"] = uid[0]
	}
	data, err := json.Marshal(dataMap)
	if err != nil {
		return folder, err
	}

	err = c.request("POST", "/api/folders", nil, bytes.NewBuffer(data), &folder)
	if err != nil {
		return folder, err
	}

	return folder, err
}

// UpdateFolder updates the folder whose ID it's passed.
func (c *Client) UpdateFolder(id string, name string) error {
	dataMap := map[string]string{
		"name": name,
	}
	data, err := json.Marshal(dataMap)
	if err != nil {
		return err
	}

	return c.request("PUT", fmt.Sprintf("/api/folders/%s", id), nil, bytes.NewBuffer(data), nil)
}

// DeleteFolder deletes the folder whose ID it's passed.
func (c *Client) DeleteFolder(id string) error {
	return c.request("DELETE", fmt.Sprintf("/api/folders/%s", id), nil, nil, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/mackerelio/gomkr/utils"
)

type HostFormat struct {
	Id            string            `json:"id,omitempty"`
	Name          string            `json:"name,omitempty"`
	Status        string            `json:"status,omitempty"`
	Memo          string            `json:"memo,omitempty"`
	RoleFullnames []string          `json:"roleFullnames,omitempty"`
	IsRetired     bool              `json:"isRetired,omitempty"`
	CreatedAt     string            `json:"createdAt,omitempty"`
	IpAddresses   map[string]string `json:"ipAddresses,omitempty"`
}

func PrettyPrintJson(src interface{}) {
	data, err := json.MarshalIndent(src, "", "    ")
	utils.DieIf(err)
	fmt.Fprintln(os.Stdout, string(data))
}
<commit_msg>Output isRetired<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/mackerelio/gomkr/utils"
)

type HostFormat struct {
	Id            string            `json:"id,omitempty"`
	Name          string            `json:"name,omitempty"`
	Status        string            `json:"status,omitempty"`
	Memo          string            `json:"memo,omitempty"`
	RoleFullnames []string          `json:"roleFullnames,omitempty"`
	IsRetired     bool              `json:"isRetired"` // 'omitempty' regard boolean 'false' as empty.
	CreatedAt     string            `json:"createdAt,omitempty"`
	IpAddresses   map[string]string `json:"ipAddresses,omitempty"`
}

func PrettyPrintJson(src interface{}) {
	data, err := json.MarshalIndent(src, "", "    ")
	utils.DieIf(err)
	fmt.Fprintln(os.Stdout, string(data))
}
<|endoftext|>