<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"archive/zip"
	gojson "encoding/json"
	"fmt"
	"os"
	"path/filepath"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var supportPerfFlags = append([]cli.Flag{
	cli.StringFlag{
		Name:  "duration",
		Usage: "duration the entire perf tests are run",
		Value: "10s",
	},
	cli.BoolFlag{
		Name:  "verbose, v",
		Usage: "display per-server stats",
	},
	cli.StringFlag{
		Name:   "size",
		Usage:  "size of the object used for uploads/downloads",
		Value:  "64MiB",
		Hidden: true,
	},
	cli.IntFlag{
		Name:   "concurrent",
		Usage:  "number of concurrent requests per server",
		Value:  32,
		Hidden: true,
	},
	cli.StringFlag{
		Name:   "bucket",
		Usage:  "provide a custom bucket name to use (NOTE: bucket must be created prior)",
		Hidden: true, // Hidden for now.
	},
	// Drive test specific flags.
	cli.StringFlag{
		Name:   "filesize",
		Usage:  "total amount of data read/written to each drive",
		Value:  "1GiB",
		Hidden: true,
	},
	cli.StringFlag{
		Name:   "blocksize",
		Usage:  "read/write block size",
		Value:  "4MiB",
		Hidden: true,
	},
	cli.BoolFlag{
		Name:   "serial",
		Usage:  "run tests on drive(s) one-by-one",
		Hidden: true,
	},
}, subnetCommonFlags...)

var supportPerfCmd = cli.Command{
	Name:            "perf",
	Usage:           "upload object, network and drive performance analysis",
	Action:          mainSupportPerf,
	OnUsageError:    onUsageError,
	Before:          setGlobalsFromContext,
	Flags:           append(supportPerfFlags, supportGlobalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [COMMAND] [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}

EXAMPLES:
  1. Upload object storage, network, and drive performance analysis for cluster with alias 'myminio' to SUBNET
     {{.Prompt}} {{.HelpName}} myminio
  2. Run object storage, network, and drive performance tests on cluster with alias 'myminio', save and upload to SUBNET manually
     {{.Prompt}} {{.HelpName}} --airgap myminio
`,
}

// PerfTestOutput - stores the final output of performance test(s)
type PerfTestOutput struct {
	ObjectResult *ObjTestResult        `json:"object,omitempty"`
	NetResult    *madmin.NetperfResult `json:"network,omitempty"`
	DriveResult  []DriveTestResult     `json:"drive,omitempty"`
	Error        string                `json:"error,omitempty"`
}

// DriveTestResult - result of the drive performance test
type DriveTestResult struct {
	Endpoint  string             `json:"endpoint"`
	DrivePerf []madmin.DrivePerf `json:"drivePerf,omitempty"`
	Error     string             `json:"error,omitempty"`
}

// ObjTestResult - result of the object performance test
type ObjTestResult struct {
	Servers    int         `json:"servers"`
	Drives     int         `json:"drives"`
	Size       int         `json:"size"`
	Concurrent int         `json:"concurrent"`
	PUTStats   ObjPUTStats `json:"PUTStats"`
	GETStats   ObjGETStats `json:"GETStats"`
}

// ObjStatServer - Server level object performance stats
type ObjStatServer struct {
	Endpoint      string `json:"endpoint"`
	Throughput    uint64 `json:"throughput"`
	ObjectsPerSec uint64 `json:"objectsPerSec"`
	Error         string `json:"error,omitempty"`
}

// ObjPUTStats - PUT stats of all the servers
type ObjPUTStats struct {
	Throughput    uint64          `json:"throughput"`
	ObjectsPerSec uint64          `json:"objectsPerSec"`
	Response      madmin.Timings  `json:"responseTime"`
	Servers       []ObjStatServer `json:"servers"`
}

// ObjGETStats - GET stats of all the servers
type ObjGETStats struct {
	ObjPUTStats
	TTFB madmin.Timings `json:"ttfb,omitempty"`
}

func objectTestVerboseResult(result *madmin.SpeedTestResult) (msg string) {
	msg += "PUT:\n"
	for _, node := range result.PUTStats.Servers {
		msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
		if node.Err != "" {
			msg += " Err: " + node.Err
		}
		msg += "\n"
	}

	msg += "GET:\n"
	for _, node := range result.GETStats.Servers {
		msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
		if node.Err != "" {
			msg += " Err: " + node.Err
		}
		msg += "\n"
	}

	return msg
}

func objectTestShortResult(result *madmin.SpeedTestResult) (msg string) {
	msg += fmt.Sprintf("MinIO %s, %d servers, %d drives, %s objects, %d threads",
		result.Version, result.Servers, result.Disks,
		humanize.IBytes(uint64(result.Size)), result.Concurrent)

	return msg
}

// String - dummy function to confirm to the 'message' interface. Not used.
func (p PerfTestOutput) String() string {
	return ""
}

// JSON - jsonified output of the perf tests
func (p PerfTestOutput) JSON() string {
	JSONBytes, e := json.MarshalIndent(p, "", "    ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(JSONBytes)
}

var globalPerfTestVerbose bool

func mainSupportPerf(ctx *cli.Context) error {
	args := ctx.Args()

	// the alias parameter from cli
	aliasedURL := ""
	perfType := ""
	switch len(args) {
	case 1:
		// cannot use alias by the name 'drive' or 'net'
		if args[0] == "drive" || args[0] == "net" || args[0] == "object" {
			showCommandHelpAndExit(ctx, 1)
		}
		aliasedURL = args[0]

	case 2:
		perfType = args[0]
		aliasedURL = args[1]
	default:
		showCommandHelpAndExit(ctx, 1) // last argument is exit code
	}

	// Main execution
	execSupportPerf(ctx, aliasedURL, perfType)

	return nil
}

func convertDriveTestResult(dr madmin.DriveSpeedTestResult) DriveTestResult {
	return DriveTestResult{
		Endpoint:  dr.Endpoint,
		DrivePerf: dr.DrivePerf,
		Error:     dr.Error,
	}
}

func convertDriveTestResults(driveResults []madmin.DriveSpeedTestResult) []DriveTestResult {
	results := []DriveTestResult{}
	for _, dr := range driveResults {
		results = append(results, convertDriveTestResult(dr))
	}
	return results
}

func convertObjStatServers(ss []madmin.SpeedTestStatServer) []ObjStatServer {
	out := []ObjStatServer{}
	for _, s := range ss {
		out = append(out, ObjStatServer{
			Endpoint:      s.Endpoint,
			Throughput:    s.ThroughputPerSec,
			ObjectsPerSec: s.ObjectsPerSec,
			Error:         s.Err,
		})
	}
	return out
}

func convertPUTStats(stats madmin.SpeedTestStats) ObjPUTStats {
	return ObjPUTStats{
		Throughput:    stats.ThroughputPerSec,
		ObjectsPerSec: stats.ObjectsPerSec,
		Response:      stats.Response,
		Servers:       convertObjStatServers(stats.Servers),
	}
}

func convertGETStats(stats madmin.SpeedTestStats) ObjGETStats {
	return ObjGETStats{
		ObjPUTStats: convertPUTStats(stats),
		TTFB:        stats.TTFB,
	}
}

func convertObjTestResult(objResult *madmin.SpeedTestResult) ObjTestResult {
	if objResult == nil {
		return ObjTestResult{}
	}
	result := ObjTestResult{
		Servers:    objResult.Servers,
		Drives:     objResult.Disks,
		Size:       objResult.Size,
		Concurrent: objResult.Concurrent,
	}
	result.PUTStats = convertPUTStats(objResult.PUTStats)
	result.GETStats = convertGETStats(objResult.GETStats)
	return result
}

func updatePerfOutput(r PerfTestResult, out *PerfTestOutput) {
	switch r.Type {
	case DrivePerfTest:
		out.DriveResult = convertDriveTestResults(r.DriveResult)
	case ObjectPerfTest:
		or := convertObjTestResult(r.ObjectResult)
		out.ObjectResult = &or
	case NetPerfTest:
		out.NetResult = r.NetResult
	default:
		fatalIf(errDummy().Trace(), fmt.Sprintf("Invalid test type %d", r.Type))
	}
}

func convertPerfResult(r PerfTestResult) PerfTestOutput {
	out := PerfTestOutput{}
	updatePerfOutput(r, &out)
	return out
}

func convertPerfResults(results []PerfTestResult) PerfTestOutput {
	out := PerfTestOutput{}
	for _, r := range results {
		updatePerfOutput(r, &out)
	}
	return out
}

func execSupportPerf(ctx *cli.Context, aliasedURL string, perfType string) {
	alias, apiKey := initSubnetConnectivity(ctx, aliasedURL, true)
	if len(apiKey) == 0 {
		// api key not passed as flag. Check that the cluster is registered.
		apiKey = validateClusterRegistered(alias, true)
	}

	results := runPerfTests(ctx, aliasedURL, perfType)
	if globalJSON {
		// No file to be saved or uploaded to SUBNET in case of `--json`
		return
	}

	resultFileNamePfx := fmt.Sprintf("%s-perf_%s", filepath.Clean(alias), UTCNow().Format("20060102150405"))
	resultFileName := resultFileNamePfx + ".json"

	regInfo := getClusterRegInfo(getAdminInfo(aliasedURL), alias)
	tmpFileName, e := zipPerfResult(convertPerfResults(results), resultFileName, regInfo)
	fatalIf(probe.NewError(e), "Error creating zip from perf test results:")

	if globalAirgapped {
		savePerfResultFile(tmpFileName, resultFileNamePfx, alias)
		return
	}

	uploadURL := subnetUploadURL("perf", tmpFileName)
	reqURL, headers := prepareSubnetUploadURL(uploadURL, alias, tmpFileName, apiKey)

	_, e = uploadFileToSubnet(alias, tmpFileName, reqURL, headers)
	if e != nil {
		console.Errorln("Unable to upload perf test results to SUBNET portal: " + e.Error())
		savePerfResultFile(tmpFileName, resultFileNamePfx, alias)
		return
	}

	clr := color.New(color.FgGreen, color.Bold)
	clr.Println("uploaded successfully to SUBNET.")
}

func savePerfResultFile(tmpFileName string, resultFileNamePfx string, alias string) {
	zipFileName := resultFileNamePfx + ".zip"
	e := moveFile(tmpFileName, zipFileName)
	fatalIf(probe.NewError(e), fmt.Sprintf("Error moving temp file %s to %s:", tmpFileName, zipFileName))
	console.Infoln("MinIO performance report saved at", zipFileName)
}

func runPerfTests(ctx *cli.Context, aliasedURL string, perfType string) []PerfTestResult {
	resultCh := make(chan PerfTestResult)
	results := []PerfTestResult{}
	defer close(resultCh)

	tests := []string{perfType}
	if len(perfType) == 0 {
		// by default run all tests
		tests = []string{"net", "drive", "object"}
	}

	for _, t := range tests {
		switch t {
		case "drive":
			mainAdminSpeedTestDrive(ctx, aliasedURL, resultCh)
		case "object":
			mainAdminSpeedTestObject(ctx, aliasedURL, resultCh)
		case "net":
			mainAdminSpeedTestNetperf(ctx, aliasedURL, resultCh)
		default:
			showCommandHelpAndExit(ctx, 1) // last argument is exit code
		}

		if !globalJSON {
			results = append(results, <-resultCh)
		}
	}

	return results
}

func writeJSONObjToZip(zipWriter *zip.Writer, obj interface{}, filename string) error {
	writer, e := zipWriter.Create(filename)
	if e != nil {
		return e
	}

	enc := gojson.NewEncoder(writer)
	if e = enc.Encode(obj); e != nil {
		return e
	}

	return nil
}

// compress MinIO performance output
func zipPerfResult(perfOutput PerfTestOutput, resultFilename string, regInfo ClusterRegistrationInfo) (string, error) {
	// Create profile zip file
	tmpArchive, e := os.CreateTemp("", "mc-perf-")

	if e != nil {
		return "", e
	}
	defer tmpArchive.Close()

	zipWriter := zip.NewWriter(tmpArchive)
	defer zipWriter.Close()

	e = writeJSONObjToZip(zipWriter, perfOutput, resultFilename)
	if e != nil {
		return "", e
	}

	e = writeJSONObjToZip(zipWriter, regInfo, "cluster.info")
	if e != nil {
		return "", e
	}

	return tmpArchive.Name(), nil
}
<commit_msg>More improvements in format of perf test results (#4350)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"archive/zip"
	gojson "encoding/json"
	"fmt"
	"os"
	"path/filepath"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var supportPerfFlags = append([]cli.Flag{
	cli.StringFlag{
		Name:  "duration",
		Usage: "duration the entire perf tests are run",
		Value: "10s",
	},
	cli.BoolFlag{
		Name:  "verbose, v",
		Usage: "display per-server stats",
	},
	cli.StringFlag{
		Name:   "size",
		Usage:  "size of the object used for uploads/downloads",
		Value:  "64MiB",
		Hidden: true,
	},
	cli.IntFlag{
		Name:   "concurrent",
		Usage:  "number of concurrent requests per server",
		Value:  32,
		Hidden: true,
	},
	cli.StringFlag{
		Name:   "bucket",
		Usage:  "provide a custom bucket name to use (NOTE: bucket must be created prior)",
		Hidden: true, // Hidden for now.
	},
	// Drive test specific flags.
	cli.StringFlag{
		Name:   "filesize",
		Usage:  "total amount of data read/written to each drive",
		Value:  "1GiB",
		Hidden: true,
	},
	cli.StringFlag{
		Name:   "blocksize",
		Usage:  "read/write block size",
		Value:  "4MiB",
		Hidden: true,
	},
	cli.BoolFlag{
		Name:   "serial",
		Usage:  "run tests on drive(s) one-by-one",
		Hidden: true,
	},
}, subnetCommonFlags...)

var supportPerfCmd = cli.Command{
	Name:            "perf",
	Usage:           "upload object, network and drive performance analysis",
	Action:          mainSupportPerf,
	OnUsageError:    onUsageError,
	Before:          setGlobalsFromContext,
	Flags:           append(supportPerfFlags, supportGlobalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [COMMAND] [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}

EXAMPLES:
  1. Upload object storage, network, and drive performance analysis for cluster with alias 'myminio' to SUBNET
     {{.Prompt}} {{.HelpName}} myminio
  2. Run object storage, network, and drive performance tests on cluster with alias 'myminio', save and upload to SUBNET manually
     {{.Prompt}} {{.HelpName}} --airgap myminio
`,
}

// PerfTestOutput - stores the final output of performance test(s)
type PerfTestOutput struct {
	ObjectResults *ObjTestResults   `json:"object,omitempty"`
	NetResults    *NetTestResults   `json:"network,omitempty"`
	DriveResults  *DriveTestResults `json:"drive,omitempty"`
	Error         string            `json:"error,omitempty"`
}

// DriveTestResult - result of the drive performance test on a given endpoint
type DriveTestResult struct {
	Endpoint string             `json:"endpoint"`
	Perf     []madmin.DrivePerf `json:"perf,omitempty"`
	Error    string             `json:"error,omitempty"`
}

// DriveTestResults - results of drive performance test across all endpoints
type DriveTestResults struct {
	Results []DriveTestResult `json:"servers"`
}

// ObjTestResults - result of the object performance test
type ObjTestResults struct {
	ObjectSize int               `json:"objectSize"`
	Threads    int               `json:"threads"`
	PUTResults ObjPUTPerfResults `json:"PUT"`
	GETResults ObjGETPerfResults `json:"GET"`
}

// ObjStats - Object performance stats
type ObjStats struct {
	Throughput    uint64 `json:"throughput"`
	ObjectsPerSec uint64 `json:"objectsPerSec"`
}

// ObjStatServer - Server level object performance stats
type ObjStatServer struct {
	Endpoint string   `json:"endpoint"`
	Perf     ObjStats `json:"perf"`
	Error    string   `json:"error,omitempty"`
}

// ObjPUTPerfResults - Object PUT performance results
type ObjPUTPerfResults struct {
	Perf    ObjPUTStats     `json:"perf"`
	Servers []ObjStatServer `json:"servers"`
}

// ObjPUTStats - PUT stats of all the servers
type ObjPUTStats struct {
	Throughput    uint64         `json:"throughput"`
	ObjectsPerSec uint64         `json:"objectsPerSec"`
	Response      madmin.Timings `json:"responseTime"`
}

// ObjGETPerfResults - Object GET performance results
type ObjGETPerfResults struct {
	Perf    ObjGETStats     `json:"perf"`
	Servers []ObjStatServer `json:"servers"`
}

// ObjGETStats - GET stats of all the servers
type ObjGETStats struct {
	ObjPUTStats
	TTFB madmin.Timings `json:"ttfb,omitempty"`
}

// NetStats - Network performance stats
type NetStats struct {
	TX uint64 `json:"tx"`
	RX uint64 `json:"rx"`
}

// NetTestResult - result of the network performance test for given endpoint
type NetTestResult struct {
	Endpoint string   `json:"endpoint"`
	Perf     NetStats `json:"perf"`
	Error    string   `json:"error,omitempty"`
}

// NetTestResults - result of the network performance test across all endpoints
type NetTestResults struct {
	Results []NetTestResult `json:"servers"`
}

func objectTestVerboseResult(result *madmin.SpeedTestResult) (msg string) {
	msg += "PUT:\n"
	for _, node := range result.PUTStats.Servers {
		msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
		if node.Err != "" {
			msg += " Err: " + node.Err
		}
		msg += "\n"
	}

	msg += "GET:\n"
	for _, node := range result.GETStats.Servers {
		msg += fmt.Sprintf("   * %s: %s/s %s objs/s", node.Endpoint, humanize.IBytes(node.ThroughputPerSec), humanize.Comma(int64(node.ObjectsPerSec)))
		if node.Err != "" {
			msg += " Err: " + node.Err
		}
		msg += "\n"
	}

	return msg
}

func objectTestShortResult(result *madmin.SpeedTestResult) (msg string) {
	msg += fmt.Sprintf("MinIO %s, %d servers, %d drives, %s objects, %d threads",
		result.Version, result.Servers, result.Disks,
		humanize.IBytes(uint64(result.Size)), result.Concurrent)

	return msg
}

// String - dummy function to confirm to the 'message' interface. Not used.
func (p PerfTestOutput) String() string {
	return ""
}

// JSON - jsonified output of the perf tests
func (p PerfTestOutput) JSON() string {
	JSONBytes, e := json.MarshalIndent(p, "", "    ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(JSONBytes)
}

var globalPerfTestVerbose bool

func mainSupportPerf(ctx *cli.Context) error {
	args := ctx.Args()

	// the alias parameter from cli
	aliasedURL := ""
	perfType := ""
	switch len(args) {
	case 1:
		// cannot use alias by the name 'drive' or 'net'
		if args[0] == "drive" || args[0] == "net" || args[0] == "object" {
			showCommandHelpAndExit(ctx, 1)
		}
		aliasedURL = args[0]

	case 2:
		perfType = args[0]
		aliasedURL = args[1]
	default:
		showCommandHelpAndExit(ctx, 1) // last argument is exit code
	}

	// Main execution
	execSupportPerf(ctx, aliasedURL, perfType)

	return nil
}

func convertDriveTestResult(dr madmin.DriveSpeedTestResult) DriveTestResult {
	return DriveTestResult{
		Endpoint: dr.Endpoint,
		Perf:     dr.DrivePerf,
		Error:    dr.Error,
	}
}

func convertDriveTestResults(driveResults []madmin.DriveSpeedTestResult) *DriveTestResults {
	if driveResults == nil {
		return nil
	}
	results := []DriveTestResult{}
	for _, dr := range driveResults {
		results = append(results, convertDriveTestResult(dr))
	}
	r := DriveTestResults{
		Results: results,
	}
	return &r
}

func convertNetTestResults(netResults *madmin.NetperfResult) *NetTestResults {
	if netResults == nil {
		return nil
	}
	results := []NetTestResult{}
	for _, nr := range netResults.NodeResults {
		results = append(results, NetTestResult{
			Endpoint: nr.Endpoint,
			Error:    nr.Error,
			Perf: NetStats{
				TX: nr.TX,
				RX: nr.RX,
			},
		})
	}
	r := NetTestResults{
		Results: results,
	}
	return &r
}

func convertObjStatServers(ss []madmin.SpeedTestStatServer) []ObjStatServer {
	out := []ObjStatServer{}
	for _, s := range ss {
		out = append(out, ObjStatServer{
			Endpoint: s.Endpoint,
			Perf: ObjStats{
				Throughput:    s.ThroughputPerSec,
				ObjectsPerSec: s.ObjectsPerSec,
			},
			Error: s.Err,
		})
	}
	return out
}

func convertPUTStats(stats madmin.SpeedTestStats) ObjPUTStats {
	return ObjPUTStats{
		Throughput:    stats.ThroughputPerSec,
		ObjectsPerSec: stats.ObjectsPerSec,
		Response:      stats.Response,
	}
}

func convertPUTResults(stats madmin.SpeedTestStats) ObjPUTPerfResults {
	return ObjPUTPerfResults{
		Perf:    convertPUTStats(stats),
		Servers: convertObjStatServers(stats.Servers),
	}
}

func convertGETResults(stats madmin.SpeedTestStats) ObjGETPerfResults {
	return ObjGETPerfResults{
		Perf: ObjGETStats{
			ObjPUTStats: convertPUTStats(stats),
			TTFB:        stats.TTFB,
		},
		Servers: convertObjStatServers(stats.Servers),
	}
}

func convertObjTestResults(objResult *madmin.SpeedTestResult) *ObjTestResults {
	if objResult == nil {
		return nil
	}
	result := ObjTestResults{
		ObjectSize: objResult.Size,
		Threads:    objResult.Concurrent,
	}
	result.PUTResults = convertPUTResults(objResult.PUTStats)
	result.GETResults = convertGETResults(objResult.GETStats)
	return &result
}

func updatePerfOutput(r PerfTestResult, out *PerfTestOutput) {
	switch r.Type {
	case DrivePerfTest:
		out.DriveResults = convertDriveTestResults(r.DriveResult)
	case ObjectPerfTest:
		out.ObjectResults = convertObjTestResults(r.ObjectResult)
	case NetPerfTest:
		out.NetResults = convertNetTestResults(r.NetResult)
	default:
		fatalIf(errDummy().Trace(), fmt.Sprintf("Invalid test type %d", r.Type))
	}
}

func convertPerfResult(r PerfTestResult) PerfTestOutput {
	out := PerfTestOutput{}
	updatePerfOutput(r, &out)
	return out
}

func convertPerfResults(results []PerfTestResult) PerfTestOutput {
	out := PerfTestOutput{}
	for _, r := range results {
		updatePerfOutput(r, &out)
	}
	return out
}

func execSupportPerf(ctx *cli.Context, aliasedURL string, perfType string) {
	alias, apiKey := initSubnetConnectivity(ctx, aliasedURL, true)
	if len(apiKey) == 0 {
		// api key not passed as flag. Check that the cluster is registered.
		apiKey = validateClusterRegistered(alias, true)
	}

	results := runPerfTests(ctx, aliasedURL, perfType)
	if globalJSON {
		// No file to be saved or uploaded to SUBNET in case of `--json`
		return
	}

	resultFileNamePfx := fmt.Sprintf("%s-perf_%s", filepath.Clean(alias), UTCNow().Format("20060102150405"))
	resultFileName := resultFileNamePfx + ".json"

	regInfo := getClusterRegInfo(getAdminInfo(aliasedURL), alias)
	tmpFileName, e := zipPerfResult(convertPerfResults(results), resultFileName, regInfo)
	fatalIf(probe.NewError(e), "Error creating zip from perf test results:")

	if globalAirgapped {
		savePerfResultFile(tmpFileName, resultFileNamePfx, alias)
		return
	}

	uploadURL := subnetUploadURL("perf", tmpFileName)
	reqURL, headers := prepareSubnetUploadURL(uploadURL, alias, tmpFileName, apiKey)

	_, e = uploadFileToSubnet(alias, tmpFileName, reqURL, headers)
	if e != nil {
		console.Errorln("Unable to upload perf test results to SUBNET portal: " + e.Error())
		savePerfResultFile(tmpFileName, resultFileNamePfx, alias)
		return
	}

	clr := color.New(color.FgGreen, color.Bold)
	clr.Println("uploaded successfully to SUBNET.")
}

func savePerfResultFile(tmpFileName string, resultFileNamePfx string, alias string) {
	zipFileName := resultFileNamePfx + ".zip"
	e := moveFile(tmpFileName, zipFileName)
	fatalIf(probe.NewError(e), fmt.Sprintf("Error moving temp file %s to %s:", tmpFileName, zipFileName))
	console.Infoln("MinIO performance report saved at", zipFileName)
}

func runPerfTests(ctx *cli.Context, aliasedURL string, perfType string) []PerfTestResult {
	resultCh := make(chan PerfTestResult)
	results := []PerfTestResult{}
	defer close(resultCh)

	tests := []string{perfType}
	if len(perfType) == 0 {
		// by default run all tests
		tests = []string{"net", "drive", "object"}
	}

	for _, t := range tests {
		switch t {
		case "drive":
			mainAdminSpeedTestDrive(ctx, aliasedURL, resultCh)
		case "object":
			mainAdminSpeedTestObject(ctx, aliasedURL, resultCh)
		case "net":
			mainAdminSpeedTestNetperf(ctx, aliasedURL, resultCh)
		default:
			showCommandHelpAndExit(ctx, 1) // last argument is exit code
		}

		if !globalJSON {
			results = append(results, <-resultCh)
		}
	}

	return results
}

func writeJSONObjToZip(zipWriter *zip.Writer, obj interface{}, filename string) error {
	writer, e := zipWriter.Create(filename)
	if e != nil {
		return e
	}

	enc := gojson.NewEncoder(writer)
	if e = enc.Encode(obj); e != nil {
		return e
	}

	return nil
}

// compress MinIO performance output
func zipPerfResult(perfOutput PerfTestOutput, resultFilename string, regInfo ClusterRegistrationInfo) (string, error) {
	// Create profile zip file
	tmpArchive, e := os.CreateTemp("", "mc-perf-")

	if e != nil {
		return "", e
	}
	defer tmpArchive.Close()

	zipWriter := zip.NewWriter(tmpArchive)
	defer zipWriter.Close()

	e = writeJSONObjToZip(zipWriter, perfOutput, resultFilename)
	if e != nil {
		return "", e
	}

	e = writeJSONObjToZip(zipWriter, regInfo, "cluster.info")
	if e != nil {
		return "", e
	}

	return tmpArchive.Name(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"gopkg.in/alecthomas/kingpin.v2"
)

const syslModuleName = "syslmodules"

type modCmd struct {
	subcommand string
}

func (m *modCmd) Name() string       { return "mod" }
func (m *modCmd) MaxSyslModule() int { return 0 }

func (m *modCmd) Configure(app *kingpin.Application) *kingpin.CmdClause {
	opts := []string{"init"}
	cmd := app.Command(m.Name(), "Configure sysl module system")
	cmd.Arg("command", fmt.Sprintf("commands: [%s]", strings.Join(opts, ","))).Required().EnumVar(&m.subcommand, opts...)

	return cmd
}

func (m *modCmd) Execute(args ExecuteArgs) error {
	if m.subcommand == "init" {
		// ignore folder creation error
		args.Filesystem.Mkdir(syslRootMarker, 0755)
		err := os.Chdir(syslRootMarker)
		if err != nil {
			return err
		}
		out, err := exec.Command("go", "mod", "init", syslModuleName).CombinedOutput()
		if err != nil {
			return err
		}
		args.Logger.Debug(out)
	}
	return nil
}
<commit_msg>Add warning on folder creation issue<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"gopkg.in/alecthomas/kingpin.v2"
)

const syslModuleName = "syslmodules"

type modCmd struct {
	subcommand string
}

func (m *modCmd) Name() string       { return "mod" }
func (m *modCmd) MaxSyslModule() int { return 0 }

func (m *modCmd) Configure(app *kingpin.Application) *kingpin.CmdClause {
	opts := []string{"init"}
	cmd := app.Command(m.Name(), "Configure sysl module system")
	cmd.Arg("command", fmt.Sprintf("commands: [%s]", strings.Join(opts, ","))).Required().EnumVar(&m.subcommand, opts...)

	return cmd
}

func (m *modCmd) Execute(args ExecuteArgs) error {
	if m.subcommand == "init" {
		// ignore folder creation error
		_err := args.Filesystem.Mkdir(syslRootMarker, 0755)
		if _err != nil {
			args.Logger.Warn(_err.Error())
		}
		err := os.Chdir(syslRootMarker)
		if err != nil {
			return err
		}
		out, err := exec.Command("go", "mod", "init", syslModuleName).CombinedOutput()
		if err != nil {
			return err
		}
		args.Logger.Debug(out)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"github.com/spf13/cobra"

	"github.com/coreos/torus"
	"github.com/coreos/torus/block"
	"github.com/coreos/torus/block/aoe"
)

var aoeCommand = &cobra.Command{
	Use:   "aoe VOLUME INTERFACE MAJOR MINOR",
	Short: "serve a volume over AoE",
	Long: strings.TrimSpace(`
Serve a volume over AoE using the specified network interface and AoE
major and minor addresses.

It is important to note that all AoE servers on the same layer 2 network
must have different major and minor addresses.

An example of serving two volumes using AoE on the same server over the
eth0 network interface:

	torusblk aoe vol01 eth0 1 1
	torusblk aoe vol02 eth0 1 2
`),
	Run: aoeAction,
}

func aoeAction(cmd *cobra.Command, args []string) {
	if len(args) != 4 {
		cmd.Usage()
		os.Exit(1)
	}

	srv := createServer()

	vol := args[0]
	ifname := args[1]
	maj := args[2]
	min := args[3]

	major, err := strconv.ParseUint(maj, 10, 16)
	if err != nil {
		die("Failed to parse major address %q: %v\n", maj, err)
	}

	minor, err := strconv.ParseUint(min, 10, 8)
	if err != nil {
		die("Failed to parse minor address %q: %v\n", min, err)
	}

	blockvol, err := block.OpenBlockVolume(srv, vol)
	if err != nil {
		fmt.Println("server doesn't support block volumes:", err)
		os.Exit(1)
	}

	ai, err := aoe.NewInterface(ifname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to set up interface %q: %v\n", ifname, err)
		os.Exit(1)
	}

	as, err := aoe.NewServer(blockvol, &aoe.ServerOptions{
		Major: uint16(major),
		Minor: uint8(minor),
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to crate AoE server: %v\n", err)
		os.Exit(1)
	}

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	go func(sv *torus.Server, iface *aoe.Interface) {
		for _ = range signalChan {
			fmt.Println("\nReceived an interrupt, stopping services...")

			iface.Close()
			sv.Close()
			as.Close()
			os.Exit(0)
		}
	}(srv, ai)

	if err = as.Serve(ai); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to serve AoE: %v\n", err)
		os.Exit(1)
	}
}
<commit_msg>torusblk.aoe: return aoe functions before calling os.Exit(1)<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"github.com/spf13/cobra"

	"github.com/coreos/torus"
	"github.com/coreos/torus/block"
	"github.com/coreos/torus/block/aoe"
)

var aoeCommand = &cobra.Command{
	Use:   "aoe VOLUME INTERFACE MAJOR MINOR",
	Short: "serve a volume over AoE",
	Long: strings.TrimSpace(`
Serve a volume over AoE using the specified network interface and AoE
major and minor addresses.

It is important to note that all AoE servers on the same layer 2 network
must have different major and minor addresses.

An example of serving two volumes using AoE on the same server over the
eth0 network interface:

	torusblk aoe vol01 eth0 1 1
	torusblk aoe vol02 eth0 1 2
`),
	Run: func(cmd *cobra.Command, args []string) {
		err := aoeAction(cmd, args)
		if err == torus.ErrUsage {
			cmd.Usage()
			os.Exit(1)
		} else if err != nil {
			die("%v", err)
		}
	},
}

func aoeAction(cmd *cobra.Command, args []string) error {
	if len(args) != 4 {
		return torus.ErrUsage
	}

	srv := createServer()

	vol := args[0]
	ifname := args[1]
	maj := args[2]
	min := args[3]

	major, err := strconv.ParseUint(maj, 10, 16)
	if err != nil {
		return fmt.Errorf("Failed to parse major address %q: %v", maj, err)
	}

	minor, err := strconv.ParseUint(min, 10, 8)
	if err != nil {
		return fmt.Errorf("Failed to parse minor address %q: %v", min, err)
	}

	blockvol, err := block.OpenBlockVolume(srv, vol)
	if err != nil {
		return fmt.Errorf("server doesn't support block volumes: %v", err)
	}

	ai, err := aoe.NewInterface(ifname)
	if err != nil {
		return fmt.Errorf("Failed to set up interface %q: %v", ifname, err)
	}

	as, err := aoe.NewServer(blockvol, &aoe.ServerOptions{
		Major: uint16(major),
		Minor: uint8(minor),
	})
	if err != nil {
		return fmt.Errorf("Failed to crate AoE server: %v", err)
	}

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	go func(sv *torus.Server, iface *aoe.Interface) {
		for _ = range signalChan {
			fmt.Println("\nReceived an interrupt, stopping services...")

			iface.Close()
			sv.Close()
			as.Close()
			os.Exit(0)
		}
	}(srv, ai)

	if err = as.Serve(ai); err != nil {
		return fmt.Errorf("Failed to serve AoE: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"fmt"
	"github.com/montanaflynn/stats"
	"gopkg.in/urfave/cli.v1"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"unicode/utf8"
)

type cpuStat struct {
	values map[string][]float64
}

type interruptStat struct {
	values map[string][]float64
}

var reportCommand = cli.Command{
	Name:      "report",
	Usage:     "summarise stats that are recored to a file",
	ArgsUsage: "[file]",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter",
			Usage: "delimiter used in the output file",
			Value: "\t",
		},
	},
	Action: reportAction,
}

func reportAction(ctx *cli.Context) error {
	args := ctx.Args()
	if len(args) == 0 {
		return cli.NewExitError(fmt.Sprintf("No stats file specified"), 3)
	}
	filename := args[0]
	delimiter := ctx.String("delimiter")
	file, err := os.Open(filename)
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to open file: %v", err), 2)
	}
	defer file.Close()
	reader := csv.NewReader(file)
	comma, _ := utf8.DecodeRuneInString(delimiter)
	reader.Comma = comma
	header := map[string]int{}
	cpuStats := map[string]cpuStat{}
	interruptStats := map[string]interruptStat{}
	sampleCount := 0
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if len(record) == 0 {
			continue
		}
		if strings.HasPrefix(record[0], "#") {
			continue
		}
		if len(header) == 0 {
			for idx, column := range record {
				header[column] = idx
			}
			continue
		}
		for column, idx := range header {
			result := strings.Split(column, ".")
			resource := result[0]
			if strings.HasPrefix(resource, "cpu") {
				rawValue := record[idx]
				value, err := strconv.Atoi(rawValue)
				if err != nil {
					return cli.NewExitError(fmt.Sprintf("Unable to parse value '%s': %v", rawValue, err), 2)
				}
				class := result[1]
				stat, ok := cpuStats[resource]
				if !ok {
					stat = cpuStat{values: map[string][]float64{}}
				}
				values := stat.values[class]
				values = append(values, float64(value))
				stat.values[class] = values
				cpuStats[resource] = stat
			}
			if strings.HasPrefix(resource, "int") {
				rawValue := record[idx]
				value, err := strconv.Atoi(rawValue)
				if err != nil {
					return cli.NewExitError(fmt.Sprintf("Unable to parse value '%s': %v", rawValue, err), 2)
				}
				class := result[1]
				stat, ok := interruptStats[resource]
				if !ok {
					stat = interruptStat{values: map[string][]float64{}}
				}
				values := stat.values[class]
				values = append(values, float64(value))
				stat.values[class] = values
				interruptStats[resource] = stat
			}
		}
		sampleCount++
	}
	fmt.Printf("Processing %s ...\n", filename)
	fmt.Printf("\n")
	fmt.Printf("N = %d\n", sampleCount)
	cpus := []string{}
	for cpu, _ := range cpuStats {
		cpus = append(cpus, cpu)
	}
	sort.Strings(cpus)
	classes := []string{"system", "usr", "nice", "irq", "softirq", "iowait", "guest", "guestnice", "steal", "idle"}
	fmt.Printf("\n")
	fmt.Printf("CPU utilization, mean (SD):\n")
	fmt.Printf("\n")
	fmt.Printf("      ")
	for _, class := range classes {
		fmt.Printf(" %-12s", class)
	}
	fmt.Printf("\n")
	for _, cpu := range cpus {
		fmt.Printf("  %-4s", cpu)
		cpuStats := cpuStats[cpu]
		for _, class := range classes {
			values := cpuStats.values[class]
			mean, err := stats.Mean(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			stddev, err := stats.StandardDeviation(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			format := fmt.Sprintf("%.2f (%.2f)", mean, stddev)
			fmt.Printf(" %-12s", format)
		}
		fmt.Printf("\n")
	}
	fmt.Printf("\n")
	if err := printInterrupts("Interrupts", interruptStats, cpus); err != nil {
		return err
	}
	fmt.Printf("\n")
	return nil
}

func printInterrupts(title string, interruptStats map[string]interruptStat, cpus []string) error {
	interrupts := []string{}
	for interrupt, _ := range interruptStats {
		interrupts = append(interrupts, interrupt)
	}
	sort.Strings(interrupts)
	fmt.Printf("%s, mean (SD):\n", title)
	fmt.Printf("\n")
	fmt.Printf("  %-10s", "interrupt")
	for _, cpu := range cpus {
		if cpu == "cpu" {
			continue
		}
		fmt.Printf("%20s", cpu)
	}
	fmt.Printf("\n")
	for _, intr := range interrupts {
		interruptStat := interruptStats[intr]
		fmt.Printf("  %-10s", intr)
		for _, cpu := range cpus {
			if cpu == "cpu" {
				continue
			}
			values, ok := interruptStat.values[cpu]
			if !ok {
				fmt.Printf("%20s", "")
				continue
			}
			mean, err := stats.Mean(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			stddev, err := stats.StandardDeviation(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			format := fmt.Sprintf("%.2f (%.2f)", mean, stddev)
			fmt.Printf("%20s", format)
		}
		fmt.Printf("\n")
	}
	return nil
}
<commit_msg>report: Add support for softirqs<commit_after>package main

import (
	"encoding/csv"
	"fmt"
	"github.com/montanaflynn/stats"
	"gopkg.in/urfave/cli.v1"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"unicode/utf8"
)

type cpuStat struct {
	values map[string][]float64
}

type interruptStat struct {
	values map[string][]float64
}

var reportCommand = cli.Command{
	Name:      "report",
	Usage:     "summarise stats that are recored to a file",
	ArgsUsage: "[file]",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "delimiter",
			Usage: "delimiter used in the output file",
			Value: "\t",
		},
	},
	Action: reportAction,
}

func reportAction(ctx *cli.Context) error {
	args := ctx.Args()
	if len(args) == 0 {
		return cli.NewExitError(fmt.Sprintf("No stats file specified"), 3)
	}
	filename := args[0]
	delimiter := ctx.String("delimiter")
	file, err := os.Open(filename)
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("Unable to open file: %v", err), 2)
	}
	defer file.Close()
	reader := csv.NewReader(file)
	comma, _ := utf8.DecodeRuneInString(delimiter)
	reader.Comma = comma
	header := map[string]int{}
	cpuStats := map[string]cpuStat{}
	interruptStats := map[string]interruptStat{}
	softIrqStats := map[string]interruptStat{}
	sampleCount := 0
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if len(record) == 0 {
			continue
		}
		if strings.HasPrefix(record[0], "#") {
			continue
		}
		if len(header) == 0 {
			for idx, column := range record {
				header[column] = idx
			}
			continue
		}
		for column, idx := range header {
			result := strings.Split(column, ".")
			resource := result[0]
			if strings.HasPrefix(resource, "cpu") {
				rawValue := record[idx]
				value, err := strconv.Atoi(rawValue)
				if err != nil {
					return cli.NewExitError(fmt.Sprintf("Unable to parse value '%s': %v", rawValue, err), 2)
				}
				class := result[1]
				stat, ok := cpuStats[resource]
				if !ok {
					stat = cpuStat{values: map[string][]float64{}}
				}
				values := stat.values[class]
				values = append(values, float64(value))
				stat.values[class] = values
				cpuStats[resource] = stat
			}
			if strings.HasPrefix(resource, "int") {
				rawValue := record[idx]
				value, err := strconv.Atoi(rawValue)
				if err != nil {
					return cli.NewExitError(fmt.Sprintf("Unable to parse value '%s': %v", rawValue, err), 2)
				}
				class := result[1]
				stat, ok := interruptStats[resource]
				if !ok {
					stat = interruptStat{values: map[string][]float64{}}
				}
				values := stat.values[class]
				values = append(values, float64(value))
				stat.values[class] = values
				interruptStats[resource] = stat
			}
			if strings.HasPrefix(resource, "softirq") {
				resource := result[1]
				rawValue := record[idx]
				value, err := strconv.Atoi(rawValue)
				if err != nil {
					return cli.NewExitError(fmt.Sprintf("Unable to parse value '%s': %v", rawValue, err), 2)
				}
				class := result[2]
				stat, ok := softIrqStats[resource]
				if !ok {
					stat = interruptStat{values: map[string][]float64{}}
				}
				values := stat.values[class]
				values = append(values, float64(value))
				stat.values[class] = values
				softIrqStats[resource] = stat
			}
		}
		sampleCount++
	}
	fmt.Printf("Processing %s ...\n", filename)
	fmt.Printf("\n")
	fmt.Printf("N = %d\n", sampleCount)
	cpus := []string{}
	for cpu, _ := range cpuStats {
		cpus = append(cpus, cpu)
	}
	sort.Strings(cpus)
	classes := []string{"system", "usr", "nice", "irq", "softirq", "iowait", "guest", "guestnice", "steal", "idle"}
	fmt.Printf("\n")
	fmt.Printf("CPU utilization, mean (SD):\n")
	fmt.Printf("\n")
	fmt.Printf("      ")
	for _, class := range classes {
		fmt.Printf(" %-12s", class)
	}
	fmt.Printf("\n")
	for _, cpu := range cpus {
		fmt.Printf("  %-4s", cpu)
		cpuStats := cpuStats[cpu]
		for _, class := range classes {
			values := cpuStats.values[class]
			mean, err := stats.Mean(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			stddev, err := stats.StandardDeviation(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			format := fmt.Sprintf("%.2f (%.2f)", mean, stddev)
			fmt.Printf(" %-12s", format)
		}
		fmt.Printf("\n")
	}
	fmt.Printf("\n")
	if err := printInterrupts("Interrupts", interruptStats, cpus); err != nil {
		return err
	}
	fmt.Printf("\n")
	if err := printInterrupts("SoftIRQs", softIrqStats, cpus); err != nil {
		return err
	}
	fmt.Printf("\n")
	return nil
}

func printInterrupts(title string, interruptStats map[string]interruptStat, cpus []string) error {
	interrupts := []string{}
	for interrupt, _ := range interruptStats {
		interrupts = append(interrupts, interrupt)
	}
	sort.Strings(interrupts)
	fmt.Printf("%s, mean (SD):\n", title)
	fmt.Printf("\n")
	fmt.Printf("  %-10s", "interrupt")
	for _, cpu := range cpus {
		if cpu == "cpu" {
			continue
		}
		fmt.Printf("%20s", cpu)
	}
	fmt.Printf("\n")
	for _, intr := range interrupts {
		interruptStat := interruptStats[intr]
		fmt.Printf("  %-10s", intr)
		for _, cpu := range cpus {
			if cpu == "cpu" {
				continue
			}
			values, ok := interruptStat.values[cpu]
			if !ok {
				fmt.Printf("%20s", "")
				continue
			}
			mean, err := stats.Mean(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			stddev, err := stats.StandardDeviation(values)
			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Error '%s': %v", err), 2)
			}
			format := fmt.Sprintf("%.2f (%.2f)", mean, stddev)
			fmt.Printf("%20s", format)
		}
		fmt.Printf("\n")
	}
	return nil
}
<|endoftext|>


<commit_before>package cod

import (
	"errors"
	"github.com/adabei/goldenbot/events"
	"github.com/adabei/goldenbot/events/cod"
	"github.com/adabei/goldenbot/rcon"
)

type Commands struct {
	commands map[string]func()
	events   chan interface{}
	requests chan rcon.RCONQuery
}

func NewCommands(requests chan rcon.RCONQuery, ea events.Aggregator) *Commands {
	c := new(Commands)
	c.requests = requests
	c.events = ea.Subscribe(c)
	return c
}

func (c *Commands) Setup() error {
	return nil
}

func (c *Commands) Start() {
	for {
		ev := <-c.events
		if ev, ok := in.(cod.Say); ok {
			if cmd, ok := c.commands[ev.Message]; ok {
				cmd()
			}
		}
	}
}

func (c *Commands) Register(cmd string, fn func()) error {
	if _, dup := c.commands[cmd]; dup {
		return errors.New("Command %v already defined", cmd)
	}

	c.commands[cmd] = fn
	return nil
}
<commit_msg>fixes imports due to github name change<commit_after>package cod

import (
	"errors"
	"github.com/schwarz/goldenbot/events"
	"github.com/schwarz/goldenbot/events/cod"
	"github.com/schwarz/goldenbot/rcon"
)

type Commands struct {
	commands map[string]func()
	events   chan interface{}
	requests chan rcon.RCONQuery
}

func NewCommands(requests chan rcon.RCONQuery, ea events.Aggregator) *Commands {
	c := new(Commands)
	c.requests = requests
	c.events = ea.Subscribe(c)
	return c
}

func (c *Commands) Setup() error {
	return nil
}

func (c *Commands) Start() {
	for {
		ev := <-c.events
		if ev, ok := in.(cod.Say); ok {
			if cmd, ok := c.commands[ev.Message]; ok {
				cmd()
			}
		}
	}
}

func (c *Commands) Register(cmd string, fn func()) error {
	if _, dup := c.commands[cmd]; dup {
		return errors.New("Command %v already defined", cmd)
	}

	c.commands[cmd] = fn
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/st3sch/whycc/bankfile"
)

func main() {
	patterns := make(map[string]*string)
	patterns["ingdiba"] = flag.String("ingdiba", "", "Pattern for ING DiDba files")
	patterns["augusta"] = flag.String("augusta", "", "Pattern for Augusta Bank files")
	patterns["krspaka"] = flag.String("krspaka", "", "Pattern for Kreissparkasse Augsburg files")
	inputdir := flag.String("i", ".", "Input directory")
	outputdir := flag.String("o", ".", "Output directory")
	flag.Parse()
	fmt.Println("Inputdir: ", *inputdir)
	fmt.Println("Outputdir: ", *outputdir)

	converterLocator := bankfile.NewConverterLocator()
	for banktype, pattern := range patterns {
		fmt.Println("Banktype: ", banktype)
		fmt.Println("Pattern: ", *pattern)
		if *pattern == "" {
			continue
		}

		files, err := filepath.Glob(*inputdir + string(filepath.Separator) + *pattern)
		fmt.Println(files)
		if err != nil {
			log.Fatal(err)
		}

		conv, err := converterLocator.FindBy(banktype)
		if err != nil {
			log.Fatal(err)
		}

		for _, filename := range files {
			fmt.Println("File: ", filename)
			f, err := os.Open(filename)
			if err != nil {
				log.Fatal(err)
			}

			err = ConvertFile(f, os.Stdout, conv)
			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

func ConvertFile(in io.Reader, out io.Writer, c bankfile.Converter) error {
	r := csv.NewReader(in)
	r.Comma = c.Comma()
	r.FieldsPerRecord = -1

	w := csv.NewWriter(out)

	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if !c.IsTransaction(record) {
			continue
		}

		record = c.Convert(record)
		err = w.Write(record)
		if err != nil {
			return err
		}
		w.Flush()
	}

	return nil
}
<commit_msg>Write file to output dir<commit_after>package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"path"

	"github.com/st3sch/whycc/bankfile"
)

func main() {
	patterns := make(map[string]*string)
	patterns["ingdiba"] = flag.String("ingdiba", "", "Pattern for ING DiDba files")
	patterns["augusta"] = flag.String("augusta", "", "Pattern for Augusta Bank files")
	patterns["krspaka"] = flag.String("krspaka", "", "Pattern for Kreissparkasse Augsburg files")
	inputdir := flag.String("i", ".", "Input directory")
	outputdir := flag.String("o", ".", "Output directory")
	flag.Parse()
	fmt.Println("Inputdir: ", *inputdir)
	fmt.Println("Outputdir: ", *outputdir)

	converterLocator := bankfile.NewConverterLocator()
	for banktype, pattern := range patterns {
		fmt.Println("Banktype: ", banktype)
		fmt.Println("Pattern: ", *pattern)
		if *pattern == "" {
			continue
		}

		files, err := filepath.Glob(*inputdir + string(filepath.Separator) + *pattern)
		fmt.Println(files)
		if err != nil {
			log.Fatal(err)
		}

		conv, err := converterLocator.FindBy(banktype)
		if err != nil {
			log.Fatal(err)
		}

		for _, filename := range files {
			fmt.Println("File: ", filename)
			f, err := os.Open(filename)
			if err != nil {
				log.Fatal(err)
			}

			outfilename := *outputdir + string(filepath.Separator) + banktype + "_" + path.Base(filename)
			outfile, err := os.Create(outfilename)
			if err != nil {
				log.Fatal(err)
			}
			defer outfile.Close()

			err = ConvertFile(f, outfile, conv)
			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

func ConvertFile(in io.Reader, out io.Writer, c bankfile.Converter) error {
	r := csv.NewReader(in)
	r.Comma = c.Comma()
	r.FieldsPerRecord = -1

	w := csv.NewWriter(out)

	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		if !c.IsTransaction(record) {
			continue
		}

		record = c.Convert(record)
		err = w.Write(record)
		if err != nil {
			return err
		}
		w.Flush()
	}

	return nil
}
<|endoftext|>


<commit_before>package interfaces

import (
	"fmt"
	"net/http"

	"code.google.com/p/go.net/context"
	"github.com/janicduplessis/projectgo/ct/domain"
	"github.com/janicduplessis/projectgo/ct/usecases"
)

const (
	urlChat string = "/ct"
)

type ChatInteractor interface {
	JoinServer(userId int64) (*domain.Client, error)
	JoinChannel(userId int64, channelId int64) error
	Channels(clientId int64) (map[int64]*domain.Channel, error)
	CreateChannel(clientId int64, name string) error
	SendMessage(userId int64, body string) error
}

type SendMessageRequest struct {
	Message string
}

type JoinChannelRequest struct {
	ChannelId int64
}

type CreateChannelRequest struct {
	Name string
}

type ChannelsResponse struct {
	List []*domain.Channel
}

type SendMessageResponse struct {
	Body      string
	Author    string
	ChannelId int64
	ClientId  int64
}

type ChatWebserviceHandler struct {
	Webservice     Webservice
	Websocket      Websocket
	ChatInteractor ChatInteractor
}

type SenderHandler struct {
	Handler WebsocketClient
	Command WebsocketCommand
}

func NewChatWebservice(ws Webservice, wsocket Websocket, ci ChatInteractor) *ChatWebserviceHandler {
	wsHandler := &ChatWebserviceHandler{
		Webservice:     ws,
		Websocket:      wsocket,
		ChatInteractor: ci,
	}

	ws.AddHandler(urlChat, true, wsHandler.JoinServer)

	wsocket.AddHandler("SendMessage", wsHandler.SendMessage)
	wsocket.AddHandler("JoinChannel", wsHandler.JoinChannel)
	wsocket.AddHandler("CreateChannel", wsHandler.CreateChannel)
	wsocket.AddHandler("Channels", wsHandler.Channels)

	return wsHandler
}

func (handler *ChatWebserviceHandler) JoinServer(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	user := ctx.Value(KeyUser).(*usecases.User)

	client, err := handler.ChatInteractor.JoinServer(user.Id)
	if err != nil {
		handler.Webservice.Error(w, err)
		return
	}

	handler.Websocket.AddClient(ctx, w, r, client)
}

func (handler *ChatWebserviceHandler) SendMessage(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	request := SendMessageRequest{}
	err := client.ReadJson(cmd, &request)
	if err != nil {
		client.Error(cmd, err)
		return
	}

	user := ctx.Value(KeyUser).(*usecases.User)

	handler.Webservice.Log(fmt.Sprintf("Sending message by id %d with content %s", user.Id, request.Message))

	err = handler.ChatInteractor.SendMessage(user.Id, request.Message)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (handler *ChatWebserviceHandler) JoinChannel(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	handler.Webservice.Log("Join channel request")
	request := JoinChannelRequest{}
	err := client.ReadJson(cmd, &request)
	if err != nil {
		client.Error(cmd, err)
		return
	}
	user := ctx.Value(KeyUser).(*usecases.User)

	handler.Webservice.Log(fmt.Sprintf("Joining channel with id %d for user %d", request.ChannelId, user.Id))

	err = handler.ChatInteractor.JoinChannel(user.Id, request.ChannelId)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (handler *ChatWebserviceHandler) CreateChannel(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	handler.Webservice.Log("Create channel request")
	request := CreateChannelRequest{}
	err := client.ReadJson(cmd, &request)
	if err != nil {
		client.Error(cmd, err)
		return
	}
	user := ctx.Value(KeyUser).(*usecases.User)
	handler.Webservice.Log(fmt.Sprintf("Creating channel with name %s for user %d", request.Name, user.Id))
	err = handler.ChatInteractor.CreateChannel(user.Id, request.Name)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (handler *ChatWebserviceHandler) Channels(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	handler.Webservice.Log("Channels request")
	user := ctx.Value(KeyUser).(*usecases.User)
	channels, err := handler.ChatInteractor.Channels(user.Id)
	if err != nil {
		client.Error(cmd, err)
		return
	}
	channelsArr := make([]*domain.Channel, 0, len(channels))
	for _, val := range channels {
		channelsArr = append(channelsArr, val)
	}

	response := ChannelsResponse{
		List: channelsArr,
	}

	err = client.SendJson(cmd, response)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (sender *SenderHandler) Send(msg *domain.Message) {
	sender.Command.SetType("SendMessage")
	response := &SendMessageResponse{
		Body:      msg.Body,
		Author:    msg.Client.DisplayName,
		ChannelId: msg.Channel.Id,
		ClientId:  msg.Client.Id,
	}
	if err := sender.Handler.SendJson(sender.Command, response); err != nil {
		sender.Handler.Error(sender.Command, err)
	}
}
<commit_msg>Fix circular reference in channels model<commit_after>package interfaces

import (
	"fmt"
	"net/http"

	"code.google.com/p/go.net/context"
	"github.com/janicduplessis/projectgo/ct/domain"
	"github.com/janicduplessis/projectgo/ct/usecases"
)

const (
	urlChat string = "/ct"
)

type ChatInteractor interface {
	JoinServer(userId int64) (*domain.Client, error)
	JoinChannel(userId int64, channelId int64) error
	Channels(clientId int64) (map[int64]*domain.Channel, error)
	CreateChannel(clientId int64, name string) error
	SendMessage(userId int64, body string) error
}

type SendMessageRequest struct {
	Message string
}

type JoinChannelRequest struct {
	ChannelId int64
}

type CreateChannelRequest struct {
	Name string
}

type ChannelsResponse struct {
	List []ChannelModel
}

type ClientModel struct {
	Id   int64
	Name string
}

type ChannelModel struct {
	Id      int64
	Name    string
	Clients []ClientModel
}

type SendMessageResponse struct {
	Body      string
	Author    string
	ChannelId int64
	ClientId  int64
}

type ChatWebserviceHandler struct {
	Webservice     Webservice
	Websocket      Websocket
	ChatInteractor ChatInteractor
}

type SenderHandler struct {
	Handler WebsocketClient
	Command WebsocketCommand
}

func NewChatWebservice(ws Webservice, wsocket Websocket, ci ChatInteractor) *ChatWebserviceHandler {
	wsHandler := &ChatWebserviceHandler{
		Webservice:     ws,
		Websocket:      wsocket,
		ChatInteractor: ci,
	}

	ws.AddHandler(urlChat, true, wsHandler.JoinServer)

	wsocket.AddHandler("SendMessage", wsHandler.SendMessage)
	wsocket.AddHandler("JoinChannel", wsHandler.JoinChannel)
	wsocket.AddHandler("CreateChannel", wsHandler.CreateChannel)
	wsocket.AddHandler("Channels", wsHandler.Channels)

	return wsHandler
}

func (handler *ChatWebserviceHandler) JoinServer(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	user := ctx.Value(KeyUser).(*usecases.User)

	client, err := handler.ChatInteractor.JoinServer(user.Id)
	if err != nil {
		handler.Webservice.Error(w, err)
		return
	}

	handler.Websocket.AddClient(ctx, w, r, client)
}

func (handler *ChatWebserviceHandler) SendMessage(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	request := SendMessageRequest{}
	err := client.ReadJson(cmd, &request)
	if err != nil {
		client.Error(cmd, err)
		return
	}

	user := ctx.Value(KeyUser).(*usecases.User)

	handler.Webservice.Log(fmt.Sprintf("Sending message by id %d with content %s", user.Id, request.Message))

	err = handler.ChatInteractor.SendMessage(user.Id, request.Message)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (handler *ChatWebserviceHandler) JoinChannel(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	handler.Webservice.Log("Join channel request")
	request := JoinChannelRequest{}
	err := client.ReadJson(cmd, &request)
	if err != nil {
		client.Error(cmd, err)
		return
	}
	user := ctx.Value(KeyUser).(*usecases.User)

	handler.Webservice.Log(fmt.Sprintf("Joining channel with id %d for user %d", request.ChannelId, user.Id))

	err = handler.ChatInteractor.JoinChannel(user.Id, request.ChannelId)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (handler *ChatWebserviceHandler) CreateChannel(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	handler.Webservice.Log("Create channel request")
	request := CreateChannelRequest{}
	err := client.ReadJson(cmd, &request)
	if err != nil {
		client.Error(cmd, err)
		return
	}
	user := ctx.Value(KeyUser).(*usecases.User)
	handler.Webservice.Log(fmt.Sprintf("Creating channel with name %s for user %d", request.Name, user.Id))
	err = handler.ChatInteractor.CreateChannel(user.Id, request.Name)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (handler *ChatWebserviceHandler) Channels(ctx context.Context, client WebsocketClient, cmd WebsocketCommand) {
	handler.Webservice.Log("Channels request")
	user := ctx.Value(KeyUser).(*usecases.User)
	channels, err := handler.ChatInteractor.Channels(user.Id)
	if err != nil {
		client.Error(cmd, err)
		return
	}
	// Create the response model
	channelsArr := make([]ChannelModel, len(channels))
	for i, curChan := range channels {
		clients := make([]ClientModel, len(curChan.Clients))
		for j, curClient := range curChan.Clients {
			clients[j] = ClientModel{
				Id:   curClient.Id,
				Name: curClient.DisplayName,
			}
		}
		channelsArr[i] = ChannelModel{
			Id:      curChan.Id,
			Name:    curChan.Name,
			Clients: clients,
		}
	}

	response := ChannelsResponse{
		List: channelsArr,
	}

	err = client.SendJson(cmd, response)
	if err != nil {
		client.Error(cmd, err)
		return
	}
}

func (sender *SenderHandler) Send(msg *domain.Message) {
	sender.Command.SetType("SendMessage")
	response := &SendMessageResponse{
		Body:      msg.Body,
		Author:    msg.Client.DisplayName,
		ChannelId: msg.Channel.Id,
		ClientId:  msg.Client.Id,
	}
	if err := sender.Handler.SendJson(sender.Command, response); err != nil {
		sender.Handler.Error(sender.Command, err)
	}
}
<|endoftext|>


<commit_before>package wipro

import (
	"errors"
	"io"
)

var (
	ErrPrefix        = "proto: "
	ErrUnexpectedEOF = errors.New(ErrPrefix + "unexpected EOF")
	ErrConn          = errors.New(ErrPrefix + "network connection error")
)

type M interface {
	Marshal(*Writer)
	Unmarshal(*Reader)
}

func Send(m M, conn io.Writer) error {
	var w Writer
	m.Marshal(&w)
	if _, err := conn.Write(w.B); err != nil {
		return ErrConn
	}
	return nil
}

func Receive(conn io.Reader, m M) error {
	r := Reader{B: make([]byte, 4)}
	if _, err := conn.Read(r.B); err != nil {
		return ErrConn
	}
	size := int(r.ReadInt32())
	r.Grow(size)
	if _, err := io.ReadAtLeast(conn, r.B[4:], size); err != nil {
		return ErrConn
	}
	r.Reset()
	m.Unmarshal(&r)
	return r.Err
}

type Writer struct {
	B []byte
}

type Reader struct {
	B      []byte
	Offset int
	Err    error
}

func (w *Writer) WriteInt8(i int8) {
	w.B = append(w.B, byte(i))
}

func (r *Reader) ReadInt8() int8 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+1 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset++
	return int8(r.B[i])
}

func (w *Writer) WriteInt16(i int16) {
	w.B = append(w.B, byte(i>>8), byte(i))
}

func (r *Reader) ReadInt16() int16 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+2 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 2
	return int16(r.B[i])<<8 | int16(r.B[i+1])
}

func (w *Writer) WriteInt32(i int32) {
	w.B = append(w.B, byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
}

func (w *Writer) WriteUint32(i uint32) {
	w.B = append(w.B, byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
}

func (r *Reader) ReadInt32() int32 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+4 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 4
	return int32(r.B[i])<<24 | int32(r.B[i+1])<<16 | int32(r.B[i+2])<<8 | int32(r.B[i+3])
}

func (r *Reader) ReadUint32() uint32 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+4 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 4
	return uint32(r.B[i])<<24 | uint32(r.B[i+1])<<16 | uint32(r.B[i+2])<<8 | uint32(r.B[i+3])
}

func (w *Writer) WriteInt64(i int64) {
	w.B = append(w.B, byte(i>>56), byte(i>>48), byte(i>>40), byte(i>>32), byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
}

func (r *Reader) ReadInt64() int64 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+8 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 8
	return int64(r.B[i])<<56 | int64(r.B[i+1])<<48 | int64(r.B[i+2])<<40 | int64(r.B[i+3])<<32 |
		int64(r.B[i+4])<<24 | int64(r.B[i+5])<<16 | int64(r.B[i+6])<<8 | int64(r.B[i+7])
}

func (w *Writer) WriteString(s string) {
	w.WriteInt16(int16(len(s)))
	w.B = append(w.B, s...)
}

func (r *Reader) ReadString() string {
	if r.Err != nil {
		return ""
	}
	l := int(r.ReadInt16())
	if l <= 0 {
		return ""
	}
	i := r.Offset
	if i+l > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return ""
	}
	r.Offset += l
	return string(r.B[i : i+l])
}

func (w *Writer) WriteBytes(bs []byte) {
	w.WriteInt32(int32(len(bs)))
	w.B = append(w.B, bs...)
}

func (r *Reader) ReadBytes() []byte {
	if r.Err != nil {
		return nil
	}
	l := int(r.ReadInt32())
	if l <= 0 {
		return nil
	}
	i := r.Offset
	if i+l > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return nil
	}
	r.Offset += l
	return r.B[i : i+l]
}

func (w *Writer) SetInt32(offset int, i int32) {
	w.B[offset] = byte(i >> 24)
	w.B[offset+1] = byte(i >> 16)
	w.B[offset+2] = byte(i >> 8)
	w.B[offset+3] = byte(i)
}

func (w *Writer) SetUint32(offset int, i uint32) {
	w.B[offset] = byte(i >> 24)
	w.B[offset+1] = byte(i >> 16)
	w.B[offset+2] = byte(i >> 8)
	w.B[offset+3] = byte(i)
}

func (r *Reader) Grow(n int) {
	b := make([]byte, len(r.B)+n)
	copy(b, r.B)
	r.B = b
}

func (r *Reader) Reset() {
	r.Offset = 0
}
<commit_msg>expose the original error<commit_after>package wipro

import (
	"errors"
	"io"
)

var (
	ErrPrefix        = "proto: "
	ErrUnexpectedEOF = errors.New(ErrPrefix + "unexpected EOF")
)

type M interface {
	Marshal(*Writer)
	Unmarshal(*Reader)
}

func Send(m M, conn io.Writer) error {
	var w Writer
	m.Marshal(&w)
	if _, err := conn.Write(w.B); err != nil {
		return err
	}
	return nil
}

func Receive(conn io.Reader, m M) error {
	r := Reader{B: make([]byte, 4)}
	if _, err := conn.Read(r.B); err != nil {
		return err
	}
	size := int(r.ReadInt32())
	r.Grow(size)
	if _, err := io.ReadAtLeast(conn, r.B[4:], size); err != nil {
		return err
	}
	r.Reset()
	m.Unmarshal(&r)
	return r.Err
}

type Writer struct {
	B []byte
}

type Reader struct {
	B      []byte
	Offset int
	Err    error
}

func (w *Writer) WriteInt8(i int8) {
	w.B = append(w.B, byte(i))
}

func (r *Reader) ReadInt8() int8 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+1 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset++
	return int8(r.B[i])
}

func (w *Writer) WriteInt16(i int16) {
	w.B = append(w.B, byte(i>>8), byte(i))
}

func (r *Reader) ReadInt16() int16 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+2 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 2
	return int16(r.B[i])<<8 | int16(r.B[i+1])
}

func (w *Writer) WriteInt32(i int32) {
	w.B = append(w.B, byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
}

func (w *Writer) WriteUint32(i uint32) {
	w.B = append(w.B, byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
}

func (r *Reader) ReadInt32() int32 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+4 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 4
	return int32(r.B[i])<<24 | int32(r.B[i+1])<<16 | int32(r.B[i+2])<<8 | int32(r.B[i+3])
}

func (r *Reader) ReadUint32() uint32 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+4 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 4
	return uint32(r.B[i])<<24 | uint32(r.B[i+1])<<16 | uint32(r.B[i+2])<<8 | uint32(r.B[i+3])
}

func (w *Writer) WriteInt64(i int64) {
	w.B = append(w.B, byte(i>>56), byte(i>>48), byte(i>>40), byte(i>>32), byte(i>>24), byte(i>>16), byte(i>>8), byte(i))
}

func (r *Reader) ReadInt64() int64 {
	if r.Err != nil {
		return 0
	}
	i := r.Offset
	if i+8 > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return 0
	}
	r.Offset += 8
	return int64(r.B[i])<<56 | int64(r.B[i+1])<<48 | int64(r.B[i+2])<<40 | int64(r.B[i+3])<<32 |
		int64(r.B[i+4])<<24 | int64(r.B[i+5])<<16 | int64(r.B[i+6])<<8 | int64(r.B[i+7])
}

func (w *Writer) WriteString(s string) {
	w.WriteInt16(int16(len(s)))
	w.B = append(w.B, s...)
}

func (r *Reader) ReadString() string {
	if r.Err != nil {
		return ""
	}
	l := int(r.ReadInt16())
	if l <= 0 {
		return ""
	}
	i := r.Offset
	if i+l > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return ""
	}
	r.Offset += l
	return string(r.B[i : i+l])
}

func (w *Writer) WriteBytes(bs []byte) {
	w.WriteInt32(int32(len(bs)))
	w.B = append(w.B, bs...)
}

func (r *Reader) ReadBytes() []byte {
	if r.Err != nil {
		return nil
	}
	l := int(r.ReadInt32())
	if l <= 0 {
		return nil
	}
	i := r.Offset
	if i+l > len(r.B) {
		r.Err = ErrUnexpectedEOF
		return nil
	}
	r.Offset += l
	return r.B[i : i+l]
}

func (w *Writer) SetInt32(offset int, i int32) {
	w.B[offset] = byte(i >> 24)
	w.B[offset+1] = byte(i >> 16)
	w.B[offset+2] = byte(i >> 8)
	w.B[offset+3] = byte(i)
}

func (w *Writer) SetUint32(offset int, i uint32) {
	w.B[offset] = byte(i >> 24)
	w.B[offset+1] = byte(i >> 16)
	w.B[offset+2] = byte(i >> 8)
	w.B[offset+3] = byte(i)
}

func (r *Reader) Grow(n int) {
	b := make([]byte, len(r.B)+n)
	copy(b, r.B)
	r.B = b
}

func (r *Reader) Reset() {
	r.Offset = 0
}
<|endoftext|>


<commit_before>// mpstat collector
// this will :
//  - call mpstat
//  - gather CPU metrics
//  - feed the collector

package collector

import (
    "log"
    "os/exec"
    "strconv"
    "strings"
    // Prometheus Go toolset
    "github.com/prometheus/client_golang/prometheus"
)

type gzCpuUsageExporter struct {
    gzCpuUsage      *prometheus.GaugeVec
}

//type LoadAverageExporter struct {
//    LoadAverage1    prometheus.Gauge
//    LoadAverage5    prometheus.Gauge
//    LoadAverage15   prometheus.Gauge
//}

func NewGzCpuUsageExporter() (*gzCpuUsageExporter, error) {
    return &gzCpuUsageExporter{
        gzCpuUsage: prometheus.NewGaugeVec(prometheus.GaugeOpts{
            Name: "smartos_gz_cpu_usage_total",
            Help: "CPU usage exposed in percent.",
        }, []string{"cpu","type"}),
    }, nil
}

//func NewLoadAverageExporter() (*LoadAverageExporter, error) {
//    return &LoadAverageExporter{
//        LoadAverage1: prometheus.NewGauge(prometheus.GaugeOpts{
//            Name: "smartos_cpu_load1",
//            Help: "CPU load average 1 minute.",
//        }),
//        LoadAverage5: prometheus.NewGauge(prometheus.GaugeOpts{
//            Name: "smartos_cpu_load5",
//            Help: "CPU load average 5 minutes.",
//        }),
//        LoadAverage15: prometheus.NewGauge(prometheus.GaugeOpts{
//            Name: "smartos_cpu_load15",
//            Help: "CPU load average 15 minutes.",
//        }),
//    }, nil
//}

func (e *gzCpuUsageExporter) Describe(ch chan<- *prometheus.Desc) {
    e.gzCpuUsage.Describe(ch)
}

//func (e *LoadAverageExporter) Describe(ch chan<- *prometheus.Desc) {
//    ch <- e.LoadAverage1.Desc()
//    ch <- e.LoadAverage5.Desc()
//    ch <- e.LoadAverage15.Desc()
//}

func (e *gzCpuUsageExporter) Collect(ch chan<- prometheus.Metric) {
    e.mpstat()
    e.gzCpuUsage.Collect(ch)
}
//func (e *LoadAverageExporter) Collect(ch chan<- prometheus.Metric) {
//    e.uptime()
//    ch <- e.LoadAverage1
//    ch <- e.LoadAverage5
//    ch <- e.LoadAverage15
//}

func (e *gzCpuUsageExporter) mpstat() {
    out, eerr := exec.Command("mpstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := e.parseMpstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

//func (e *LoadAverageExporter) uptime() {
//    out, eerr := exec.Command("uptime").Output()
//    if eerr != nil {
//        log.Fatal(eerr)
//    }
//    perr := e.parseUptimeOutput(string(out))
//    if perr != nil {
//        log.Fatal(perr)
//    }
//}

func (e *gzCpuUsageExporter) parseMpstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        cpuId := parsedLine[0]
        cpuUsr, err := strconv.ParseFloat(parsedLine[12], 64)
        if err != nil {
            return err
        }
        cpuSys, err := strconv.ParseFloat(parsedLine[13], 64)
        if err != nil {
            return err
        }
        cpuIdl, err := strconv.ParseFloat(parsedLine[15], 64)
        if err != nil {
            return err
        }
        e.gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"user"}).Set(cpuUsr)
        e.gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"system"}).Set(cpuSys)
        e.gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"idle"}).Set(cpuIdl)
        //fmt.Printf("cpuId : %d, cpuUsr : %d, cpuSys : %d \n", cpuId, cpuUsr, cpuSys)
    }
    return nil
}

//func (e *LoadAverageExporter) parseUptimeOutput(out string) (error) {
//    // we will use regex in order to be sure to catch good numbers
//    r,_ := regexp.Compile(`load average: (\d.\d+), (\d.\d+), (\d.\d+)`)
//    loads := r.FindStringSubmatch(out)
//
//    load1, err := strconv.ParseFloat(loads[1], 64)
//    if err != nil {
//        return err
//    }
//    load5, err := strconv.ParseFloat(loads[2], 64)
//    if err != nil {
//        return err
//    }
//    load15, err := strconv.ParseFloat(loads[3], 64)
//    if err != nil {
//        return err
//    }
//
//    e.LoadAverage1.Set(load1)
//    e.LoadAverage5.Set(load5)
//    e.LoadAverage15.Set(load15)
//
//    return nil
//}
<commit_msg>Remove comments and tag mpstat collector as broken<commit_after>// mpstat collector
// this will :
//  - call mpstat
//  - gather CPU metrics
//  - feed the collector

// XXX COLLECTOR BROKEN
// $(mpstat 1 1) always returns the same value

package collector

import (
    "log"
    "os/exec"
    "strconv"
    "strings"
    // Prometheus Go toolset
    "github.com/prometheus/client_golang/prometheus"
)

type gzCpuUsageExporter struct {
    gzCpuUsage      *prometheus.GaugeVec
}

func NewGzCpuUsageExporter() (*gzCpuUsageExporter, error) {
    return &gzCpuUsageExporter{
        gzCpuUsage: prometheus.NewGaugeVec(prometheus.GaugeOpts{
            Name: "smartos_gz_cpu_usage_total",
            Help: "CPU usage exposed in percent.",
        }, []string{"cpu","type"}),
    }, nil
}

func (e *gzCpuUsageExporter) Describe(ch chan<- *prometheus.Desc) {
    e.gzCpuUsage.Describe(ch)
}

func (e *gzCpuUsageExporter) Collect(ch chan<- prometheus.Metric) {
    e.mpstat()
    e.gzCpuUsage.Collect(ch)
}

func (e *gzCpuUsageExporter) mpstat() {
    out, eerr := exec.Command("mpstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := e.parseMpstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func (e *gzCpuUsageExporter) parseMpstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        cpuId := parsedLine[0]
        cpuUsr, err := strconv.ParseFloat(parsedLine[12], 64)
        if err != nil {
            return err
        }
        cpuSys, err := strconv.ParseFloat(parsedLine[13], 64)
        if err != nil {
            return err
        }
        cpuIdl, err := strconv.ParseFloat(parsedLine[15], 64)
        if err != nil {
            return err
        }
        e.gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"user"}).Set(cpuUsr)
        e.gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"system"}).Set(cpuSys)
        e.gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"idle"}).Set(cpuIdl)
        //fmt.Printf("cpuId : %d, cpuUsr : %d, cpuSys : %d \n", cpuId, cpuUsr, cpuSys)
    }
    return nil
}
<|endoftext|>


<commit_before>// Title：表格，及其下属控件
//
// Description:
//
// Author:black
//
// Createtime:2013-08-08 09:29
//
// Version:1.0
//
// 修改历史:版本号 修改日期 修改人 修改说明
//
// 1.0 2013-08-08 09:29 black 创建文档
package lessgo

import (
	"bytes"
	"text/template"
)

type gridPanel struct {
	Entity      string       `xml:"entity,attr"`
	PageSize    int          `xml:"pageSize,attr"`
	LoadUrl     string       `xml:"loadUrl,attr"`
	Id          string       `xml:"id,attr"`
	Title       string       `xml:"title,attr"`
	Width       string       `xml:"width,attr"`
	Height      string       `xml:"height,attr"`
	Columns     []column     `xml:"column"`
	Actions     []action     `xml:"action"`
	Searchs     []search     `xml:"search"`
	ToolActions []toolaction `xml:"toolaction"`
}

//link目前可以支持，直接跳转，打开浏览器新窗口跳转，iframe弹窗，询问提示窗
//linkType=currentPage，
//以下为通用配置
//url 必填
//iconUrl 选填 如果有配置iconUrl，则会生成一个可点击的图标
//loadParamName 选填，不填就不带参数
//loadParamValue 如果loadParamName有值，则此配置必填，可取值为id 或者 this
type column struct {
	Field     string `xml:"field,attr"`
	Desc      string `xml:"desc,attr"`
	Hidden    string `xml:"hidden,attr"`
	Formatter string `xml:"formatter"`
}

type action struct {
	Desc         string `xml:"desc,attr"`
	Url          string `xml:"url,attr"`
	ActionParams string `xml:"actionParams,attr"`
	LinkType     string `xml:"linkType,attr"`
}

type toolaction struct {
	Desc     string `xml:"desc,attr"`
	Url      string `xml:"url,attr"`
	LinkType string `xml:"linkType,attr"`
}

type search struct {
	Field      string `xml:"field,attr"`
	SearchType string `xml:"searchType,attr"`
	InputType  string `xml:"inputType,attr"`
	LocalData  string `xml:"localData,attr"`
	Desc       string `xml:"desc,attr"`
	Url        string `xml:"url,attr"`
	ValueField string `xml:"valueField,attr"`
	DescField  string `xml:"descField,attr"`
	//存储实际的搜索值
	Value string
}

func (gridpanel gridPanel) generate(entity Entity, terminal, packageName string) []byte {

	var t *template.Template

	var buf bytes.Buffer

	gridpanel.Id = packageName + "." + gridpanel.Id

	runtimeComponentContain[gridpanel.Id] = gridpanel

	t = template.New("gridpanel.html")

	t = t.Funcs(template.FuncMap{
		"getComponentId": getComponentId,
		"compareInt":     CompareInt,
		"compareString":  CompareString,
	})

	t, err := t.ParseFiles("../lessgo/template/component/" + terminal + "/gridpanel.html")

	if err != nil {
		Log.Error(err.Error())
		return []byte{}
	}

	data := make(map[string]interface{})

	data["Gridpanel"] = gridpanel
	data["Entity"] = entity
	data["Terminal"] = terminal
	data["SearchLength"] = len(gridpanel.Searchs)
	data["ActionLength"] = len(gridpanel.Actions)

	err = t.Execute(&buf, data)

	if err != nil {
		Log.Error(err.Error())
		return []byte{}
	}

	return buf.Bytes()

}
<commit_msg>为表格添加工具条<commit_after>// Title：表格，及其下属控件
//
// Description:
//
// Author:black
//
// Createtime:2013-08-08 09:29
//
// Version:1.0
//
// 修改历史:版本号 修改日期 修改人 修改说明
//
// 1.0 2013-08-08 09:29 black 创建文档
package lessgo

import (
	"bytes"
	"text/template"
)

type gridPanel struct {
	Entity      string       `xml:"entity,attr"`
	PageSize    int          `xml:"pageSize,attr"`
	LoadUrl     string       `xml:"loadUrl,attr"`
	Id          string       `xml:"id,attr"`
	Title       string       `xml:"title,attr"`
	Width       string       `xml:"width,attr"`
	Height      string       `xml:"height,attr"`
	Columns     []column     `xml:"column"`
	Actions     []action     `xml:"action"`
	Searchs     []search     `xml:"search"`
	ToolActions []toolaction `xml:"toolaction"`
}

//link目前可以支持，直接跳转，打开浏览器新窗口跳转，iframe弹窗，询问提示窗
//linkType=currentPage，
//以下为通用配置
//url 必填
//iconUrl 选填 如果有配置iconUrl，则会生成一个可点击的图标
//loadParamName 选填，不填就不带参数
//loadParamValue 如果loadParamName有值，则此配置必填，可取值为id 或者 this
type column struct {
	Field     string `xml:"field,attr"`
	Desc      string `xml:"desc,attr"`
	Hidden    string `xml:"hidden,attr"`
	Formatter string `xml:"formatter"`
}

type action struct {
	Desc         string `xml:"desc,attr"`
	Url          string `xml:"url,attr"`
	ActionParams string `xml:"actionParams,attr"`
	LinkType     string `xml:"linkType,attr"`
}

type toolaction struct {
	Desc       string `xml:"desc,attr"`
	Url        string `xml:"url,attr"`
	LinkType   string `xml:"linkType,attr"`
	ColorClass string `xml:"colorClass,attr"`
}

type search struct {
	Field      string `xml:"field,attr"`
	SearchType string `xml:"searchType,attr"`
	InputType  string `xml:"inputType,attr"`
	LocalData  string `xml:"localData,attr"`
	Desc       string `xml:"desc,attr"`
	Url        string `xml:"url,attr"`
	ValueField string `xml:"valueField,attr"`
	DescField  string `xml:"descField,attr"`
	//存储实际的搜索值
	Value string
}

func (gridpanel gridPanel) generate(entity Entity, terminal, packageName string) []byte {

	var t *template.Template

	var buf bytes.Buffer

	gridpanel.Id = packageName + "." + gridpanel.Id

	runtimeComponentContain[gridpanel.Id] = gridpanel

	t = template.New("gridpanel.html")

	t = t.Funcs(template.FuncMap{
		"getComponentId": getComponentId,
		"compareInt":     CompareInt,
		"compareString":  CompareString,
	})

	t, err := t.ParseFiles("../lessgo/template/component/" + terminal + "/gridpanel.html")

	if err != nil {
		Log.Error(err.Error())
		return []byte{}
	}

	data := make(map[string]interface{})

	data["Gridpanel"] = gridpanel
	data["Entity"] = entity
	data["Terminal"] = terminal
	data["SearchLength"] = len(gridpanel.Searchs)
	data["ActionLength"] = len(gridpanel.Actions)

	err = t.Execute(&buf, data)

	if err != nil {
		Log.Error(err.Error())
		return []byte{}
	}

	return buf.Bytes()

}
<|endoftext|>