<commit_before>package projectInit

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"

	"strings"

	"github.com/getgauge/common"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"github.com/getgauge/gauge/plugin/install"
	"github.com/getgauge/gauge/runner"
	"github.com/getgauge/gauge/util"
)

const (
	specsDirName      = "specs"
	skelFileName      = "example.spec"
	envDefaultDirName = "default"
	metadataFileName  = "metadata.json"
)

var defaultPlugins = []string{"html-report"}

type templateMetadata struct {
	Name           string
	Description    string
	Version        string
	PostInstallCmd string
}

func initializeTemplate(templateName string) error {
	unzippedTemplate, err := util.DownloadAndUnzip(getTemplateURL(templateName))
	if err != nil {
		return err
	}
	defer util.Remove(unzippedTemplate)

	wd := config.ProjectRoot

	logger.Info("Copying Gauge template %s to current directory ...", templateName)
	filesAdded, err := common.MirrorDir(filepath.Join(unzippedTemplate, templateName), wd)
	if err != nil {
		return fmt.Errorf("Failed to copy Gauge template: %s", err.Error())
	}

	metadataFile := filepath.Join(wd, metadataFileName)
	metadataContents, err := common.ReadFileContents(metadataFile)
	if err != nil {
		return fmt.Errorf("Failed to read file contents of %s: %s", metadataFile, err.Error())
	}

	metadata := &templateMetadata{}
	err = json.Unmarshal([]byte(metadataContents), metadata)
	if err != nil {
		return err
	}

	if metadata.PostInstallCmd != "" {
		command := strings.Split(metadata.PostInstallCmd, " ")
		cmd, err := common.ExecuteSystemCommand(command, wd, os.Stdout, os.Stderr)
		cmd.Wait()
		if err != nil {
			for _, file := range filesAdded {
				pathSegments := strings.Split(file, string(filepath.Separator))
				util.Remove(filepath.Join(wd, pathSegments[0]))
			}
			return fmt.Errorf("Failed to run post install commands: %s", err.Error())
		}
	}

	util.Remove(metadataFile)
	return nil
}

func getTemplateURL(templateName string) string {
	return config.GaugeTemplatesUrl() + "/" + templateName + ".zip"
}

// InitializeProject initializes a Gauge project with specified template
func InitializeProject(templateName string) {
	wd, err := os.Getwd()
	if err != nil {
		logger.Fatal("Failed to find working directory. %s", err.Error())
	}
	config.ProjectRoot = wd

	exists, _ := common.UrlExists(getTemplateURL(templateName))

	if exists {
		err = initializeTemplate(templateName)
	} else {
		err = createProjectTemplate(templateName)
	}

	if err != nil {
		logger.Fatal("Failed to initialize. %s", err.Error())
	}
	logger.Info("\nSuccessfully initialized the project. Run specifications with \"gauge specs/\"")
}

// ListTemplates lists all the Gauge templates available in GaugeTemplatesURL
func ListTemplates() {
	templatesURL := config.GaugeTemplatesUrl()
	_, err := common.UrlExists(templatesURL)
	if err != nil {
		fmt.Println("Failed to access Gauge templates URL: %s", err.Error())
		os.Exit(1)
	}
	tempDir := common.GetTempDir()
	defer util.Remove(tempDir)
	templatesPage, err := common.Download(templatesURL, tempDir)
	if err != nil {
		fmt.Println("Failed to fetch list of templates: %s", err.Error())
		os.Exit(1)
	}

	templatePageContents, err := common.ReadFileContents(templatesPage)
	if err != nil {
		fmt.Println("Failed to read contents of file %s: %s", templatesPage, err.Error())
		os.Exit(1)
	}
	templates := getTemplateNames(templatePageContents)
	for _, template := range templates {
		fmt.Println(template)
	}
	fmt.Println("\nRun `gauge --init <template_name>` to initialize a Gauge project with template.")
}

func getTemplateNames(text string) []string {
	templateRegex, _ := regexp.Compile(`rel="nofollow">([a-z_0-9]*).zip</a></pre>`)
	submatches := templateRegex.FindAllStringSubmatch(text, -1)
	var matches []string
	for _, match := range submatches {
		matches = append(matches, match[1])
	}
	sort.Strings(matches)
	return matches
}

func showMessage(action, filename string) {
	logger.Info(" %s  %s", action, filename)
}

func createProjectTemplate(language string) error {
	if !install.IsCompatibleLanguagePluginInstalled(language) {
		logger.Info("Compatible %s plugin is not installed \n", language)
		logger.Info("Installing plugin => %s ... \n\n", language)

		if result := install.InstallPlugin(language, ""); !result.Success {
			return fmt.Errorf("Failed to install plugin %s . %s \n", language, result.Error.Error())
		}
	}
	// Create the project manifest
	showMessage("create", common.ManifestFile)
	if common.FileExists(common.ManifestFile) {
		showMessage("skip", common.ManifestFile)
	}
	manifest := &manifest.Manifest{Language: language, Plugins: defaultPlugins}
	if err := manifest.Save(); err != nil {
		return err
	}

	// creating the spec directory
	showMessage("create", specsDirName)
	if !common.DirExists(specsDirName) {
		err := os.Mkdir(specsDirName, common.NewDirectoryPermissions)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", specsDirName, err.Error()))
		}
	} else {
		showMessage("skip", specsDirName)
	}

	// Copying the skeleton file
	skelFile, err := common.GetSkeletonFilePath(skelFileName)
	if err != nil {
		return err
	}
	specFile := path.Join(specsDirName, skelFileName)
	showMessage("create", specFile)
	if common.FileExists(specFile) {
		showMessage("skip", specFile)
	} else {
		err = common.CopyFile(skelFile, specFile)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", specFile, err.Error()))
		}
	}

	// Creating the env directory
	showMessage("create", common.EnvDirectoryName)
	if !common.DirExists(common.EnvDirectoryName) {
		err = os.Mkdir(common.EnvDirectoryName, common.NewDirectoryPermissions)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", common.EnvDirectoryName, err.Error()))
		}
	}
	defaultEnv := path.Join(common.EnvDirectoryName, envDefaultDirName)
	showMessage("create", defaultEnv)
	if !common.DirExists(defaultEnv) {
		err = os.Mkdir(defaultEnv, common.NewDirectoryPermissions)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", defaultEnv, err.Error()))
		}
	}
	defaultJSON, err := common.GetSkeletonFilePath(path.Join(common.EnvDirectoryName, common.DefaultEnvFileName))
	if err != nil {
		return err
	}
	defaultJSONDest := path.Join(defaultEnv, common.DefaultEnvFileName)
	showMessage("create", defaultJSONDest)
	err = common.CopyFile(defaultJSON, defaultJSONDest)
	if err != nil {
		showMessage("error", fmt.Sprintf("Failed to create %s. %s", defaultJSONDest, err.Error()))
	}

	return runner.ExecuteInitHookForRunner(language)
}

// SetWorkingDir sets the current working directory to specified location
func SetWorkingDir(workingDir string) {
	targetDir, err := filepath.Abs(workingDir)
	if err != nil {
		logger.Fatal("Unable to set working directory : %s", err.Error())
	}

	if !common.DirExists(targetDir) {
		err = os.Mkdir(targetDir, 0777)
		if err != nil {
			logger.Fatal("Unable to set working directory : %s", err.Error())
		}
	}

	err = os.Chdir(targetDir)
	if err != nil {
		logger.Fatal("Unable to set working directory : %s", err.Error())
	}

	_, err = os.Getwd()
	if err != nil {
		logger.Fatal("Unable to set working directory : %s", err.Error())
	}
}
<commit_msg>changing println to printf<commit_after>package projectInit

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"

	"strings"

	"github.com/getgauge/common"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"github.com/getgauge/gauge/plugin/install"
	"github.com/getgauge/gauge/runner"
	"github.com/getgauge/gauge/util"
)

const (
	specsDirName      = "specs"
	skelFileName      = "example.spec"
	envDefaultDirName = "default"
	metadataFileName  = "metadata.json"
)

var defaultPlugins = []string{"html-report"}

type templateMetadata struct {
	Name           string
	Description    string
	Version        string
	PostInstallCmd string
}

func initializeTemplate(templateName string) error {
	unzippedTemplate, err := util.DownloadAndUnzip(getTemplateURL(templateName))
	if err != nil {
		return err
	}
	defer util.Remove(unzippedTemplate)

	wd := config.ProjectRoot

	logger.Info("Copying Gauge template %s to current directory ...", templateName)
	filesAdded, err := common.MirrorDir(filepath.Join(unzippedTemplate, templateName), wd)
	if err != nil {
		return fmt.Errorf("Failed to copy Gauge template: %s", err.Error())
	}

	metadataFile := filepath.Join(wd, metadataFileName)
	metadataContents, err := common.ReadFileContents(metadataFile)
	if err != nil {
		return fmt.Errorf("Failed to read file contents of %s: %s", metadataFile, err.Error())
	}

	metadata := &templateMetadata{}
	err = json.Unmarshal([]byte(metadataContents), metadata)
	if err != nil {
		return err
	}

	if metadata.PostInstallCmd != "" {
		command := strings.Split(metadata.PostInstallCmd, " ")
		cmd, err := common.ExecuteSystemCommand(command, wd, os.Stdout, os.Stderr)
		cmd.Wait()
		if err != nil {
			for _, file := range filesAdded {
				pathSegments := strings.Split(file, string(filepath.Separator))
				util.Remove(filepath.Join(wd, pathSegments[0]))
			}
			return fmt.Errorf("Failed to run post install commands: %s", err.Error())
		}
	}

	util.Remove(metadataFile)
	return nil
}

func getTemplateURL(templateName string) string {
	return config.GaugeTemplatesUrl() + "/" + templateName + ".zip"
}

// InitializeProject initializes a Gauge project with specified template
func InitializeProject(templateName string) {
	wd, err := os.Getwd()
	if err != nil {
		logger.Fatal("Failed to find working directory. %s", err.Error())
	}
	config.ProjectRoot = wd

	exists, _ := common.UrlExists(getTemplateURL(templateName))

	if exists {
		err = initializeTemplate(templateName)
	} else {
		err = createProjectTemplate(templateName)
	}

	if err != nil {
		logger.Fatal("Failed to initialize. %s", err.Error())
	}
	logger.Info("\nSuccessfully initialized the project. Run specifications with \"gauge specs/\"")
}

// ListTemplates lists all the Gauge templates available in GaugeTemplatesURL
func ListTemplates() {
	templatesURL := config.GaugeTemplatesUrl()
	_, err := common.UrlExists(templatesURL)
	if err != nil {
		fmt.Printf("Gauge templates URL is not reachable: %s\n", err.Error())
		os.Exit(1)
	}
	tempDir := common.GetTempDir()
	defer util.Remove(tempDir)
	templatesPage, err := common.Download(templatesURL, tempDir)
	if err != nil {
		fmt.Printf("Error occurred while downloading templates list: %s\n", err.Error())
		os.Exit(1)
	}

	templatePageContents, err := common.ReadFileContents(templatesPage)
	if err != nil {
		fmt.Printf("Failed to read contents of file %s: %s\n", templatesPage, err.Error())
		os.Exit(1)
	}
	templates := getTemplateNames(templatePageContents)
	for _, template := range templates {
		fmt.Println(template)
	}
	fmt.Println("\nRun `gauge --init <template_name>` to create a new Gauge project.")
}

func getTemplateNames(text string) []string {
	templateRegex, _ := regexp.Compile(`rel="nofollow">([a-z_0-9]*).zip</a></pre>`)
	submatches := templateRegex.FindAllStringSubmatch(text, -1)
	var matches []string
	for _, match := range submatches {
		matches = append(matches, match[1])
	}
	sort.Strings(matches)
	return matches
}

func showMessage(action, filename string) {
	logger.Info(" %s  %s", action, filename)
}

func createProjectTemplate(language string) error {
	if !install.IsCompatibleLanguagePluginInstalled(language) {
		logger.Info("Compatible %s plugin is not installed \n", language)
		logger.Info("Installing plugin => %s ... \n\n", language)

		if result := install.InstallPlugin(language, ""); !result.Success {
			return fmt.Errorf("Failed to install plugin %s . %s \n", language, result.Error.Error())
		}
	}
	// Create the project manifest
	showMessage("create", common.ManifestFile)
	if common.FileExists(common.ManifestFile) {
		showMessage("skip", common.ManifestFile)
	}
	manifest := &manifest.Manifest{Language: language, Plugins: defaultPlugins}
	if err := manifest.Save(); err != nil {
		return err
	}

	// creating the spec directory
	showMessage("create", specsDirName)
	if !common.DirExists(specsDirName) {
		err := os.Mkdir(specsDirName, common.NewDirectoryPermissions)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", specsDirName, err.Error()))
		}
	} else {
		showMessage("skip", specsDirName)
	}

	// Copying the skeleton file
	skelFile, err := common.GetSkeletonFilePath(skelFileName)
	if err != nil {
		return err
	}
	specFile := path.Join(specsDirName, skelFileName)
	showMessage("create", specFile)
	if common.FileExists(specFile) {
		showMessage("skip", specFile)
	} else {
		err = common.CopyFile(skelFile, specFile)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", specFile, err.Error()))
		}
	}

	// Creating the env directory
	showMessage("create", common.EnvDirectoryName)
	if !common.DirExists(common.EnvDirectoryName) {
		err = os.Mkdir(common.EnvDirectoryName, common.NewDirectoryPermissions)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", common.EnvDirectoryName, err.Error()))
		}
	}
	defaultEnv := path.Join(common.EnvDirectoryName, envDefaultDirName)
	showMessage("create", defaultEnv)
	if !common.DirExists(defaultEnv) {
		err = os.Mkdir(defaultEnv, common.NewDirectoryPermissions)
		if err != nil {
			showMessage("error", fmt.Sprintf("Failed to create %s. %s", defaultEnv, err.Error()))
		}
	}
	defaultJSON, err := common.GetSkeletonFilePath(path.Join(common.EnvDirectoryName, common.DefaultEnvFileName))
	if err != nil {
		return err
	}
	defaultJSONDest := path.Join(defaultEnv, common.DefaultEnvFileName)
	showMessage("create", defaultJSONDest)
	err = common.CopyFile(defaultJSON, defaultJSONDest)
	if err != nil {
		showMessage("error", fmt.Sprintf("Failed to create %s. %s", defaultJSONDest, err.Error()))
	}

	return runner.ExecuteInitHookForRunner(language)
}

// SetWorkingDir sets the current working directory to specified location
func SetWorkingDir(workingDir string) {
	targetDir, err := filepath.Abs(workingDir)
	if err != nil {
		logger.Fatal("Unable to set working directory : %s", err.Error())
	}

	if !common.DirExists(targetDir) {
		err = os.Mkdir(targetDir, 0777)
		if err != nil {
			logger.Fatal("Unable to set working directory : %s", err.Error())
		}
	}

	err = os.Chdir(targetDir)
	if err != nil {
		logger.Fatal("Unable to set working directory : %s", err.Error())
	}

	_, err = os.Getwd()
	if err != nil {
		logger.Fatal("Unable to set working directory : %s", err.Error())
	}
}
<|endoftext|>


<commit_before>package parser

import (
	"bytes"
	"errors"
	"log"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/etl/annotation"
	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
	"github.com/prometheus/client_golang/prometheus"
)

// Parser for parsing sidestream tests.

// RowBuffer for SS.
type RowBuffer struct {
	bufferSize int
	rows       []interface{}
}

// AddRow simply inserts a row into the buffer.  Returns error if buffer is full.
// Not thread-safe.  Should only be called by owning thread.
func (buf *RowBuffer) AddRow(row schema.SS) error {
	for len(buf.rows) >= buf.bufferSize-1 {
		return etl.ErrBufferFull
	}
	buf.rows = append(buf.rows, &row)
	return nil
}

// TakeRows returns all rows in the buffer, and clears the buffer.
// Not thread-safe.  Should only be called by owning thread.
func (buf *RowBuffer) TakeRows() []interface{} {
	res := buf.rows
	buf.rows = make([]interface{}, 0, buf.bufferSize)
	return res
}

// Annotate fetches annotations for all rows in the buffer.
// Not thread-safe.  Should only be called by owning thread.
func (buf *RowBuffer) Annotate(tableBase string) {
	metrics.WorkerState.WithLabelValues(tableBase, "annotate").Inc()
	defer metrics.WorkerState.WithLabelValues(tableBase, "annotate").Dec()
	if len(buf.rows) == 0 {
		return
	}

	ipSlice := make([]string, 2*len(buf.rows))
	geoSlice := make([]*api.GeolocationIP, 2*len(buf.rows))
	for i := range buf.rows {
		row := buf.rows[i].(*schema.SS)
		connSpec := &row.Web100_log_entry.Connection_spec
		ipSlice[i+i] = connSpec.Local_ip
		geoSlice[i+i] = &connSpec.Local_geolocation
		ipSlice[i+i+1] = connSpec.Remote_ip
		geoSlice[i+i+1] = &connSpec.Remote_geolocation
	}
	// Just use the logtime of the first row.
	logTime := time.Unix(buf.rows[0].(*schema.SS).Web100_log_entry.LogTime, 0)
	start := time.Now()
	// TODO - are there any errors we should process from Fetch?
	annotation.FetchGeoAnnotations(ipSlice, logTime, geoSlice)
	metrics.AnnotationTimeSummary.With(prometheus.Labels{"test_type": "SS"}).Observe(float64(time.Since(start).Nanoseconds()))
}

// SSParser provides a parser implementation for SideStream data.
type SSParser struct {
	inserter etl.Inserter
	etl.RowStats
	RowBuffer
}

// NewSSParser creates a new sidestream parser.
func NewSSParser(ins etl.Inserter) *SSParser {
	bufSize := etl.SS.BQBufferSize()
	buf := RowBuffer{bufSize, make([]interface{}, 0, bufSize)}
	return &SSParser{ins, ins, buf}
}

// ExtractLogtimeFromFilename extracts the log time.
// legacy filename is like  "20170203T00:00:00Z_ALL0.web100"
// The current filename is like "20170315T01:00:00Z_173.205.3.39_0.web100"
// Return time stamp if the filename is in right format
func ExtractLogtimeFromFilename(fileName string) (time.Time, error) {
	testName := filepath.Base(fileName)
	if len(testName) < 19 || !strings.Contains(testName, ".web100") {
		log.Println(testName)
		return time.Time{}, errors.New("Invalid sidestream filename")
	}

	t, err := time.Parse("20060102T15:04:05.999999999Z_", testName[0:17]+".000000000Z_")
	if err != nil {
		return time.Time{}, err
	}

	return t, nil
}

// ParseKHeader parses the first line of SS file, in format "K: cid PollTime LocalAddress LocalPort ... other_web100_variables_separated_by_space"
func ParseKHeader(header string) ([]string, error) {
	var varNames []string
	web100Vars := strings.Split(header, " ")
	if web100Vars[0] != "K:" {
		return varNames, errors.New("Corrupted header")
	}

	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		// TODO - convert this panic to something else.
		panic("tcp-kis.txt not found")
	}
	b := bytes.NewBuffer(data)

	mapping, err := web100.ParseWeb100Definitions(b)
	if err != nil {
		// TODO - convert this panic to something else.
		panic("tcp-kis.txt not found")
	}

	for index, name := range web100Vars {
		if index == 0 {
			continue
		}

		if mapping[name] != "" {
			varNames = append(varNames, mapping[name])
		} else {
			varNames = append(varNames, name)
		}
	}
	return varNames, nil
}

// TaskError return the task level error, based on failed rows, or any other criteria.
func (ss *SSParser) TaskError() error {
	return nil
}

// TableName of the table that this Parser inserts into.
func (ss *SSParser) TableName() string {
	return ss.inserter.TableBase()
}

// FullTableName of the BQ table that the uploader pushes to,
// including $YYYYMMNN, or _YYYYMMNN
func (ss *SSParser) FullTableName() string {
	return ss.inserter.FullTableName()
}

// Flush flushes any pending rows.
func (ss *SSParser) Flush() error {
	ss.inserter.Put(ss.TakeRows())
	return ss.inserter.Flush()
}

// PackDataIntoSchema packs data into sidestream BigQeury schema and buffers it.
func PackDataIntoSchema(ssValue map[string]string, logTime time.Time, testName string) (schema.SS, error) {
	localPort, err := strconv.Atoi(ssValue["LocalPort"])
	if err != nil {
		return schema.SS{}, err
	}
	remotePort, err := strconv.Atoi(ssValue["RemPort"])
	if err != nil {
		return schema.SS{}, err
	}

	connSpec := &schema.Web100ConnectionSpecification{
		Local_ip:    ssValue["LocalAddress"],
		Local_af:    web100.ParseIPFamily(ssValue["LocalAddress"]),
		Local_port:  int64(localPort),
		Remote_ip:   ssValue["RemAddress"],
		Remote_port: int64(remotePort),
	}

	// TODO: Move this annotation
	//AddGeoDataSSConnSpec(conn_spec, logTime)
	snap, err := PopulateSnap(ssValue)
	if err != nil {
		return schema.SS{}, err
	}
	web100Log := &schema.Web100LogEntry{
		LogTime:         logTime.Unix(), // TODO: Should use timestamp, not integer
		Version:         "unknown",
		Group_name:      "read", // TODO: Use Camelcase, with json annotations?
		Connection_spec: *connSpec,
		Snap:            snap,
	}

	ssTest := &schema.SS{
		TestID:           testName,
		LogTime:          logTime.Unix(),
		Type:             int64(1),
		Project:          int64(2),
		Web100_log_entry: *web100Log,
	}
	return *ssTest, nil
}

// ParseOneLine parses a single line of sidestream data.
func ParseOneLine(snapshot string, varNames []string) (map[string]string, error) {
	value := strings.Split(snapshot, " ")
	ssValue := make(map[string]string)
	if value[0] != "C:" || len(value) != len(varNames)+1 {
		log.Printf("corrupted content:")
		log.Printf(snapshot)
		return ssValue, errors.New("corrupted content")
	}

	for index, val := range value[1:] {
		// Match value with var_name
		ssValue[varNames[index]] = val
	}
	return ssValue, nil
}

// PopulateSnap fills in the snapshot data.
func PopulateSnap(ssValue map[string]string) (schema.Web100Snap, error) {
	var snap = &schema.Web100Snap{}
	var startTimeUsec int64

	// First, extract StartTimeUsec value before all others so we can combine
	// it with StartTimeStamp below.
	if valueStr, ok := ssValue["StartTimeUsec"]; ok {
		value, err := strconv.ParseInt(valueStr, 10, 64)
		if err == nil {
			startTimeUsec = value
		}
	}

	// Process every other snap key.
	for key := range ssValue {
		// Skip cid and PollTime. They are SideStream-specific fields, not web100 variables.
		if key == "cid" || key == "PollTime" {
			continue
		}
		// Skip StartTimeUsec because this is not part of the Web100Snap struct.
		if key == "StartTimeUsec" {
			continue
		}
		x := reflect.ValueOf(snap).Elem().FieldByName(key)

		switch x.Type().String() {
		case "int64":
			value, err := strconv.ParseInt(ssValue[key], 10, 64)
			if err != nil {
				return *snap, err
			}
			x.SetInt(value)
		case "string":
			x.Set(reflect.ValueOf(ssValue[key]))
		case "bool":
			if ssValue[key] == "0" {
				x.Set(reflect.ValueOf(false))
			} else if ssValue[key] == "1" {
				x.Set(reflect.ValueOf(true))
			} else {
				return *snap, errors.New("Cannot parse field " + key + " into a valie bool value.")
			}
		}
	}
	// Combine the StartTimeStamp and StartTimeUsec values.
	snap.StartTimeStamp = snap.StartTimeStamp*1000000 + startTimeUsec

	// TODO: check whether snap has valid LocalAddress, RemAddress. Return error if not.
	return *snap, nil
}

// IsParsable returns the canonical test type and whether to parse data.
func (ss *SSParser) IsParsable(testName string, data []byte) (string, bool) {
	if strings.HasSuffix(testName, ".web100") {
		return "web100", true
	}
	if strings.HasSuffix(testName, ".tra") {
		// Ignore the trace file for sidestream test.
		return "trace", false
	}
	return "unknown", false
}

// ParseAndInsert extracts each sidestream record from the rawContent and inserts each into a separate row.
func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, rawContent []byte) error {
	// TODO: for common metric states with constant labels, define global constants.
	metrics.WorkerState.WithLabelValues(ss.TableName(), "ss").Inc()
	defer metrics.WorkerState.WithLabelValues(ss.TableName(), "ss").Dec()

	logTime, err := ExtractLogtimeFromFilename(testName)
	if err != nil {
		return err
	}
	testContent := strings.Split(string(rawContent[:]), "\n")
	if len(testContent) < 2 {
		return errors.New("empty test file")
	}
	varNames, err := ParseKHeader(testContent[0])
	if err != nil {
		metrics.ErrorCount.WithLabelValues(
			ss.TableName(), "ss", "corrupted header").Inc()
		return err
	}
	for _, oneLine := range testContent[1:] {
		oneLine = strings.TrimSuffix(oneLine, "\n")

		if len(oneLine) == 0 {
			continue
		}
		ssValue, err := ParseOneLine(oneLine, varNames)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted content").Inc()
			continue
		}
		err = web100.ValidateIP(ssValue["LocalAddress"])
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid server IP").Inc()
			log.Printf("Invalid server IP address: %s with error: %s\n", ssValue["LocalAddress"], err)
			continue
		}
		err = web100.ValidateIP(ssValue["RemAddress"])
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid client IP").Inc()
			log.Printf("Invalid client IP address: %s with error: %s", ssValue["RemAddress"], err)
			continue
		}
		ssTest, err := PackDataIntoSchema(ssValue, logTime, testName)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			log.Printf("cannot pack data into sidestream schema: %v\n", err)
			continue
		}

		ssTest.ParseTime = time.Now() // for map, use string(time.Now().MarshalText())
		ssTest.ParserVersion = Version()
		if meta["filename"] != nil {
			ssTest.TaskFileName = meta["filename"].(string)
		}
		// Add row to buffer, possibly flushing buffer if it is full.
		err = ss.AddRow(ssTest)
		if err == etl.ErrBufferFull {
			// Flush asynchronously, to improve throughput.
			ss.Annotate(ss.inserter.TableBase())
			ss.inserter.PutAsync(ss.TakeRows())
			err = ss.AddRow(ssTest)
		}
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			log.Printf("insert-err: %v\n", err)
			continue
		}
		metrics.TestCount.WithLabelValues(ss.TableName(), "ss", "ok").Inc()
	}
	return nil
}
<commit_msg>Replace commented code with historical comment<commit_after>package parser

import (
	"bytes"
	"errors"
	"log"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/etl/annotation"
	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
	"github.com/prometheus/client_golang/prometheus"
)

// Parser for parsing sidestream tests.

// RowBuffer for SS.
type RowBuffer struct {
	bufferSize int
	rows       []interface{}
}

// AddRow simply inserts a row into the buffer.  Returns error if buffer is full.
// Not thread-safe.  Should only be called by owning thread.
func (buf *RowBuffer) AddRow(row schema.SS) error {
	for len(buf.rows) >= buf.bufferSize-1 {
		return etl.ErrBufferFull
	}
	buf.rows = append(buf.rows, &row)
	return nil
}

// TakeRows returns all rows in the buffer, and clears the buffer.
// Not thread-safe.  Should only be called by owning thread.
func (buf *RowBuffer) TakeRows() []interface{} {
	res := buf.rows
	buf.rows = make([]interface{}, 0, buf.bufferSize)
	return res
}

// Annotate fetches annotations for all rows in the buffer.
// Not thread-safe.  Should only be called by owning thread.
func (buf *RowBuffer) Annotate(tableBase string) {
	metrics.WorkerState.WithLabelValues(tableBase, "annotate").Inc()
	defer metrics.WorkerState.WithLabelValues(tableBase, "annotate").Dec()
	if len(buf.rows) == 0 {
		return
	}

	ipSlice := make([]string, 2*len(buf.rows))
	geoSlice := make([]*api.GeolocationIP, 2*len(buf.rows))
	for i := range buf.rows {
		row := buf.rows[i].(*schema.SS)
		connSpec := &row.Web100_log_entry.Connection_spec
		ipSlice[i+i] = connSpec.Local_ip
		geoSlice[i+i] = &connSpec.Local_geolocation
		ipSlice[i+i+1] = connSpec.Remote_ip
		geoSlice[i+i+1] = &connSpec.Remote_geolocation
	}
	// Just use the logtime of the first row.
	logTime := time.Unix(buf.rows[0].(*schema.SS).Web100_log_entry.LogTime, 0)
	start := time.Now()
	// TODO - are there any errors we should process from Fetch?
	annotation.FetchGeoAnnotations(ipSlice, logTime, geoSlice)
	metrics.AnnotationTimeSummary.With(prometheus.Labels{"test_type": "SS"}).Observe(float64(time.Since(start).Nanoseconds()))
}

// SSParser provides a parser implementation for SideStream data.
type SSParser struct {
	inserter etl.Inserter
	etl.RowStats
	RowBuffer
}

// NewSSParser creates a new sidestream parser.
func NewSSParser(ins etl.Inserter) *SSParser {
	bufSize := etl.SS.BQBufferSize()
	buf := RowBuffer{bufSize, make([]interface{}, 0, bufSize)}
	return &SSParser{ins, ins, buf}
}

// ExtractLogtimeFromFilename extracts the log time.
// legacy filename is like  "20170203T00:00:00Z_ALL0.web100"
// The current filename is like "20170315T01:00:00Z_173.205.3.39_0.web100"
// Return time stamp if the filename is in right format
func ExtractLogtimeFromFilename(fileName string) (time.Time, error) {
	testName := filepath.Base(fileName)
	if len(testName) < 19 || !strings.Contains(testName, ".web100") {
		log.Println(testName)
		return time.Time{}, errors.New("Invalid sidestream filename")
	}

	t, err := time.Parse("20060102T15:04:05.999999999Z_", testName[0:17]+".000000000Z_")
	if err != nil {
		return time.Time{}, err
	}

	return t, nil
}

// ParseKHeader parses the first line of SS file, in format "K: cid PollTime LocalAddress LocalPort ... other_web100_variables_separated_by_space"
func ParseKHeader(header string) ([]string, error) {
	var varNames []string
	web100Vars := strings.Split(header, " ")
	if web100Vars[0] != "K:" {
		return varNames, errors.New("Corrupted header")
	}

	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		// TODO - convert this panic to something else.
		panic("tcp-kis.txt not found")
	}
	b := bytes.NewBuffer(data)

	mapping, err := web100.ParseWeb100Definitions(b)
	if err != nil {
		// TODO - convert this panic to something else.
		panic("tcp-kis.txt not found")
	}

	for index, name := range web100Vars {
		if index == 0 {
			continue
		}

		if mapping[name] != "" {
			varNames = append(varNames, mapping[name])
		} else {
			varNames = append(varNames, name)
		}
	}
	return varNames, nil
}

// TaskError return the task level error, based on failed rows, or any other criteria.
func (ss *SSParser) TaskError() error {
	return nil
}

// TableName of the table that this Parser inserts into.
func (ss *SSParser) TableName() string {
	return ss.inserter.TableBase()
}

// FullTableName of the BQ table that the uploader pushes to,
// including $YYYYMMNN, or _YYYYMMNN
func (ss *SSParser) FullTableName() string {
	return ss.inserter.FullTableName()
}

// Flush flushes any pending rows.
func (ss *SSParser) Flush() error {
	ss.inserter.Put(ss.TakeRows())
	return ss.inserter.Flush()
}

// PackDataIntoSchema packs data into sidestream BigQeury schema and buffers it.
func PackDataIntoSchema(ssValue map[string]string, logTime time.Time, testName string) (schema.SS, error) {
	localPort, err := strconv.Atoi(ssValue["LocalPort"])
	if err != nil {
		return schema.SS{}, err
	}
	remotePort, err := strconv.Atoi(ssValue["RemPort"])
	if err != nil {
		return schema.SS{}, err
	}

	connSpec := &schema.Web100ConnectionSpecification{
		Local_ip:    ssValue["LocalAddress"],
		Local_af:    web100.ParseIPFamily(ssValue["LocalAddress"]),
		Local_port:  int64(localPort),
		Remote_ip:   ssValue["RemAddress"],
		Remote_port: int64(remotePort),
	}

	// NOTE: Annotation was previously done here, using AddGeoDataSS...(), but it now done
	// in ss.Annotate, prior to inserter.PutAsync
	snap, err := PopulateSnap(ssValue)
	if err != nil {
		return schema.SS{}, err
	}
	web100Log := &schema.Web100LogEntry{
		LogTime:         logTime.Unix(), // TODO: Should use timestamp, not integer
		Version:         "unknown",
		Group_name:      "read", // TODO: Use Camelcase, with json annotations?
		Connection_spec: *connSpec,
		Snap:            snap,
	}

	ssTest := &schema.SS{
		TestID:           testName,
		LogTime:          logTime.Unix(),
		Type:             int64(1),
		Project:          int64(2),
		Web100_log_entry: *web100Log,
	}
	return *ssTest, nil
}

// ParseOneLine parses a single line of sidestream data.
func ParseOneLine(snapshot string, varNames []string) (map[string]string, error) {
	value := strings.Split(snapshot, " ")
	ssValue := make(map[string]string)
	if value[0] != "C:" || len(value) != len(varNames)+1 {
		log.Printf("corrupted content:")
		log.Printf(snapshot)
		return ssValue, errors.New("corrupted content")
	}

	for index, val := range value[1:] {
		// Match value with var_name
		ssValue[varNames[index]] = val
	}
	return ssValue, nil
}

// PopulateSnap fills in the snapshot data.
func PopulateSnap(ssValue map[string]string) (schema.Web100Snap, error) {
	var snap = &schema.Web100Snap{}
	var startTimeUsec int64

	// First, extract StartTimeUsec value before all others so we can combine
	// it with StartTimeStamp below.
	if valueStr, ok := ssValue["StartTimeUsec"]; ok {
		value, err := strconv.ParseInt(valueStr, 10, 64)
		if err == nil {
			startTimeUsec = value
		}
	}

	// Process every other snap key.
	for key := range ssValue {
		// Skip cid and PollTime. They are SideStream-specific fields, not web100 variables.
		if key == "cid" || key == "PollTime" {
			continue
		}
		// Skip StartTimeUsec because this is not part of the Web100Snap struct.
		if key == "StartTimeUsec" {
			continue
		}
		x := reflect.ValueOf(snap).Elem().FieldByName(key)

		switch x.Type().String() {
		case "int64":
			value, err := strconv.ParseInt(ssValue[key], 10, 64)
			if err != nil {
				return *snap, err
			}
			x.SetInt(value)
		case "string":
			x.Set(reflect.ValueOf(ssValue[key]))
		case "bool":
			if ssValue[key] == "0" {
				x.Set(reflect.ValueOf(false))
			} else if ssValue[key] == "1" {
				x.Set(reflect.ValueOf(true))
			} else {
				return *snap, errors.New("Cannot parse field " + key + " into a valie bool value.")
			}
		}
	}
	// Combine the StartTimeStamp and StartTimeUsec values.
	snap.StartTimeStamp = snap.StartTimeStamp*1000000 + startTimeUsec

	// TODO: check whether snap has valid LocalAddress, RemAddress. Return error if not.
	return *snap, nil
}

// IsParsable returns the canonical test type and whether to parse data.
func (ss *SSParser) IsParsable(testName string, data []byte) (string, bool) {
	if strings.HasSuffix(testName, ".web100") {
		return "web100", true
	}
	if strings.HasSuffix(testName, ".tra") {
		// Ignore the trace file for sidestream test.
		return "trace", false
	}
	return "unknown", false
}

// ParseAndInsert extracts each sidestream record from the rawContent and inserts each into a separate row.
func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, rawContent []byte) error {
	// TODO: for common metric states with constant labels, define global constants.
	metrics.WorkerState.WithLabelValues(ss.TableName(), "ss").Inc()
	defer metrics.WorkerState.WithLabelValues(ss.TableName(), "ss").Dec()

	logTime, err := ExtractLogtimeFromFilename(testName)
	if err != nil {
		return err
	}
	testContent := strings.Split(string(rawContent[:]), "\n")
	if len(testContent) < 2 {
		return errors.New("empty test file")
	}
	varNames, err := ParseKHeader(testContent[0])
	if err != nil {
		metrics.ErrorCount.WithLabelValues(
			ss.TableName(), "ss", "corrupted header").Inc()
		return err
	}
	for _, oneLine := range testContent[1:] {
		oneLine = strings.TrimSuffix(oneLine, "\n")

		if len(oneLine) == 0 {
			continue
		}
		ssValue, err := ParseOneLine(oneLine, varNames)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted content").Inc()
			continue
		}
		err = web100.ValidateIP(ssValue["LocalAddress"])
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid server IP").Inc()
			log.Printf("Invalid server IP address: %s with error: %s\n", ssValue["LocalAddress"], err)
			continue
		}
		err = web100.ValidateIP(ssValue["RemAddress"])
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "Invalid client IP").Inc()
			log.Printf("Invalid client IP address: %s with error: %s", ssValue["RemAddress"], err)
			continue
		}
		ssTest, err := PackDataIntoSchema(ssValue, logTime, testName)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			log.Printf("cannot pack data into sidestream schema: %v\n", err)
			continue
		}

		ssTest.ParseTime = time.Now() // for map, use string(time.Now().MarshalText())
		ssTest.ParserVersion = Version()
		if meta["filename"] != nil {
			ssTest.TaskFileName = meta["filename"].(string)
		}
		// Add row to buffer, possibly flushing buffer if it is full.
		err = ss.AddRow(ssTest)
		if err == etl.ErrBufferFull {
			// Flush asynchronously, to improve throughput.
			ss.Annotate(ss.inserter.TableBase())
			ss.inserter.PutAsync(ss.TakeRows())
			err = ss.AddRow(ssTest)
		}
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			log.Printf("insert-err: %v\n", err)
			continue
		}
		metrics.TestCount.WithLabelValues(ss.TableName(), "ss", "ok").Inc()
	}
	return nil
}
<|endoftext|>


<commit_before>// Parse Sidestream tests.
package parser

import (
	"bytes"
	"cloud.google.com/go/bigquery"
	"errors"
	"log"
	"net"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
)

type SSParser struct {
	inserter etl.Inserter
	etl.RowStats
}

func NewSSParser(ins etl.Inserter) *SSParser {
	return &SSParser{ins, ins}
}

// The legacy filename is like  "20170203T00:00:00Z_ALL0.web100"
// The current filename is like "20170315T01:00:00Z_173.205.3.39_0.web100"
// Return time stamp if the filename is in right format
func ExtractLogtimeFromFilename(fileName string) (int64, error) {
	testName := filepath.Base(fileName)
	if len(testName) < 19 || !strings.Contains(testName, ".web100") {
		log.Println(testName)
		return 0, errors.New("Invalid sidestream filename")
	}

	t, err := time.Parse("20060102T15:04:05.999999999Z_", testName[0:17]+".000000000Z_")
	if err != nil {
		return 0, err
	}

	return t.Unix(), nil
}

func ParseIPFamily(ipStr string) int {
	ip := net.ParseIP(ipStr)
	if ip.To4() != nil {
		return syscall.AF_INET
	} else if ip.To16() != nil {
		return syscall.AF_INET6
	}
	return -1
}

// the first line of SS test is in format "K: cid PollTime LocalAddress LocalPort ... other_web100_variables_separated_by_space"
func ParseKHeader(header string) ([]string, error) {
	var var_names []string
	web100_vars := strings.Split(header, " ")
	if web100_vars[0] != "K:" {
		return var_names, errors.New("Corrupted header")
	}

	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		panic("tcp-kis.txt not found")
	}
	b := bytes.NewBuffer(data)

	mapping, err := web100.ParseWeb100Definitions(b)

	for index, name := range web100_vars {
		if index == 0 {
			continue
		}

		if mapping[name] != "" {
			var_names = append(var_names, mapping[name])
		} else {
			var_names = append(var_names, name)
		}
	}
	return var_names, nil
}

func (ss *SSParser) TableName() string {
	return ss.inserter.TableBase()
}

func (ss *SSParser) FullTableName() string {
	return ss.inserter.FullTableName()
}

func (ss *SSParser) Flush() error {
	return ss.inserter.Flush()
}

// TODO: add metrics
// Prepare data into sidestream BigQeury schema and insert it.
func PackDataIntoSchema(ss_value map[string]string, log_time int64, testName string) (schema.SS, error) {
	local_port, err := strconv.Atoi(ss_value["LocalPort"])
	if err != nil {
		return schema.SS{}, err
	}
	remote_port, err := strconv.Atoi(ss_value["RemPort"])
	if err != nil {
		return schema.SS{}, err
	}
	conn_spec := &schema.Web100ConnectionSpecification{
		Local_ip:    ss_value["LocalAddress"],
		Local_af:    int64(ParseIPFamily(ss_value["LocalAddress"])),
		Local_port:  int64(local_port),
		Remote_ip:   ss_value["RemAddress"],
		Remote_port: int64(remote_port),
	}
	snap, err := PopulateSnap(ss_value)
	if err != nil {
		return schema.SS{}, err
	}
	web100_log := &schema.Web100LogEntry{
		Log_time:        log_time,
		Version:         "unknown",
		Group_name:      "read",
		Connection_spec: *conn_spec,
		Snap:            snap,
	}

	ss_test := &schema.SS{
		Test_id:          testName,
		Log_time:         log_time,
		Type:             int64(1),
		Project:          int64(2),
		Web100_log_entry: *web100_log,
	}
	return *ss_test, nil
}

func ParseOneLine(snapshot string, var_names []string) (map[string]string, error) {
	value := strings.Split(snapshot, " ")
	ss_value := make(map[string]string)
	if value[0] != "C:" || len(value) != len(var_names)+1 {
		log.Printf("corrupted content:")
		log.Printf(snapshot)
		return ss_value, errors.New("corrupted content")
	}

	for index, val := range value[1:] {
		// Match value with var_name
		ss_value[var_names[index]] = val
	}
	return ss_value, nil
}

func PopulateSnap(ss_value map[string]string) (schema.Web100Snap, error) {
	var snap = &schema.Web100Snap{}
	for key := range ss_value {
		// Skip cid and PollTime. They are SideStream-specific fields, not web100 variables.
		if key == "cid" || key == "PollTime" {
			continue
		}
		// We do special handling for this variable
		if key == "StartTimeUsec" {
			// TODO: func CalculateStartTimeStamp() to get correct StartTimeStamp value.
			continue
		}
		x := reflect.ValueOf(snap).Elem().FieldByName(key)

		switch x.Type().String() {
		case "int64":
			value, err := strconv.Atoi(ss_value[key])
			if err != nil {
				return *snap, err
			}
			x.SetInt(int64(value))
		case "string":
			x.Set(reflect.ValueOf(ss_value[key]))
		case "bool":
			if ss_value[key] == "0" {
				x.Set(reflect.ValueOf(false))
			} else if ss_value[key] == "1" {
				x.Set(reflect.ValueOf(true))
			} else {
				return *snap, errors.New("Cannot parse field " + key + " into a valie bool value.")
			}
		}
	}
	// TODO: check whether snap has valid LocalAddress, RemAddress. Return error if not.
	return *snap, nil
}

// TODO: add metrics.
func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, rawContent []byte) error {
	if strings.Contains(testName, ".tra") {
		// Ignore the trace file for sidestream test.
		return nil
	}
	log_time, err := ExtractLogtimeFromFilename(testName)
	if err != nil {
		return err
	}
	var var_names []string
	testContent := strings.Split(string(rawContent[:]), "\n")
	if len(testContent) < 2 {
		return errors.New("empty test file.")
	}
	var_names, err = ParseKHeader(testContent[0])
	if err != nil {
		return err
	}
	for _, oneLine := range testContent[1:] {
		oneLine := strings.TrimSuffix(oneLine, "\n")

		if len(oneLine) == 0 {
			continue
		}
		ss_value, err := ParseOneLine(oneLine, var_names)
		if err != nil {
			return err
		}
		ss_test, err := PackDataIntoSchema(ss_value, log_time, testName)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			log.Printf("cannot pack data into sidestream schema: %v\n", err)
			return err
		}
		err = ss.inserter.InsertRow(ss_test)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			log.Printf("insert-err: %v\n", err)
			continue
		}

	}
	return nil
}
<commit_msg>add metrics<commit_after>// Parse Sidestream tests.
package parser

import (
	"bytes"
	"cloud.google.com/go/bigquery"
	"errors"
	"log"
	"net"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
)

type SSParser struct {
	inserter etl.Inserter
	etl.RowStats
}

func NewSSParser(ins etl.Inserter) *SSParser {
	return &SSParser{ins, ins}
}

// The legacy filename is like  "20170203T00:00:00Z_ALL0.web100"
// The current filename is like "20170315T01:00:00Z_173.205.3.39_0.web100"
// Return time stamp if the filename is in right format
func ExtractLogtimeFromFilename(fileName string) (int64, error) {
	testName := filepath.Base(fileName)
	if len(testName) < 19 || !strings.Contains(testName, ".web100") {
		log.Println(testName)
		return 0, errors.New("Invalid sidestream filename")
	}

	t, err := time.Parse("20060102T15:04:05.999999999Z_", testName[0:17]+".000000000Z_")
	if err != nil {
		return 0, err
	}

	return t.Unix(), nil
}

func ParseIPFamily(ipStr string) int {
	ip := net.ParseIP(ipStr)
	if ip.To4() != nil {
		return syscall.AF_INET
	} else if ip.To16() != nil {
		return syscall.AF_INET6
	}
	return -1
}

// the first line of SS test is in format "K: cid PollTime LocalAddress LocalPort ... other_web100_variables_separated_by_space"
func ParseKHeader(header string) ([]string, error) {
	var var_names []string
	web100_vars := strings.Split(header, " ")
	if web100_vars[0] != "K:" {
		return var_names, errors.New("Corrupted header")
	}

	data, err := web100.Asset("tcp-kis.txt")
	if err != nil {
		panic("tcp-kis.txt not found")
	}
	b := bytes.NewBuffer(data)

	mapping, err := web100.ParseWeb100Definitions(b)

	for index, name := range web100_vars {
		if index == 0 {
			continue
		}

		if mapping[name] != "" {
			var_names = append(var_names, mapping[name])
		} else {
			var_names = append(var_names, name)
		}
	}
	return var_names, nil
}

func (ss *SSParser) TableName() string {
	return ss.inserter.TableBase()
}

func (ss *SSParser) FullTableName() string {
	return ss.inserter.FullTableName()
}

func (ss *SSParser) Flush() error {
	return ss.inserter.Flush()
}

// Prepare data into sidestream BigQeury schema and insert it.
func PackDataIntoSchema(ss_value map[string]string, log_time int64, testName string) (schema.SS, error) {
	local_port, err := strconv.Atoi(ss_value["LocalPort"])
	if err != nil {
		return schema.SS{}, err
	}
	remote_port, err := strconv.Atoi(ss_value["RemPort"])
	if err != nil {
		return schema.SS{}, err
	}
	conn_spec := &schema.Web100ConnectionSpecification{
		Local_ip:    ss_value["LocalAddress"],
		Local_af:    int64(ParseIPFamily(ss_value["LocalAddress"])),
		Local_port:  int64(local_port),
		Remote_ip:   ss_value["RemAddress"],
		Remote_port: int64(remote_port),
	}
	snap, err := PopulateSnap(ss_value)
	if err != nil {
		return schema.SS{}, err
	}
	web100_log := &schema.Web100LogEntry{
		Log_time:        log_time,
		Version:         "unknown",
		Group_name:      "read",
		Connection_spec: *conn_spec,
		Snap:            snap,
	}

	ss_test := &schema.SS{
		Test_id:          testName,
		Log_time:         log_time,
		Type:             int64(1),
		Project:          int64(2),
		Web100_log_entry: *web100_log,
	}
	return *ss_test, nil
}

func ParseOneLine(snapshot string, var_names []string) (map[string]string, error) {
	value := strings.Split(snapshot, " ")
	ss_value := make(map[string]string)
	if value[0] != "C:" || len(value) != len(var_names)+1 {
		log.Printf("corrupted content:")
		log.Printf(snapshot)
		return ss_value, errors.New("corrupted content")
	}

	for index, val := range value[1:] {
		// Match value with var_name
		ss_value[var_names[index]] = val
	}
	return ss_value, nil
}

func PopulateSnap(ss_value map[string]string) (schema.Web100Snap, error) {
	var snap = &schema.Web100Snap{}
	for key := range ss_value {
		// Skip cid and PollTime. They are SideStream-specific fields, not web100 variables.
		if key == "cid" || key == "PollTime" {
			continue
		}
		// We do special handling for this variable
		if key == "StartTimeUsec" {
			// TODO: func CalculateStartTimeStamp() to get correct StartTimeStamp value.
			continue
		}
		x := reflect.ValueOf(snap).Elem().FieldByName(key)

		switch x.Type().String() {
		case "int64":
			value, err := strconv.Atoi(ss_value[key])
			if err != nil {
				return *snap, err
			}
			x.SetInt(int64(value))
		case "string":
			x.Set(reflect.ValueOf(ss_value[key]))
		case "bool":
			if ss_value[key] == "0" {
				x.Set(reflect.ValueOf(false))
			} else if ss_value[key] == "1" {
				x.Set(reflect.ValueOf(true))
			} else {
				return *snap, errors.New("Cannot parse field " + key + " into a valie bool value.")
			}
		}
	}
	// TODO: check whether snap has valid LocalAddress, RemAddress. Return error if not.
	return *snap, nil
}

func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, rawContent []byte) error {
	metrics.WorkerState.WithLabelValues("ss").Inc()
	defer metrics.WorkerState.WithLabelValues("ss").Dec()
	if strings.Contains(testName, ".tra") {
		// Ignore the trace file for sidestream test.
		return nil
	}
	log_time, err := ExtractLogtimeFromFilename(testName)
	if err != nil {
		return err
	}
	var var_names []string
	testContent := strings.Split(string(rawContent[:]), "\n")
	if len(testContent) < 2 {
		return errors.New("empty test file.")
	}
	var_names, err = ParseKHeader(testContent[0])
	if err != nil {
		metrics.ErrorCount.WithLabelValues(
			ss.TableName(), "ss", "corrupted header").Inc()
		return err
	}
	for _, oneLine := range testContent[1:] {
		oneLine := strings.TrimSuffix(oneLine, "\n")

		if len(oneLine) == 0 {
			continue
		}
		ss_value, err := ParseOneLine(oneLine, var_names)
		if err != nil {
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted content").Inc()
			return err
		}
		ss_test, err := PackDataIntoSchema(ss_value, log_time, testName)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "corrupted data").Inc()
			log.Printf("cannot pack data into sidestream schema: %v\n", err)
			return err
		}
		err = ss.inserter.InsertRow(ss_test)
		if err != nil {
			metrics.ErrorCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			metrics.TestCount.WithLabelValues(
				ss.TableName(), "ss", "insert-err").Inc()
			log.Printf("insert-err: %v\n", err)
			continue
		}
		metrics.TestCount.WithLabelValues(ss.TableName(), "ss", "ok").Inc()

	}
	return nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

// runPersister() implements the persister task.
func (m *collection) runPersister() {
	ssCh := make(chan *segmentStack)

	defer close(ssCh)

	go m.runPersisterInner(ssCh)

	var ssLast *segmentStack

	// Keep only the last segmentStack, which will happen when the
	// runPersisterInner is busy.
	for {
		select {
		case <-m.stopCh:
			return

		case ssIn := <-m.awakePersisterCh:
			if ssLast != nil {
				ssLast.Close()
			}
			ssLast = ssIn

		case ssCh <- ssLast:
			ssLast = nil
		}
	}
}

func (m *collection) runPersisterInner(ssCh chan *segmentStack) {
	defer close(m.donePersisterCh)

LOOP:
	for ss := range ssCh {
		if m.options.LowerLevelUpdate != nil {
			llssNext, err := m.options.LowerLevelUpdate(ss)
			if err != nil {
				ss.Close()

				m.Log("collection: runPersisterInner, err: %v", err)

				continue LOOP
			}

			m.m.Lock()
			llssPrev := m.lowerLevelSnapshot
			m.lowerLevelSnapshot = newSnapshotWrapper(llssNext)
			m.m.Unlock()

			if llssPrev != nil {
				llssPrev.Close()
			}
		}

		ss.Close()
	}
}
<commit_msg>persister handles nil ssLast<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

// runPersister() implements the persister task.
func (m *collection) runPersister() {
	ssCh := make(chan *segmentStack)

	defer close(ssCh)

	go m.runPersisterInner(ssCh)

	var ssLast *segmentStack

	// Keep only the last segmentStack, which will happen when the
	// runPersisterInner is busy.
	for {
		if ssLast != nil {
			select {
			case <-m.stopCh:
				return

			case ssIn := <-m.awakePersisterCh:
				if ssLast != nil {
					ssLast.Close()
				}
				ssLast = ssIn

			case ssCh <- ssLast:
				ssLast = nil
			}
		} else {
			select {
			case <-m.stopCh:
				return

			case ssLast = <-m.awakePersisterCh:
				// NO-OP.
			}
		}
	}
}

func (m *collection) runPersisterInner(ssCh chan *segmentStack) {
	defer close(m.donePersisterCh)

LOOP:
	for ss := range ssCh {
		if m.options.LowerLevelUpdate != nil {
			llssNext, err := m.options.LowerLevelUpdate(ss)
			if err != nil {
				ss.Close()

				m.Log("collection: runPersisterInner, err: %v", err)

				continue LOOP
			}

			m.m.Lock()
			llssPrev := m.lowerLevelSnapshot
			m.lowerLevelSnapshot = newSnapshotWrapper(llssNext)
			m.m.Unlock()

			if llssPrev != nil {
				llssPrev.Close()
			}
		}

		ss.Close()
	}
}
<|endoftext|>


<commit_before>package main

// TODO(ig): draw board on the center. positions are hardcoded currently.
// BUG(ig): drawBoard can't display some letters for some reason. 'NEYCE' appears as 'N YCE'

import (
	"log"
	"strconv"
	"time"
	"unicode/utf8"

	"github.com/igungor/quackle"
	"github.com/nsf/termbox-go"
)

const (
	fgcolor = termbox.ColorDefault
	bgcolor = termbox.ColorDefault
)

func main() {
	if err := realMain(); err != nil {
		log.Fatal(err)
	}
}

func realMain() error {
	if err := termbox.Init(); err != nil {
		return err
	}
	defer termbox.Close()
	termbox.SetInputMode(termbox.InputEsc)

	initGame()

	drawBoard(getCurPos(game).Board(), 110, 25)
	termbox.Flush()
mainloop:
	for {
		if getCurPos(game).GameOver() {
			// TODO(ig): control game over
			time.Sleep(3 * time.Second)
			break mainloop
		}
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			switch ev.Key {
			case termbox.KeyEsc, termbox.KeyCtrlC:
				break mainloop
			case termbox.KeyEnter:
				game.HaveComputerPlay()
			}
		case termbox.EventError:
			return ev.Err
		}
		drawBoard(getCurPos(game).Board(), 110, 25)
		termbox.Flush()
	}
	return nil
}

// drawBoard draws the board onto (x,y) position of the grid.
func drawBoard(board quackle.Board, x, y int) {
	w, h := board.Width(), board.Height()
	// columns on the top
	for dx := 0; dx < w; dx++ {
		termbox.SetCell(x+1+dx*2, y-2, rune('A'+dx), fgcolor, bgcolor)
		termbox.SetCell(x+1+dx*2+1, y-2, ' ', fgcolor, bgcolor)
	}

	// rows on the left
	for dy := 0; dy < h; dy++ {
		if dy < 10 {
			tbprint(strconv.Itoa(dy+1), x-2, y+dy, fgcolor, bgcolor)
		} else {
			tbprint(strconv.Itoa(dy+1), x-3, y+dy, fgcolor, bgcolor)
		}
	}

	// top border
	termbox.SetCell(x-1, y-1, '┌', fgcolor, bgcolor)
	fill(x, y-1, w*2, 1, '─')
	termbox.SetCell(x+w*2, y-1, '┐', fgcolor, bgcolor)

	// body border
	fill(x-1, y, 1, h, '│')
	fill(x+w*2, y, 1, h, '│')

	// bottom border
	termbox.SetCell(x-1, y+h, '└', fgcolor, bgcolor)
	fill(x, y+h, w*2, 1, '─')
	termbox.SetCell(x+w*2, y+h, '┘', fgcolor, bgcolor)

	// mark multipliers and letters
	for row := 0; row < h; row++ {
		for col := 0; col < w; col++ {
			bl := board.Letter(row, col)
			var r rune
			if dm.AlphabetParameters().IsPlainLetter(bl) {
				letter := dm.AlphabetParameters().UserVisible(bl)
				r, _ = utf8.DecodeRuneInString(letter)
			} else {
				letterMult := dm.BoardParameters().LetterMultiplier(row, col)
				wordMult := dm.BoardParameters().WordMultiplier(row, col)
				switch {
				case letterMult == 2:
					r = '\''
				case letterMult == 3:
					r = '"'
				case letterMult == 4:
					r = '^'
				case wordMult == 2:
					r = '-'
				case wordMult == 3:
					r = '='
				case wordMult == 4:
					r = '~'
				default:
					r = ' '
				}
			}
			termbox.SetCell(x+col*2, y+row, r, fgcolor, bgcolor)
		}
	}
}

// tbprint prints the msg onto (x,y) position of the grid.
func tbprint(msg string, x, y int, fg, bg termbox.Attribute) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

// fill fills a box of w*h area with r runes starting from (x,y) position of
// the grid.
func fill(x, y, w, h int, r rune) {
	for ly := 0; ly < h; ly++ {
		for lx := 0; lx < w; lx++ {
			termbox.SetCell(x+lx, y+ly, r, fgcolor, bgcolor)
		}
	}
}

// func move() {
// 	// play
// 	for {
// 		if getCurPos(game).GameOver() {
// 			fmt.Println("GAME OVER!")
// 			break
// 		}
// 		player := getCurPos(game).CurrentPlayer().(quackle.Player)

// 		// go on cold heartless machine!
// 		if quackle.QuacklePlayerPlayerType(player.Xtype()) == quackle.PlayerComputerPlayerType {
// 			move := game.HaveComputerPlay()
// 			fmt.Println("Rack: ", player.Rack().ToString())
// 			fmt.Println("Move: ", move.ToString())
// 			fmt.Printf("Board:\n %v\n", getCurPos(game).Board().ToString())
// 			continue
// 		}

// 		game.AdvanceToNoncomputerPlayer()
// 		fmt.Println("Rack: ", player.Rack().ToString())

// 		// read input
// 		var move quackle.Move
// 	MOVELOOP:
// 		for {
// 			r := bufio.NewReader(os.Stdin)
// 			input, _ := r.ReadString('\n')
// 			input = strings.TrimSuffix(input, "\n")
// 			fields := strings.Fields(input)
// 			switch len(fields) {
// 			case 1:
// 				// pass
// 				if fields[0] == "-" {
// 					move = quackle.MoveCreatePassMove()
// 					game.CommitMove(move)
// 					break MOVELOOP
// 				}
// 				fmt.Println("NEIN! gecerli biseyler yaz")
// 				continue MOVELOOP
// 			case 2:
// 				place, word := fields[0], fields[1]
// 				move = quackle.MoveCreatePlaceMove(place, dm.AlphabetParameters().Encode(word))
// 				if getCurPos(game).ValidateMove(move) == int(quackle.GamePositionValidMove) {
// 					game.CommitMove(move)
// 					break MOVELOOP
// 				}
// 				fmt.Println("NEIN! gecerli bir hamle degil")
// 				continue MOVELOOP
// 			default:
// 				fmt.Println("NEIN! gecerli biseyler yaz")
// 				continue MOVELOOP
// 			}
// 		}

// 		fmt.Println("Rack: ", player.Rack().ToString())
// 		fmt.Println("Move: ", move.ToString())
// 		fmt.Printf("Board:\n %v\n", getCurPos(game).Board().ToString())
// 	}
// }
<commit_msg>pfft: handle window resize<commit_after>package main

// TODO(ig): draw board on the center. positions are hardcoded currently.
// BUG(ig): drawBoard can't display some letters for some reason. 'NEYCE' appears as 'N YCE'

import (
	"log"
	"strconv"
	"time"
	"unicode/utf8"

	"github.com/igungor/quackle"
	"github.com/nsf/termbox-go"
)

const (
	fgcolor = termbox.ColorDefault
	bgcolor = termbox.ColorDefault
)

func main() {
	if err := realMain(); err != nil {
		log.Fatal(err)
	}
}

func realMain() error {
	if err := termbox.Init(); err != nil {
		return err
	}
	defer termbox.Close()
	termbox.SetInputMode(termbox.InputEsc)

	initGame()

	drawBoard(getCurPos(game).Board())
	termbox.Flush()

mainloop:
	for {
		if getCurPos(game).GameOver() {
			// TODO(ig): handle gameover
			time.Sleep(3 * time.Second)
			break mainloop
		}
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			switch ev.Key {
			case termbox.KeyEsc, termbox.KeyCtrlC:
				break mainloop
			case termbox.KeyEnter:
				game.HaveComputerPlay()
			}
		case termbox.EventResize:
			drawBoard(getCurPos(game).Board())
			termbox.Flush()
		case termbox.EventError:
			return ev.Err
		}
		drawBoard(getCurPos(game).Board())
		termbox.Flush()
	}
	return nil
}

// drawBoard draws the board at the center of the grid.
func drawBoard(board quackle.Board) {
	sw, sh := termbox.Size()
	x := (sw - boardsize*2 + 2 + 1 + 1) / 2
	y := (sh - boardsize + 1 + 1 + 1) / 2
	w, h := boardsize, boardsize

	termbox.Clear(fgcolor, bgcolor)
	// columns on the top
	for dx := 0; dx < w; dx++ {
		termbox.SetCell(x+1+dx*2, y-2, rune('A'+dx), fgcolor, bgcolor)
		termbox.SetCell(x+1+dx*2+1, y-2, ' ', fgcolor, bgcolor)
	}

	// rows on the left
	for dy := 0; dy < h; dy++ {
		if dy < 10 {
			tbprint(strconv.Itoa(dy+1), x-2, y+dy, fgcolor, bgcolor)
		} else {
			tbprint(strconv.Itoa(dy+1), x-3, y+dy, fgcolor, bgcolor)
		}
	}

	// top border
	termbox.SetCell(x-1, y-1, '┌', fgcolor, bgcolor)
	fill(x, y-1, w*2, 1, '─')
	termbox.SetCell(x+w*2, y-1, '┐', fgcolor, bgcolor)

	// body border
	fill(x-1, y, 1, h, '│')
	fill(x+w*2, y, 1, h, '│')

	// bottom border
	termbox.SetCell(x-1, y+h, '└', fgcolor, bgcolor)
	fill(x, y+h, w*2, 1, '─')
	termbox.SetCell(x+w*2, y+h, '┘', fgcolor, bgcolor)

	// mark multipliers and letters
	for row := 0; row < h; row++ {
		for col := 0; col < w; col++ {
			bl := board.Letter(row, col)
			var r rune
			if dm.AlphabetParameters().IsPlainLetter(bl) {
				letter := dm.AlphabetParameters().UserVisible(bl)
				r, _ = utf8.DecodeRuneInString(letter)
			} else {
				letterMult := dm.BoardParameters().LetterMultiplier(row, col)
				wordMult := dm.BoardParameters().WordMultiplier(row, col)
				switch {
				case letterMult == 2:
					r = '\''
				case letterMult == 3:
					r = '"'
				case letterMult == 4:
					r = '^'
				case wordMult == 2:
					r = '-'
				case wordMult == 3:
					r = '='
				case wordMult == 4:
					r = '~'
				default:
					r = ' '
				}
			}
			termbox.SetCell(x+col*2, y+row, r, fgcolor, bgcolor)
		}
	}
}

// tbprint prints the msg onto (x,y) position of the grid.
func tbprint(msg string, x, y int, fg, bg termbox.Attribute) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

// fill fills a box of w*h area with r runes starting from (x,y) position of
// the grid.
func fill(x, y, w, h int, r rune) {
	for ly := 0; ly < h; ly++ {
		for lx := 0; lx < w; lx++ {
			termbox.SetCell(x+lx, y+ly, r, fgcolor, bgcolor)
		}
	}
}

// func move() {
// 	// play
// 	for {
// 		if getCurPos(game).GameOver() {
// 			fmt.Println("GAME OVER!")
// 			break
// 		}
// 		player := getCurPos(game).CurrentPlayer().(quackle.Player)

// 		// go on cold heartless machine!
// 		if quackle.QuacklePlayerPlayerType(player.Xtype()) == quackle.PlayerComputerPlayerType {
// 			move := game.HaveComputerPlay()
// 			fmt.Println("Rack: ", player.Rack().ToString())
// 			fmt.Println("Move: ", move.ToString())
// 			fmt.Printf("Board:\n %v\n", getCurPos(game).Board().ToString())
// 			continue
// 		}

// 		game.AdvanceToNoncomputerPlayer()
// 		fmt.Println("Rack: ", player.Rack().ToString())

// 		// read input
// 		var move quackle.Move
// 	MOVELOOP:
// 		for {
// 			r := bufio.NewReader(os.Stdin)
// 			input, _ := r.ReadString('\n')
// 			input = strings.TrimSuffix(input, "\n")
// 			fields := strings.Fields(input)
// 			switch len(fields) {
// 			case 1:
// 				// pass
// 				if fields[0] == "-" {
// 					move = quackle.MoveCreatePassMove()
// 					game.CommitMove(move)
// 					break MOVELOOP
// 				}
// 				fmt.Println("NEIN! gecerli biseyler yaz")
// 				continue MOVELOOP
// 			case 2:
// 				place, word := fields[0], fields[1]
// 				move = quackle.MoveCreatePlaceMove(place, dm.AlphabetParameters().Encode(word))
// 				if getCurPos(game).ValidateMove(move) == int(quackle.GamePositionValidMove) {
// 					game.CommitMove(move)
// 					break MOVELOOP
// 				}
// 				fmt.Println("NEIN! gecerli bir hamle degil")
// 				continue MOVELOOP
// 			default:
// 				fmt.Println("NEIN! gecerli biseyler yaz")
// 				continue MOVELOOP
// 			}
// 		}

// 		fmt.Println("Rack: ", player.Rack().ToString())
// 		fmt.Println("Move: ", move.ToString())
// 		fmt.Printf("Board:\n %v\n", getCurPos(game).Board().ToString())
// 	}
// }
<|endoftext|>


<commit_before>package photosync

import (
	"github.com/garyburd/go-oauth/oauth"
	"fmt"
	"net/http"
	"net/url"
	"log"
	"io/ioutil"
	"encoding/json"
	"strconv"
	"sync"
)

type Config struct {
	Consumer oauth.Credentials
	Access oauth.Credentials
	FlickrUserId string `json:"flickr_user_id"`
}

type Photo struct {
	Id int `json:"string"`
	Owner string
	Secret string
	Title string
	Ispublic int `json:"string"`
	Isfriend int `json:"string"`
	Isfamily int `json:"string"`
}

type Response struct {
	Stat string
	Data struct {
		Page int
		Pages int
		Perpage int
		Total string
		Photos []Photo `json:"photo"`
	} `json:"photos"`
	User FlickrUser `json:"user"`
}

type FlickrUser struct {
	Id string
	Username struct {
		Content string `json:"_content"`
	} `json:"username"`
}

// API Error type
type Error struct {
	response string
}
func (e *Error) Error() string {
	return fmt.Sprintf("API fail: %s", e.response)
}

var oauthClient = oauth.Client {
	TemporaryCredentialRequestURI: "https://api.flickr.com/services/oauth/request_token",
	ResourceOwnerAuthorizationURI: "https://api.flickr.com/services/oauth/authorize",
	TokenRequestURI:               "https://api.flickr.com/services/oauth/access_token",
}

var config Config
var apiBase = "https://api.flickr.com/services/rest"
var form = url.Values{}

// Load the consumer key and secret in from the config file
func LoadConfig(configPath *string) error {
	b, err := ioutil.ReadFile(*configPath)
	if err != nil {
		return err
	}

	errr := json.Unmarshal(b, &config)
	if errr != nil {
		return errr
	}

	// setup the consumer key and secret from the confis
	oauthClient.Credentials = config.Consumer

	return nil
}

func apiGet() (*Response, error) {
	resp := Response{}
	r, err := oauthClient.Get(http.DefaultClient, &config.Access, apiBase, form)
	if err != nil {
		log.Fatal(err)
	}

	defer r.Body.Close()

	if r.StatusCode != 200 {
		log.Fatal(r.Status)
	}

	contents, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(contents, &resp)
	if err != nil {
		return nil, err
	}
	if resp.Stat != "ok" {
		return nil, &Error{ string(contents) }
	}

	return &resp, nil
}

func getAllPages(fn func(*Response)) {
	var wg sync.WaitGroup

	data, err := apiGet()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("got page 1")
	wg.Add(data.Data.Pages)
	go func() {
		defer wg.Done()
		fn(data)
	}()

	// get the rest of the pages
	for page := 2; page <= data.Data.Pages; page++ {
		go func(page int) {
			defer wg.Done()

			form.Set("page", strconv.Itoa(page))

			data, err := apiGet()
			if err != nil {
				log.Fatal(err)
			}

			fmt.Println("got page ",page)

			fn(data)
		}(page)
	}

	wg.Wait()
}

func GetPhotos(flickrUserId string) (*map[string]Photo) {
	form.Set("method", "flickr.photos.search")
	form.Set("format", "json")
	form.Set("nojsoncallback", "1")
	form.Set("user_id", flickrUserId)
	form.Set("per_page", "500") // max page size

	photos := make(map[string]Photo)

	getAllPages(func(data *Response) {
		// extract into photos map
		for _, img := range data.Data.Photos {
			photos[img.Title] = img
		}
	})

	return &photos
}

func GetLogin() *FlickrUser {
	form.Set("method", "flickr.test.login")
	form.Set("format", "json")
	form.Set("nojsoncallback", "1")

	data, err := apiGet()
	if err != nil {
		log.Fatal(err)
	}

	return &data.User
}
<commit_msg>get rid of the parallel gets of the pages due to flickr error<commit_after>package photosync

import (
	"github.com/garyburd/go-oauth/oauth"
	"fmt"
	"net/http"
	"net/url"
	"log"
	"io/ioutil"
	"encoding/json"
	"strconv"
	"sync"
)

type PhotosMap map[string]Photo

type Config struct {
	Consumer oauth.Credentials
	Access oauth.Credentials
	FlickrUserId string `json:"flickr_user_id"`
}

type Photo struct {
	Id int `json:"string"`
	Owner string
	Secret string
	Title string
	Ispublic int `json:"string"`
	Isfriend int `json:"string"`
	Isfamily int `json:"string"`
}

type Response struct {
	Stat string
	Data struct {
		Page int
		Pages int
		Perpage int
		Total string
		Photos []Photo `json:"photo"`
	} `json:"photos"`
	User FlickrUser `json:"user"`
}

type FlickrUser struct {
	Id string
	Username struct {
		Content string `json:"_content"`
	} `json:"username"`
}

// API Error type
type Error struct {
	response string
}
func (e *Error) Error() string {
	return fmt.Sprintf("API fail: %s", e.response)
}

var oauthClient = oauth.Client {
	TemporaryCredentialRequestURI: "https://api.flickr.com/services/oauth/request_token",
	ResourceOwnerAuthorizationURI: "https://api.flickr.com/services/oauth/authorize",
	TokenRequestURI:               "https://api.flickr.com/services/oauth/access_token",
}

var config Config
var apiBase = "https://api.flickr.com/services/rest"
var form = url.Values{}

// Load the consumer key and secret in from the config file
func LoadConfig(configPath *string) error {
	b, err := ioutil.ReadFile(*configPath)
	if err != nil {
		return err
	}

	errr := json.Unmarshal(b, &config)
	if errr != nil {
		return errr
	}

	// setup the consumer key and secret from the confis
	oauthClient.Credentials = config.Consumer

	return nil
}

func apiGet() (*Response, error) {
	resp := Response{}
	r, err := oauthClient.Get(http.DefaultClient, &config.Access, apiBase, form)
	if err != nil {
		log.Fatal(err)
	}

	defer r.Body.Close()

	if r.StatusCode != 200 {
		log.Fatal(r.Status)
	}

	contents, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(contents, &resp)
	if err != nil {
		return nil, err
	}
	if resp.Stat != "ok" {
		return nil, &Error{ string(contents) }
	}

	return &resp, nil
}

func getAllPages(fn func(*Response)) {
	var wg sync.WaitGroup

	data, err := apiGet()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Print("\rloading: ",int((float32(1)/float32(data.Data.Pages))*100),"%")
	wg.Add(data.Data.Pages)
	//go func() {
	func() {
		defer wg.Done()
		fn(data)
	}()

	// get the rest of the pages
	for page := 2; page <= data.Data.Pages; page++ {
		// comment out the parallel requesting as the flickr api seems occasionally return a dup page response
		//go func(page int) {
		func(page int) {
			defer wg.Done()

			form.Set("page", strconv.Itoa(page))

			data, err := apiGet()
			if err != nil {
				log.Fatal(err)
			}

			fmt.Print("\rloading: ",int((float32(page)/float32(data.Data.Pages))*100),"%")

			fn(data)
		}(page)
	}

	wg.Wait()
	fmt.Println("")
}

func GetPhotos(flickrUserId string) (*PhotosMap) {
	form.Set("method", "flickr.photos.search")
	form.Set("format", "json")
	form.Set("nojsoncallback", "1")
	form.Set("user_id", flickrUserId)
	form.Set("per_page", "500") // max page size

	photos := make(PhotosMap)

	getAllPages(func(data *Response) {
		// extract into photos map
		for _, img := range data.Data.Photos {
			photos[img.Title] = img
		}
	})

	return &photos
}

func GetLogin() *FlickrUser {
	form.Set("method", "flickr.test.login")
	form.Set("format", "json")
	form.Set("nojsoncallback", "1")

	data, err := apiGet()
	if err != nil {
		log.Fatal(err)
	}

	return &data.User
}
<|endoftext|>


<commit_before>package rtfdoc

import "fmt"

// AddParagraph return new instance of Paragraph
func (doc *Document) AddParagraph() *Paragraph {
	p := Paragraph{
		align:   AlignCenter,
		indent:  "\\fl360",
		content: nil,
		generalSettings: generalSettings{
			colorTable: doc.colorTable,
			fontColor:  doc.fontColor,
		},
		allowedWidth: doc.maxWidth,
	}
	p.updateMaxWidth()
	doc.content = append(doc.content, &p)
	return &p
}

func (par *Paragraph) updateMaxWidth() *Paragraph {
	par.maxWidth = par.allowedWidth
	return par
}

func (par Paragraph) compose() string {
	indentStr := fmt.Sprintf("\\fi%d \\li%d \\ri%d",
		par.indentFirstLine,
		par.indentLeftIndent,
		par.indentRightIndent)
	res := fmt.Sprintf("\n{\\pard %s \\q%s", indentStr, par.align)
	if par.isTable {
		res += "\\intbl"
	}

	for _, c := range par.content {
		res += c.compose()
	}
	// res += "\n\\par}"
	return res
}

// SetIndentFirstLine function sets first line indent in twips
func (par *Paragraph) SetIndentFirstLine(value int) *Paragraph {
	par.indentFirstLine = value
	return par
}

// SetIndentRight function sets right indent in twips
func (par *Paragraph) SetIndentRight(value int) *Paragraph {
	par.indentRightIndent = value
	return par
}

// SetIndentLeft function sets left indent in twips
func (par *Paragraph) SetIndentLeft(value int) *Paragraph {
	par.indentLeftIndent = value
	return par
}

// SetAlign sets Paragraph align (c/center, l/left, r/right, j/justify)
func (par *Paragraph) SetAlign(align string) *Paragraph {
	for _, i := range []string{
		AlignCenter,
		AlignLeft,
		AlignRight,
		AlignJustify,
		AlignDistribute,
	} {
		if i == align {
			par.align = i
		}
	}

	return par
}
<commit_msg>par before pard fix test #5<commit_after>package rtfdoc

import "fmt"

// AddParagraph return new instance of Paragraph
func (doc *Document) AddParagraph() *Paragraph {
	p := Paragraph{
		align:   AlignCenter,
		indent:  "\\fl360",
		content: nil,
		generalSettings: generalSettings{
			colorTable: doc.colorTable,
			fontColor:  doc.fontColor,
		},
		allowedWidth: doc.maxWidth,
	}
	p.updateMaxWidth()
	doc.content = append(doc.content, &p)
	return &p
}

func (par *Paragraph) updateMaxWidth() *Paragraph {
	par.maxWidth = par.allowedWidth
	return par
}

func (par Paragraph) compose() string {
	indentStr := fmt.Sprintf("\\fi%d \\li%d \\ri%d",
		par.indentFirstLine,
		par.indentLeftIndent,
		par.indentRightIndent)
	res := fmt.Sprintf("\n{\\par \\pard %s \\q%s", indentStr, par.align)
	if par.isTable {
		res += "\\intbl"
	}

	for _, c := range par.content {
		res += c.compose()
	}
	// res += "\n\\par}"
	res += "}"
	return res
}

// SetIndentFirstLine function sets first line indent in twips
func (par *Paragraph) SetIndentFirstLine(value int) *Paragraph {
	par.indentFirstLine = value
	return par
}

// SetIndentRight function sets right indent in twips
func (par *Paragraph) SetIndentRight(value int) *Paragraph {
	par.indentRightIndent = value
	return par
}

// SetIndentLeft function sets left indent in twips
func (par *Paragraph) SetIndentLeft(value int) *Paragraph {
	par.indentLeftIndent = value
	return par
}

// SetAlign sets Paragraph align (c/center, l/left, r/right, j/justify)
func (par *Paragraph) SetAlign(align string) *Paragraph {
	for _, i := range []string{
		AlignCenter,
		AlignLeft,
		AlignRight,
		AlignJustify,
		AlignDistribute,
	} {
		if i == align {
			par.align = i
		}
	}

	return par
}
<|endoftext|>


<commit_before>//	Copyright 2013 slowfei And The Contributors All rights reserved.
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//
//  Create on 2013-8-16
//  Update on 2014-06-24
//  Email  slowfei#foxmail.com
//  Home   http://www.slowfei.com

//
//	web form params pack struct
//
package leafveingo

import (
	"github.com/slowfei/gosfcore/log"
	"github.com/slowfei/gosfcore/utils/reflect"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

var (
	//	判断数组标识的正则
	_arrayTagRex = regexp.MustCompile("\\[\\d*\\]")
)

//
//	form params pack struct
//
type parampack struct {
	params  url.Values                         // form or get params
	files   map[string][]*multipart.FileHeader // file
	fileNum int                                // all file number
}

/**
 *	parse form
 *
 *	@param req
 *	@return param		form params pack struct
 *	@return err
 */
func parampackParseForm(req *http.Request, fileUploadSize int64) (param *parampack, err error) {

	switch req.Method {
	case "GET":
		param = new(parampack)
		param.params = req.URL.Query()
	case "POST":
		contentType := req.Header.Get("Content-Type")
		enctype, _, e := mime.ParseMediaType(contentType)
		if nil != e {
			err = e
			return
		}

		param = new(parampack)

		switch {
		case enctype == "application/x-www-form-urlencoded":
			err = req.ParseForm()
			if nil != err {
				return
			}
			//	考虑安全因素，让调用则知道请求的参数来自Form还是Query所以进行POST请求就只获取Form的参数
			param.params = req.PostForm

		case enctype == "multipart/form-data":
			err = req.ParseMultipartForm(fileUploadSize)
			if nil != err {
				return
			}

			// ParseMultipartForm()解析已经调用了ParseForm()
			param.params = url.Values(req.MultipartForm.Value)

			if nil != req.MultipartForm && 0 < len(req.MultipartForm.File) {
				for k, v := range req.MultipartForm.File {
					//	添加空字符串的主要目的是为了能够在创建结构时初始化切片的数量
					param.params.Set(k, "")
					fNum := len(v)
					if 1 < fNum {
						param.fileNum += fNum
					} else {
						param.fileNum++
					}
				}
				param.files = req.MultipartForm.File
			}

		}
	default:
		param = new(parampack)
		param.params = make(url.Values)
	}

	return
}

/**
 *	根据field的字段名称设置struct字段属性值
 *	字段名称可为"type.tag.TagName", 以"."作为子字段的名称分割
 *
 *	@param sutValue		被设值的结构反射类型
 *	@param fieldName	字段名(type.tag.TagName) or (tagnName)
 *	@param value		设值值
 */
func parampackSetStructFieldValue(sutValue reflect.Value, fieldName string, value interface{}) {
	fieldName = strings.TrimSpace(fieldName)
	sutValueElem := reflect.Indirect(sutValue)

	if len(fieldName) == 0 {
		return
	}

	//	由于考虑到key的值可能为(type.tag.TagName)，嵌套的赋值，所以需要进行"."的分割，每次获取slice的第一项
	fieldsName := strings.Split(fieldName, ".")
	childFieldName := strings.Title(fieldsName[0])

	if sutValueElem.Kind() == reflect.Slice {
		//	集合字段的设值操作

		strIndex := _arrayTagRex.FindString(childFieldName)
		if 0 >= len(strIndex) {
			//	判断如果是数组类型，但是设值的字段名称不是数组的标识则跳过(数组标识Users[0])
			//	由于有可能是需要直接设置数组参数，但是字段名称中(childFieldName = "Files")中未包含"[\d]"的标识
			//	所以尝试直接设值
			parampackSetParamValue(sutValueElem, fieldName, value)
			return
		}

		intIndex, e := strconv.Atoi(strIndex[1 : len(strIndex)-1])
		if nil == e && intIndex < sutValueElem.Len() {
			fieldValue := sutValueElem.Index(intIndex)

			parampackSetFieldValue(fieldValue, fieldName, fieldsName, value)
		}

	} else {

		//	判断是否为数组标识，如果是的话就删除，例如：Users[0] 删除[0] = Users
		//	这样便于FieldByName查找到相应的字段信息
		reIndex := _arrayTagRex.FindStringIndex(childFieldName)
		if 0 < len(reIndex) {
			childFieldName = childFieldName[:reIndex[0]]
		}

		//	查找字段
		fieldValue := sutValueElem.FieldByName(childFieldName)

		parampackSetFieldValue(fieldValue, fieldName, fieldsName, value)
	}
}

/**
 *	根据url和form参数信息创建一个struct体，如果存在集合结构会根据form参数信息make分配切片大小。
 *
 *	@param structType 需要操作的的结构
 *	@param urlValues  url或form请求参数(主要为了操作slice的创建元素)
 *	@return	返回创建好的函数反射对象信息（指针类型的）
 */
func parampackNewStructPtr(structType reflect.Type, urlValues url.Values) reflect.Value {
	//	考虑到结构内包含指针，如果不进行new的话直接(.)会爆出空指针异常，所以这里需要遍历每个对象

	if structType.Kind() != reflect.Struct {
		return reflect.Zero(structType)
	}

	structValue := reflect.New(structType)

	if 0 == len(urlValues) {
		return structValue
	}

	//	begin 分析数组	TODO 需要调整集合下的集合 type[0].tmpe[0]
	arraySizeMap := make(map[string]int) //	key = jsonfiledname ,value = array size

	for k, _ := range urlValues {
		//	存储(oneUser.hobbys.names)操作的连接名 如果存在集合则存储为key
		tempJoinName := ""

		fields := strings.Split(k, ".")

		for _, field := range fields {

			fieldLower := strings.ToLower(field)

			strIndex := _arrayTagRex.FindString(fieldLower)
			if 0 != len(strIndex) {
				intIndex, err := strconv.Atoi(strIndex[1 : len(strIndex)-1])
				if nil == err {

					//	这里为key做准备
					tempJoinName += fieldLower[:len(fieldLower)-len(strIndex)]

					//	由于下标是从0开始计算，所以需要+1作为 array size
					intIndex += 1

					if sizeIndex, ok := arraySizeMap[tempJoinName]; ok {

						if sizeIndex < intIndex {
							arraySizeMap[tempJoinName] = intIndex
						}
					} else {
						arraySizeMap[tempJoinName] = intIndex
					}

					//	由于可能存在数组下还存在集合（temp[0].temp2[0]），所以继续累加"[index]."继续执行。
					tempJoinName += strIndex + "."
				} else {
					//	进入到这部基本上是获取的 "[ 错误下标 ]" 才导致的，所欲当没有下标出现，继续累加
					tempJoinName += fieldLower + "."
				}
			} else {
				tempJoinName += fieldLower + "."
			}

		}
	}
	//	end 分析数组

	//	执行递归子字段操作
	parampackNewStructFindFiled(structValue, "", arraySizeMap)

	return structValue
}

/**
 *	创建结构体递归遍历子字段操作
 *
 *	@param structV			递归操作字段
 *	@param joinFieldName	字段连接名
 *	@param arraySizeMap		解析后的集合数量设值map
 */
func parampackNewStructFindFiled(structV reflect.Value, joinFieldName string, arraySizeMap map[string]int) {
	structV = reflect.Indirect(structV)
	if reflect.Struct != structV.Kind() {
		return
	}

	//	遍历结构字段寻找需要初始化的slice
	filedCount := structV.NumField()
	for i := 0; i < filedCount; i++ {
		childField := structV.Field(i)
		childFieldType := childField.Type()
		childFieldName := structV.Type().Field(i).Name

		if !childField.CanSet() {
			continue
		}

		if 0 != len(joinFieldName) && '.' != joinFieldName[len(joinFieldName)-1] {
			joinFieldName += "."
		}

		if reflect.Ptr == childFieldType.Kind() && childField.IsNil() {
			//	初始化子结构指针的操作
			childField.Set(reflect.New(childField.Type().Elem()))
		}

		switch reflect.Indirect(childField).Kind() {
		case reflect.Struct:
			if !parampackFilterParamPackStructType(childField.Type()) {
				parampackNewStructFindFiled(childField, joinFieldName+childFieldName, arraySizeMap)
			}
		case reflect.Slice:

			tempJoinName := joinFieldName + childFieldName

			if sliceSize, ok := arraySizeMap[strings.ToLower(tempJoinName)]; ok {

				valueElem := reflect.Indirect(childField)
				valueElem.Set(reflect.MakeSlice(valueElem.Type(), sliceSize, sliceSize))

				//	继续执行集合子元素的遍历
				if !parampackFilterParamPackStructType(valueElem.Type()) {
					for j := 0; j < sliceSize; j++ {
						childIndex := valueElem.Index(j)

						if reflect.Ptr == childIndex.Kind() && childIndex.IsNil() {
							//	初始化子元素指针的操作
							childIndex.Set(reflect.New(childIndex.Type().Elem()))
						}

						//	当前由于是数组类型，所以传递连接名的时候加上"[index]"下标往下进行操作
						tempJoinName = joinFieldName + childFieldName + "[" + strconv.Itoa(j) + "]"

						parampackNewStructFindFiled(childIndex, tempJoinName, arraySizeMap)
					}
				}
			}
		}
	}
}

/**
 *	set param value
 *
 *	@param fieldValue 字段的反射对象
 *	@param fieldName  字段名
 *	@param value	  set value
 */
func parampackSetParamValue(fieldValue reflect.Value, fieldName string, value interface{}) {
	err := SFReflectUtil.SetBaseTypeValue(fieldValue, value)
	if nil != err {
		SFLog.Error("%s set value error: %s", fieldName, err.Error())

	}
}

/**
 *	过滤参数封装一些系统的struct
 *	在有些设置结构参数时，系统的struct不必要再递归分析
 *
 *	@return true 属于过滤字段 false不是过滤的字段
 */
func parampackFilterParamPackStructType(valueType reflect.Type) bool {
	result := true
	// strings.Index(fieldValue.Type().String(), "multipart.FileHeader")
	switch valueType.String() {
	case "[]multipart.FileHeader", "[]*multipart.FileHeader", "*multipart.FileHeader", "multipart.FileHeader":
	case "":
		SFLog.Error("can not read type.String() : %v", valueType)
	default:
		result = false
	}
	return result
}

/**
 *	针对字段进行设值
 *
 *	@param fieldValue		字段反射对象
 *	@param fieldName		字段名，以便递归寻找下个设值字段
 *	@param fieldSplitName	字段名分割集合，以"."进行分割，主要是为了递归子字段进行拼接传递
 *	@param value			设置值
 */
func parampackSetFieldValue(fieldValue reflect.Value, fieldName string, fieldSplitName []string, value interface{}) {

	if fieldValue.IsValid() {

		//	为递归下一个参数做准备，保留后面的参数名(tag.TagName)
		isRec := false
		joinLaterFieldName := ""
		if 1 < len(fieldSplitName) {
			joinLaterFieldName = strings.Join(fieldSplitName[1:], ".")
			isRec = true
			//	进入这里表明还需要进行一次字段查询，所以需要进行递归操作，直到截取到最后一位的参数名标识(TagName)
		}

		switch fieldValue.Kind() {
		case reflect.Ptr:

			if fieldValue.IsNil() {
				fieldValue.Set(reflect.New(fieldValue.Type().Elem()))
			}

			//	指针与非指针区分开，主要是在进行参数设值的时候对应与设置值相同的类型，减免指针的过多操作。
			switch fieldValue.Elem().Kind() {
			case reflect.Struct:
				if isRec && !parampackFilterParamPackStructType(fieldValue.Type().Elem()) {
					parampackSetStructFieldValue(fieldValue, joinLaterFieldName, value)
				} else {
					parampackSetParamValue(fieldValue, fieldName, value)
				}

			case reflect.Slice:
				//	如果属于切片类型，传递fieldName主要是在操作一遍集合元素的赋值，因为不是结构类型无需再向下查找
				parampackSetStructFieldValue(fieldValue, fieldName, value)
			default:
				parampackSetParamValue(fieldValue, fieldName, value)
			}

		case reflect.Struct:

			if isRec && !parampackFilterParamPackStructType(fieldValue.Type()) {
				parampackSetStructFieldValue(fieldValue, joinLaterFieldName, value)
			} else {
				//	如果检测的是系统或则非用户定义的struct就可以直接赋值了，赋值那里已经做了匹配类型才进行赋值的处理
				parampackSetParamValue(fieldValue, fieldName, value)
			}
		case reflect.Slice:
			parampackSetStructFieldValue(fieldValue, fieldName, value)
		default:
			parampackSetParamValue(fieldValue, fieldName, value)
		}

	}
}
<commit_msg>修改其他Method请求方式的参数获取<commit_after>//	Copyright 2013 slowfei And The Contributors All rights reserved.
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//
//  Create on 2013-8-16
//  Update on 2014-07-11
//  Email  slowfei#foxmail.com
//  Home   http://www.slowfei.com

//
//	web form params pack struct
//
package leafveingo

import (
	"github.com/slowfei/gosfcore/log"
	"github.com/slowfei/gosfcore/utils/reflect"
	"mime"
	"mime/multipart"
	"net/http"
	"net/url"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

var (
	//	判断数组标识的正则
	_arrayTagRex = regexp.MustCompile("\\[\\d*\\]")
)

//
//	form params pack struct
//
type parampack struct {
	params  url.Values                         // form or get params
	files   map[string][]*multipart.FileHeader // file
	fileNum int                                // all file number
}

/**
 *	parse form
 *
 *	@param req
 *	@return param		form params pack struct
 *	@return err
 */
func parampackParseForm(req *http.Request, fileUploadSize int64) (param *parampack, err error) {

	switch req.Method {
	case "GET":
		param = new(parampack)
		param.params = req.URL.Query()
	case "POST":
		contentType := req.Header.Get("Content-Type")
		enctype, _, e := mime.ParseMediaType(contentType)
		if nil != e {
			err = e
			return
		}

		param = new(parampack)

		switch {
		case enctype == "application/x-www-form-urlencoded":
			err = req.ParseForm()
			if nil != err {
				return
			}
			//	考虑安全因素，让调用则知道请求的参数来自Form还是Query所以进行POST请求就只获取Form的参数
			param.params = req.PostForm

		case enctype == "multipart/form-data":
			err = req.ParseMultipartForm(fileUploadSize)
			if nil != err {
				return
			}

			// ParseMultipartForm()解析已经调用了ParseForm()
			param.params = url.Values(req.MultipartForm.Value)

			if nil != req.MultipartForm && 0 < len(req.MultipartForm.File) {
				for k, v := range req.MultipartForm.File {
					//	添加空字符串的主要目的是为了能够在创建结构时初始化切片的数量
					param.params.Set(k, "")
					fNum := len(v)
					if 1 < fNum {
						param.fileNum += fNum
					} else {
						param.fileNum++
					}
				}
				param.files = req.MultipartForm.File
			}

		}
	default:
		param = new(parampack)
		param.params = req.URL.Query()
	}

	return
}

/**
 *	根据field的字段名称设置struct字段属性值
 *	字段名称可为"type.tag.TagName", 以"."作为子字段的名称分割
 *
 *	@param sutValue		被设值的结构反射类型
 *	@param fieldName	字段名(type.tag.TagName) or (tagnName)
 *	@param value		设值值
 */
func parampackSetStructFieldValue(sutValue reflect.Value, fieldName string, value interface{}) {
	fieldName = strings.TrimSpace(fieldName)
	sutValueElem := reflect.Indirect(sutValue)

	if len(fieldName) == 0 {
		return
	}

	//	由于考虑到key的值可能为(type.tag.TagName)，嵌套的赋值，所以需要进行"."的分割，每次获取slice的第一项
	fieldsName := strings.Split(fieldName, ".")
	childFieldName := strings.Title(fieldsName[0])

	if sutValueElem.Kind() == reflect.Slice {
		//	集合字段的设值操作

		strIndex := _arrayTagRex.FindString(childFieldName)
		if 0 >= len(strIndex) {
			//	判断如果是数组类型，但是设值的字段名称不是数组的标识则跳过(数组标识Users[0])
			//	由于有可能是需要直接设置数组参数，但是字段名称中(childFieldName = "Files")中未包含"[\d]"的标识
			//	所以尝试直接设值
			parampackSetParamValue(sutValueElem, fieldName, value)
			return
		}

		intIndex, e := strconv.Atoi(strIndex[1 : len(strIndex)-1])
		if nil == e && intIndex < sutValueElem.Len() {
			fieldValue := sutValueElem.Index(intIndex)

			parampackSetFieldValue(fieldValue, fieldName, fieldsName, value)
		}

	} else {

		//	判断是否为数组标识，如果是的话就删除，例如：Users[0] 删除[0] = Users
		//	这样便于FieldByName查找到相应的字段信息
		reIndex := _arrayTagRex.FindStringIndex(childFieldName)
		if 0 < len(reIndex) {
			childFieldName = childFieldName[:reIndex[0]]
		}

		//	查找字段
		fieldValue := sutValueElem.FieldByName(childFieldName)

		parampackSetFieldValue(fieldValue, fieldName, fieldsName, value)
	}
}

/**
 *	根据url和form参数信息创建一个struct体，如果存在集合结构会根据form参数信息make分配切片大小。
 *
 *	@param structType 需要操作的的结构
 *	@param urlValues  url或form请求参数(主要为了操作slice的创建元素)
 *	@return	返回创建好的函数反射对象信息（指针类型的）
 */
func parampackNewStructPtr(structType reflect.Type, urlValues url.Values) reflect.Value {
	//	考虑到结构内包含指针，如果不进行new的话直接(.)会爆出空指针异常，所以这里需要遍历每个对象

	if structType.Kind() != reflect.Struct {
		return reflect.Zero(structType)
	}

	structValue := reflect.New(structType)

	if 0 == len(urlValues) {
		return structValue
	}

	//	begin 分析数组	TODO 需要调整集合下的集合 type[0].tmpe[0]
	arraySizeMap := make(map[string]int) //	key = jsonfiledname ,value = array size

	for k, _ := range urlValues {
		//	存储(oneUser.hobbys.names)操作的连接名 如果存在集合则存储为key
		tempJoinName := ""

		fields := strings.Split(k, ".")

		for _, field := range fields {

			fieldLower := strings.ToLower(field)

			strIndex := _arrayTagRex.FindString(fieldLower)
			if 0 != len(strIndex) {
				intIndex, err := strconv.Atoi(strIndex[1 : len(strIndex)-1])
				if nil == err {

					//	这里为key做准备
					tempJoinName += fieldLower[:len(fieldLower)-len(strIndex)]

					//	由于下标是从0开始计算，所以需要+1作为 array size
					intIndex += 1

					if sizeIndex, ok := arraySizeMap[tempJoinName]; ok {

						if sizeIndex < intIndex {
							arraySizeMap[tempJoinName] = intIndex
						}
					} else {
						arraySizeMap[tempJoinName] = intIndex
					}

					//	由于可能存在数组下还存在集合（temp[0].temp2[0]），所以继续累加"[index]."继续执行。
					tempJoinName += strIndex + "."
				} else {
					//	进入到这部基本上是获取的 "[ 错误下标 ]" 才导致的，所欲当没有下标出现，继续累加
					tempJoinName += fieldLower + "."
				}
			} else {
				tempJoinName += fieldLower + "."
			}

		}
	}
	//	end 分析数组

	//	执行递归子字段操作
	parampackNewStructFindFiled(structValue, "", arraySizeMap)

	return structValue
}

/**
 *	创建结构体递归遍历子字段操作
 *
 *	@param structV			递归操作字段
 *	@param joinFieldName	字段连接名
 *	@param arraySizeMap		解析后的集合数量设值map
 */
func parampackNewStructFindFiled(structV reflect.Value, joinFieldName string, arraySizeMap map[string]int) {
	structV = reflect.Indirect(structV)
	if reflect.Struct != structV.Kind() {
		return
	}

	//	遍历结构字段寻找需要初始化的slice
	filedCount := structV.NumField()
	for i := 0; i < filedCount; i++ {
		childField := structV.Field(i)
		childFieldType := childField.Type()
		childFieldName := structV.Type().Field(i).Name

		if !childField.CanSet() {
			continue
		}

		if 0 != len(joinFieldName) && '.' != joinFieldName[len(joinFieldName)-1] {
			joinFieldName += "."
		}

		if reflect.Ptr == childFieldType.Kind() && childField.IsNil() {
			//	初始化子结构指针的操作
			childField.Set(reflect.New(childField.Type().Elem()))
		}

		switch reflect.Indirect(childField).Kind() {
		case reflect.Struct:
			if !parampackFilterParamPackStructType(childField.Type()) {
				parampackNewStructFindFiled(childField, joinFieldName+childFieldName, arraySizeMap)
			}
		case reflect.Slice:

			tempJoinName := joinFieldName + childFieldName

			if sliceSize, ok := arraySizeMap[strings.ToLower(tempJoinName)]; ok {

				valueElem := reflect.Indirect(childField)
				valueElem.Set(reflect.MakeSlice(valueElem.Type(), sliceSize, sliceSize))

				//	继续执行集合子元素的遍历
				if !parampackFilterParamPackStructType(valueElem.Type()) {
					for j := 0; j < sliceSize; j++ {
						childIndex := valueElem.Index(j)

						if reflect.Ptr == childIndex.Kind() && childIndex.IsNil() {
							//	初始化子元素指针的操作
							childIndex.Set(reflect.New(childIndex.Type().Elem()))
						}

						//	当前由于是数组类型，所以传递连接名的时候加上"[index]"下标往下进行操作
						tempJoinName = joinFieldName + childFieldName + "[" + strconv.Itoa(j) + "]"

						parampackNewStructFindFiled(childIndex, tempJoinName, arraySizeMap)
					}
				}
			}
		}
	}
}

/**
 *	set param value
 *
 *	@param fieldValue 字段的反射对象
 *	@param fieldName  字段名
 *	@param value	  set value
 */
func parampackSetParamValue(fieldValue reflect.Value, fieldName string, value interface{}) {
	err := SFReflectUtil.SetBaseTypeValue(fieldValue, value)
	if nil != err {
		SFLog.Error("%s set value error: %s", fieldName, err.Error())

	}
}

/**
 *	过滤参数封装一些系统的struct
 *	在有些设置结构参数时，系统的struct不必要再递归分析
 *
 *	@return true 属于过滤字段 false不是过滤的字段
 */
func parampackFilterParamPackStructType(valueType reflect.Type) bool {
	result := true
	// strings.Index(fieldValue.Type().String(), "multipart.FileHeader")
	switch valueType.String() {
	case "[]multipart.FileHeader", "[]*multipart.FileHeader", "*multipart.FileHeader", "multipart.FileHeader":
	case "":
		SFLog.Error("can not read type.String() : %v", valueType)
	default:
		result = false
	}
	return result
}

/**
 *	针对字段进行设值
 *
 *	@param fieldValue		字段反射对象
 *	@param fieldName		字段名，以便递归寻找下个设值字段
 *	@param fieldSplitName	字段名分割集合，以"."进行分割，主要是为了递归子字段进行拼接传递
 *	@param value			设置值
 */
func parampackSetFieldValue(fieldValue reflect.Value, fieldName string, fieldSplitName []string, value interface{}) {

	if fieldValue.IsValid() {

		//	为递归下一个参数做准备，保留后面的参数名(tag.TagName)
		isRec := false
		joinLaterFieldName := ""
		if 1 < len(fieldSplitName) {
			joinLaterFieldName = strings.Join(fieldSplitName[1:], ".")
			isRec = true
			//	进入这里表明还需要进行一次字段查询，所以需要进行递归操作，直到截取到最后一位的参数名标识(TagName)
		}

		switch fieldValue.Kind() {
		case reflect.Ptr:

			if fieldValue.IsNil() {
				fieldValue.Set(reflect.New(fieldValue.Type().Elem()))
			}

			//	指针与非指针区分开，主要是在进行参数设值的时候对应与设置值相同的类型，减免指针的过多操作。
			switch fieldValue.Elem().Kind() {
			case reflect.Struct:
				if isRec && !parampackFilterParamPackStructType(fieldValue.Type().Elem()) {
					parampackSetStructFieldValue(fieldValue, joinLaterFieldName, value)
				} else {
					parampackSetParamValue(fieldValue, fieldName, value)
				}

			case reflect.Slice:
				//	如果属于切片类型，传递fieldName主要是在操作一遍集合元素的赋值，因为不是结构类型无需再向下查找
				parampackSetStructFieldValue(fieldValue, fieldName, value)
			default:
				parampackSetParamValue(fieldValue, fieldName, value)
			}

		case reflect.Struct:

			if isRec && !parampackFilterParamPackStructType(fieldValue.Type()) {
				parampackSetStructFieldValue(fieldValue, joinLaterFieldName, value)
			} else {
				//	如果检测的是系统或则非用户定义的struct就可以直接赋值了，赋值那里已经做了匹配类型才进行赋值的处理
				parampackSetParamValue(fieldValue, fieldName, value)
			}
		case reflect.Slice:
			parampackSetStructFieldValue(fieldValue, fieldName, value)
		default:
			parampackSetParamValue(fieldValue, fieldName, value)
		}

	}
}
<|endoftext|>