<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net"
	"encoding/json"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"syscall"
	"strconv"
	"time"
	"github.com/hoisie/redis"
)

const (
	VERSION                 = "0.1.0"
	MAX_UNPROCESSED_PACKETS = 1000
	MAX_UDP_PACKET_SIZE     = 512
)

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	expiryInterval   = flag.Int64("expiry-interval", 1, "Expiry interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
)

const (
	ACTION_SET_WARN = "set-warn"
	ACTION_SET_ERR = "set-err"
	ACTION_BEAT = "beat"
)

const (
	STATE_PAUSED  = "paused"
	STATE_OK      = "ok"
	STATE_WARNING = "warning"
	STATE_ERROR   = "error"
)

type Cmd struct {
	Action   string
	Service  string
	Value    int
}

type Service struct {
	Name           string
	LastValue      int
	LastBeat       int64
	WarningTimeout int64
	ErrorTimeout   int64
	State          string
}

var (
	In       = make(chan *Cmd, MAX_UNPROCESSED_PACKETS)
	signalchan chan os.Signal
	client redis.Client
)

func now() int64 { return time.Now().Unix() }

func getOrCreate(name string) (*Service, *Service) {
	service := &Service{
		Name: name,
		LastValue: -1,
		LastBeat: -1,
		WarningTimeout: -1,
		ErrorTimeout: -1,
		State: STATE_PAUSED,
	}

	if data, err := client.Get("lb.service." + name); err == nil {
		json.Unmarshal(data, &service)
	}
	var ref = *service
	return service, &ref
}

func (s *Service)GetExpiry(timeout int64) int64 {
	if timeout <= 0 {
		return 0
	}
	return s.LastBeat + timeout
}

func (s *Service)GetNextExpiry(ts int64) int64 {
	var next int64 = 0
	var warningExpiry = s.GetExpiry(s.WarningTimeout)
	var errorExpiry = s.GetExpiry(s.ErrorTimeout)
	if warningExpiry > 0 && warningExpiry > ts && (next == 0 || warningExpiry < next) {
		next = warningExpiry
	}
	if errorExpiry > 0 && errorExpiry > ts && (next == 0 || errorExpiry < next) {
		next = errorExpiry
	}
	log.Printf("now: %d, warning: %d, error: %d, chosen: %d", ts, warningExpiry, errorExpiry, next)
	return next
}

func (s *Service) UpdateState(ts int64) {
	s.State = STATE_OK
	var warningExpiry = s.GetExpiry(s.WarningTimeout)
	var errorExpiry = s.GetExpiry(s.ErrorTimeout)
	if warningExpiry > 0 && ts >= warningExpiry {
		s.State = STATE_WARNING
	}
	if errorExpiry > 0 && ts >= errorExpiry {
		s.State = STATE_ERROR
	}
}

func updateExpiry(service *Service, ts int64) {
	if service.State != STATE_PAUSED {
		if expiry := service.GetNextExpiry(ts); expiry > 0 {
			client.Zadd("lb.expiry", []byte(service.Name), float64(expiry))
			return
		}
	}
	client.Zrem("lb.expiry", []byte(service.Name))
}

func (s *Service)Save(ref *Service) {
	if *s != *ref {
		if s.State != ref.State {
			log.Printf("service %s, state %s -> %s", s.Name, ref.State, s.State)
		}
		if s.WarningTimeout != ref.WarningTimeout {
			log.Printf("service %s, warn %d -> %d", s.Name, ref.WarningTimeout, s.WarningTimeout)
		}
		if s.ErrorTimeout != ref.ErrorTimeout {
			log.Printf("service %s, err %d -> %d", s.Name, ref.ErrorTimeout, s.ErrorTimeout)
		}
		b, _ := json.Marshal(s)
		client.Set("lb.service." + s.Name, b)
	}
}

func monitor() {
	period := time.Duration(*expiryInterval) * time.Second
	ticker := time.NewTicker(period)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			return
		case <-ticker.C:
			var ts = now()
			if expired, err := client.Zrangebyscore("lb.expiry", 0, float64(now())); err == nil {
				for _, elem := range expired {
					var service, ref = getOrCreate(string(elem))
					service.UpdateState(ts)
					service.Save(ref)
					updateExpiry(service, ts)
				}
			}
		case s := <-In:
			var ts = now()
			var service, ref = getOrCreate(s.Service)
			switch s.Action {
			case ACTION_SET_WARN:
				service.WarningTimeout = int64(s.Value)
			case ACTION_SET_ERR:
				service.ErrorTimeout = int64(s.Value)
			case ACTION_BEAT:
				service.LastBeat = ts
			}
			service.UpdateState(ts)
			service.Save(ref)
			updateExpiry(service, ts)
		}
	}
}

var packetRegexp = regexp.MustCompile("^([^:]+)\\.(beat|warn|err):(-?[0-9]+)\\|(g|c|ms)(\\|@([0-9\\.]+))?\n?$")

func parseMessage(data []byte) []*Cmd {
	var output []*Cmd
	for _, line := range bytes.Split(data, []byte("\n")) {
		if len(line) == 0 {
			continue
		}

		item := packetRegexp.FindSubmatch(line)
		if len(item) == 0 {
			continue
		}

		var value int
		modifier := string(item[4])
		switch modifier {
		case "c":
			var vali, err = strconv.ParseInt(string(item[3]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseInt %s - %s", item[3], err)
				continue
			}
			value = int(vali)
		default:
			var valu, err = strconv.ParseUint(string(item[3]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", item[3], err)
				continue
			}
			value = int(valu)
		}
		var action string
		switch string(item[2]) {
		case "warn":
			action = ACTION_SET_WARN
		case "err":
			action = ACTION_SET_ERR
		case "beat":
			action = ACTION_BEAT
		}

		packet := &Cmd{
			Action: action,
			Service: string(item[1]),
			Value:    value,
		}
		output = append(output, packet)
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ERROR: ListenUDP - %s", err)
	}
	defer listener.Close()

	message := make([]byte, MAX_UDP_PACKET_SIZE)
	for {
		n, remaddr, err := listener.ReadFromUDP(message)
		if err != nil {
			log.Printf("ERROR: reading UDP packet from %+v - %s", remaddr, err)
			continue
		}

		for _, p := range parseMessage(message[:n]) {
			In <- p
		}
	}
}

func main() {
	flag.Parse()
	if *showVersion {
		fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
		return
	}

	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)

	go udpListener()
	monitor()
}
<commit_msg>Adding services to the "all"-label<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net"
	"encoding/json"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"syscall"
	"strconv"
	"time"
	"github.com/hoisie/redis"
)

const (
	VERSION                 = "0.1.0"
	MAX_UNPROCESSED_PACKETS = 1000
	MAX_UDP_PACKET_SIZE     = 512
)

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	expiryInterval   = flag.Int64("expiry-interval", 1, "Expiry interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
)

const (
	ACTION_SET_WARN = "set-warn"
	ACTION_SET_ERR = "set-err"
	ACTION_BEAT = "beat"
)

const (
	STATE_PAUSED  = "paused"
	STATE_OK      = "ok"
	STATE_WARNING = "warning"
	STATE_ERROR   = "error"
)

type Cmd struct {
	Action   string
	Service  string
	Value    int
}

type Service struct {
	Name           string
	LastValue      int
	LastBeat       int64
	LastUpdated    int64
	WarningTimeout int64
	ErrorTimeout   int64
	State          string
}

var (
	In       = make(chan *Cmd, MAX_UNPROCESSED_PACKETS)
	signalchan chan os.Signal
	client redis.Client
)

func now() int64 { return time.Now().Unix() }

func getOrCreate(name string) (*Service, *Service) {
	service := &Service{
		Name: name,
		LastValue: -1,
		LastBeat: -1,
		LastUpdated: -1,
		WarningTimeout: -1,
		ErrorTimeout: -1,
		State: STATE_PAUSED,
	}

	if data, err := client.Get("lb.service." + name); err == nil {
		json.Unmarshal(data, &service)
	}
	var ref = *service
	return service, &ref
}

func (s *Service)GetExpiry(timeout int64) int64 {
	if timeout <= 0 {
		return 0
	}
	return s.LastBeat + timeout
}

func (s *Service)GetNextExpiry(ts int64) int64 {
	var next int64 = 0
	var warningExpiry = s.GetExpiry(s.WarningTimeout)
	var errorExpiry = s.GetExpiry(s.ErrorTimeout)
	if warningExpiry > 0 && warningExpiry > ts && (next == 0 || warningExpiry < next) {
		next = warningExpiry
	}
	if errorExpiry > 0 && errorExpiry > ts && (next == 0 || errorExpiry < next) {
		next = errorExpiry
	}
	log.Printf("now: %d, warning: %d, error: %d, chosen: %d", ts, warningExpiry, errorExpiry, next)
	return next
}

func (s *Service) UpdateState(ts int64) {
	s.State = STATE_OK
	var warningExpiry = s.GetExpiry(s.WarningTimeout)
	var errorExpiry = s.GetExpiry(s.ErrorTimeout)
	if warningExpiry > 0 && ts >= warningExpiry {
		s.State = STATE_WARNING
	}
	if errorExpiry > 0 && ts >= errorExpiry {
		s.State = STATE_ERROR
	}
}

func updateExpiry(service *Service, ts int64) {
	if service.State != STATE_PAUSED {
		if expiry := service.GetNextExpiry(ts); expiry > 0 {
			client.Zadd("lb.expiry", []byte(service.Name), float64(expiry))
			return
		}
	}
	client.Zrem("lb.expiry", []byte(service.Name))
}

func (s *Service)Save(ref *Service, ts int64) {
	if *s != *ref {
		if s.State != ref.State {
			log.Printf("service %s, state %s -> %s", s.Name, ref.State, s.State)
		}
		if s.WarningTimeout != ref.WarningTimeout {
			log.Printf("service %s, warn %d -> %d", s.Name, ref.WarningTimeout, s.WarningTimeout)
		}
		if s.ErrorTimeout != ref.ErrorTimeout {
			log.Printf("service %s, err %d -> %d", s.Name, ref.ErrorTimeout, s.ErrorTimeout)
		}
		s.LastUpdated = ts
		b, _ := json.Marshal(s)
		client.Set("lb.service." + s.Name, b)
		if ref.LastUpdated < 0 {
			client.Sadd("lb.services.all", []byte(s.Name))
		}
	}
}

func monitor() {
	period := time.Duration(*expiryInterval) * time.Second
	ticker := time.NewTicker(period)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			return
		case <-ticker.C:
			var ts = now()
			if expired, err := client.Zrangebyscore("lb.expiry", 0, float64(now())); err == nil {
				for _, elem := range expired {
					var service, ref = getOrCreate(string(elem))
					service.UpdateState(ts)
					service.Save(ref, ts)
					updateExpiry(service, ts)
				}
			}
		case s := <-In:
			var ts = now()
			var service, ref = getOrCreate(s.Service)
			switch s.Action {
			case ACTION_SET_WARN:
				service.WarningTimeout = int64(s.Value)
			case ACTION_SET_ERR:
				service.ErrorTimeout = int64(s.Value)
			case ACTION_BEAT:
				service.LastBeat = ts
			}
			service.UpdateState(ts)
			service.Save(ref, ts)
			updateExpiry(service, ts)
		}
	}
}

var packetRegexp = regexp.MustCompile("^([^:]+)\\.(beat|warn|err):(-?[0-9]+)\\|(g|c|ms)(\\|@([0-9\\.]+))?\n?$")

func parseMessage(data []byte) []*Cmd {
	var output []*Cmd
	for _, line := range bytes.Split(data, []byte("\n")) {
		if len(line) == 0 {
			continue
		}

		item := packetRegexp.FindSubmatch(line)
		if len(item) == 0 {
			continue
		}

		var value int
		modifier := string(item[4])
		switch modifier {
		case "c":
			var vali, err = strconv.ParseInt(string(item[3]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseInt %s - %s", item[3], err)
				continue
			}
			value = int(vali)
		default:
			var valu, err = strconv.ParseUint(string(item[3]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", item[3], err)
				continue
			}
			value = int(valu)
		}
		var action string
		switch string(item[2]) {
		case "warn":
			action = ACTION_SET_WARN
		case "err":
			action = ACTION_SET_ERR
		case "beat":
			action = ACTION_BEAT
		}

		packet := &Cmd{
			Action: action,
			Service: string(item[1]),
			Value:    value,
		}
		output = append(output, packet)
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ERROR: ListenUDP - %s", err)
	}
	defer listener.Close()

	message := make([]byte, MAX_UDP_PACKET_SIZE)
	for {
		n, remaddr, err := listener.ReadFromUDP(message)
		if err != nil {
			log.Printf("ERROR: reading UDP packet from %+v - %s", remaddr, err)
			continue
		}

		for _, p := range parseMessage(message[:n]) {
			In <- p
		}
	}
}

func main() {
	flag.Parse()
	if *showVersion {
		fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
		return
	}

	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)

	go udpListener()
	monitor()
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"os"
	"reflect"
	"unicode"
)

type Controller struct {
	ControlMachine   string
	ControlAddr      string
	BackupController string
	BackupAddr       string
}

type General struct {
	AuthType                       string
	AuthInfo                       string
	CacheGroups                    string
	CheckpointType                 string
	CryptoType                     string
	CoreSpecPlugin                 string
	DisableRootJobs                string
	EnforcePartLimits              string
	Epilog                         string
	EpilogSlurmctld                string
	FirstJobId                     string
	MaxJobId                       string
	GresTypes                      string
	GroupUpdateForce               string
	GroupUpdateTime                string
	JobCheckpointDir               string
	JobCredentialPrivateKey        string
	JobCredentialPublicCertificate string
	JobFileAppend                  string
	JobRequeue                     string
	JobSubmitPlugins               string
	KillOnBadExit                  string
	Licenses                       string
	MailProg                       string
	MaxJobCount                    string
	MaxStepCount                   string
	MaxTasksPerNode                string
	MpiDefault                     string
	MpiParams                      string
	PluginDir                      string
	PlugStackConfig                string
	PrivateData                    string
	ProctrackType                  string
	Prolog                         string
	PrologSlurmctld                string
	PropagatePrioProcess           string
	PropagateResourceLimits        string
	PropagateResourceLimitsExcept  string
	ReturnToService                string
	SallocDefaultCommand           string
	SlurmctldPidFile               string
	SlurmctldPort                  string
	SlurmdPidFile                  string
	SlurmdPort                     string
	SlurmdSpoolDir                 string
	SlurmUser                      string
	SlurmdUser                     string
	SrunEpilog                     string
	SrunProlog                     string
	StateSaveLocation              string
	SwitchType                     string
	TaskEpilog                     string
	TaskPlugin                     string
	TaskPluginParam                string
	TaskProlog                     string
	TopologyPlugin                 string
	TmpFS                          string
	TrackWCKey                     string
	TreeWidth                      string
	UnkillableStepProgram          string
	UsePAM                         string
}

type Timers struct {
	BatchStartTimeout     string
	CompleteWait          string
	EpilogMsgTime         string
	GetEnvTimeout         string
	HealthCheckInterval   string
	HealthCheckProgram    string
	InactiveLimit         string
	KillWait              string
	MessageTimeout        string
	ResvOverRun           string
	MinJobAge             string
	OverTimeLimit         string
	SlurmctldTimeout      string
	SlurmdTimeout         string
	UnkillableStepTimeout string
	VSizeFactor           string
	Waittime              string
}

type Scheduling struct {
	DefMemPerCPU         string
	FastSchedule         string
	MaxMemPerCPU         string
	SchedulerRootFilter  string
	SchedulerTimeSlice   string
	SchedulerType        string
	SchedulerPort        string
	SelectType           string
	SelectTypeParameters string
}

type JobPriority struct {
	PriorityType             string
	PriorityDecayHalfLife    string
	PriorityCalcPeriod       string
	PriorityFavorSmall       string
	PriorityMaxAge           string
	PriorityUsageResetPeriod string
	PriorityWeightAge        string
	PriorityWeightFairshare  string
	PriorityWeightJobSize    string
	PriorityWeightPartition  string
	PriorityWeightQOS        string
}

type LoggingAccounting struct {
	AccountingStorageEnforce    string
	AccountingStorageHost       string
	AccountingStorageBackupHost string
	AccountingStorageLoc        string
	AccountingStoragePass       string
	AccountingStoragePort       string
	AccountingStorageType       string
	AccountingStorageUser       string
	AccountingStoreJobComment   string
	AcctGatherNodeFreq          string
	AcctGatherInfinibandType    string
	AcctGatherFilesystemType    string
	AcctGatherProfileType       string
	ClusterName                 string
	ChosLoc                     string
	DebugFlags                  string
	JobCompHost                 string
	JobCompLoc                  string
	JobCompPass                 string
	JobCompPort                 string
	JobCompType                 string
	JobCompUser                 string
	JobAcctGatherFrequency      string
	JobAcctGatherType           string
	SlurmctldDebug              string
	SlurmctldLogFile            string
	SlurmdDebug                 string
	SlurmdLogFile               string
	SlurmSchedLogFile           string
	SlurmSchedLogLevel          string
}

type PowerSave struct {
	SuspendProgram  string
	ResumeProgram   string
	SuspendTimeout  string
	ResumeTimeout   string
	ResumeRate      string
	SuspendExcNodes string
	SuspendExcParts string
	SuspendRate     string
	SuspendTime     string
}

type Node struct {
	NodeName        string
	NodeHostname    string
	NodeAddr        string
	Boards          string
	CoreSpecCount   string
	CoresPerSocket  string
	CPUs            string
	CPUSpecList     string
	Feature         string
	Gres            string
	MemSpecLimit    string
	Port            string
	Procs           string
	RealMemory      string
	Reason          string
	Sockets         string
	SocketsPerBoard string
	State           string
	ThreadsPerCore  string
	TmpDisk         string
	Weight          string
	DownNodes       string
}

type Partition struct {
	PartitionName        string
	AllocNodes           string
	AllowAccounts        string
	AllowGroups          string
	AllowQos             string
	Alternate            string
	Default              string
	DefMemPerCPU         string
	DefMemPerNode        string
	DenyAccounts         string
	DenyQos              string
	DefaultTime          string
	DisableRootJobs      string
	GraceTime            string
	Hidden               string
	LLN                  string
	MaxCPUsPerNode       string
	MaxMemPerCPU         string
	MaxMemPerNode        string
	MaxNodes             string
	MaxTime              string
	MinNodes             string
	Nodes                string
	PreemptMode          string
	Priority             string
	ReqResv              string
	RootOnly             string
	SelectTypeParameters string
	Shared               string
	State                string
}

type Frontend struct {
	AllowGroups  string
	AllowUsers   string
	DenyGroups   string
	DenyUsers    string
	FrontendName string
	FrontendAddr string
	Port         string
	Reason       string
	State        string
}

type Config struct {
	Controller        Controller
	General           General
	Timers            Timers
	Scheduling        Scheduling
	JobPriority       JobPriority
	LoggingAccounting LoggingAccounting
	PowerSave         PowerSave
	Node              []Node
	Partition         []Partition
	Frontend          []Frontend
}

func main() {
	var out io.Writer
	if len(os.Args) > 1 {
		fi, err := os.Create(os.Args[1])
		if err != nil {
			log.Println(err)
		}
		out = fi
	} else {
		out = os.Stdout
	}
	conf := Config{}
	conf.FromEvn()
	conf.Write(out)
}

func (c *Config) Write(w io.Writer) {

	w.Write([]byte("#slurm.conf\n\n"))
	w.Write([]byte("#\n# Controll Machines\n#\n"))
	writeStructToConfig(c.Controller, w)
}

func writeStructToConfig(st interface{}, w io.Writer) {

	dlim := []byte{'='}
	endl := []byte{'\n'}

	val := reflect.ValueOf(st)
	typ := val.Type()

	for i := 0; i < val.NumField(); i++ {
		configField := typ.Field(i)
		configValue := val.Field(i)
		s := configValue.String()
		if s == "" {
			continue
		}

		w.Write([]byte(configField.Name))
		w.Write(dlim)
		w.Write([]byte(s))
		w.Write(endl)
	}

}

func (c *Config) FromEvn() {
	val := reflect.ValueOf(c).Elem()
	for i := 0; i < val.NumField(); i++ {
		confVal := val.Field(i)
		switch confVal.Kind() {
		case reflect.Slice:
			for i := 0; i < confVal.Len(); i++ {
				fillStructFromEnv(confVal.Index(i))
			}
		case reflect.Struct:
			fillStructFromEnv(confVal)
		}
	}
}

func fillStructFromEnv(val reflect.Value) {
	typ := val.Type()
	for i := 0; i < val.NumField(); i++ {
		for i := 0; i < val.NumField(); i++ {
			configField := typ.Field(i)
			configVal := val.Field(i)

			envName := toEnvName(configField.Name)
			configVal.SetString(os.Getenv(envName))
		}
	}
}

func toEnvName(name string) string {
	envName := []rune("SLURM")
	for _, r := range name {
		if unicode.IsUpper(r) {
			envName = append(envName, '_', r)
		} else {
			envName = append(envName, unicode.ToUpper(r))
		}
	}
	return string(envName)
}
<commit_msg>Write out most of the config<commit_after>package main

import (
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"unicode"
)

type Controller struct {
	ControlMachine   string
	ControlAddr      string
	BackupController string
	BackupAddr       string
}

type General struct {
	AuthType                       string
	AuthInfo                       string
	CacheGroups                    string
	CheckpointType                 string
	CryptoType                     string
	CoreSpecPlugin                 string
	DisableRootJobs                string
	EnforcePartLimits              string
	Epilog                         string
	EpilogSlurmctld                string
	FirstJobId                     string
	MaxJobId                       string
	GresTypes                      string
	GroupUpdateForce               string
	GroupUpdateTime                string
	JobCheckpointDir               string
	JobCredentialPrivateKey        string
	JobCredentialPublicCertificate string
	JobFileAppend                  string
	JobRequeue                     string
	JobSubmitPlugins               string
	KillOnBadExit                  string
	Licenses                       string
	MailProg                       string
	MaxJobCount                    string
	MaxStepCount                   string
	MaxTasksPerNode                string
	MpiDefault                     string
	MpiParams                      string
	PluginDir                      string
	PlugStackConfig                string
	PrivateData                    string
	ProctrackType                  string
	Prolog                         string
	PrologSlurmctld                string
	PropagatePrioProcess           string
	PropagateResourceLimits        string
	PropagateResourceLimitsExcept  string
	ReturnToService                string
	SallocDefaultCommand           string
	SlurmctldPidFile               string
	SlurmctldPort                  string
	SlurmdPidFile                  string
	SlurmdPort                     string
	SlurmdSpoolDir                 string
	SlurmUser                      string
	SlurmdUser                     string
	SrunEpilog                     string
	SrunProlog                     string
	StateSaveLocation              string
	SwitchType                     string
	TaskEpilog                     string
	TaskPlugin                     string
	TaskPluginParam                string
	TaskProlog                     string
	TopologyPlugin                 string
	TmpFS                          string
	TrackWCKey                     string
	TreeWidth                      string
	UnkillableStepProgram          string
	UsePAM                         string
}

type Timers struct {
	BatchStartTimeout     string
	CompleteWait          string
	EpilogMsgTime         string
	GetEnvTimeout         string
	HealthCheckInterval   string
	HealthCheckProgram    string
	InactiveLimit         string
	KillWait              string
	MessageTimeout        string
	ResvOverRun           string
	MinJobAge             string
	OverTimeLimit         string
	SlurmctldTimeout      string
	SlurmdTimeout         string
	UnkillableStepTimeout string
	VSizeFactor           string
	Waittime              string
}

type Scheduling struct {
	DefMemPerCPU         string
	FastSchedule         string
	MaxMemPerCPU         string
	SchedulerRootFilter  string
	SchedulerTimeSlice   string
	SchedulerType        string
	SchedulerPort        string
	SelectType           string
	SelectTypeParameters string
}

type JobPriority struct {
	PriorityType             string
	PriorityDecayHalfLife    string
	PriorityCalcPeriod       string
	PriorityFavorSmall       string
	PriorityMaxAge           string
	PriorityUsageResetPeriod string
	PriorityWeightAge        string
	PriorityWeightFairshare  string
	PriorityWeightJobSize    string
	PriorityWeightPartition  string
	PriorityWeightQOS        string
}

type LoggingAccounting struct {
	AccountingStorageEnforce    string
	AccountingStorageHost       string
	AccountingStorageBackupHost string
	AccountingStorageLoc        string
	AccountingStoragePass       string
	AccountingStoragePort       string
	AccountingStorageType       string
	AccountingStorageUser       string
	AccountingStoreJobComment   string
	AcctGatherNodeFreq          string
	AcctGatherInfinibandType    string
	AcctGatherFilesystemType    string
	AcctGatherProfileType       string
	ClusterName                 string
	ChosLoc                     string
	DebugFlags                  string
	JobCompHost                 string
	JobCompLoc                  string
	JobCompPass                 string
	JobCompPort                 string
	JobCompType                 string
	JobCompUser                 string
	JobAcctGatherFrequency      string
	JobAcctGatherType           string
	SlurmctldDebug              string
	SlurmctldLogFile            string
	SlurmdDebug                 string
	SlurmdLogFile               string
	SlurmSchedLogFile           string
	SlurmSchedLogLevel          string
}

type PowerSave struct {
	SuspendProgram  string
	ResumeProgram   string
	SuspendTimeout  string
	ResumeTimeout   string
	ResumeRate      string
	SuspendExcNodes string
	SuspendExcParts string
	SuspendRate     string
	SuspendTime     string
}

type Node struct {
	_name           string
	NodeName        string
	NodeHostname    string
	NodeAddr        string
	Boards          string
	CoreSpecCount   string
	CoresPerSocket  string
	CPUs            string
	CPUSpecList     string
	Feature         string
	Gres            string
	MemSpecLimit    string
	Port            string
	Procs           string
	RealMemory      string
	Reason          string
	Sockets         string
	SocketsPerBoard string
	State           string
	ThreadsPerCore  string
	TmpDisk         string
	Weight          string
	DownNodes       string
}

type Partition struct {
	_name                string
	PartitionName        string
	AllocNodes           string
	AllowAccounts        string
	AllowGroups          string
	AllowQos             string
	Alternate            string
	Default              string
	DefMemPerCPU         string
	DefMemPerNode        string
	DenyAccounts         string
	DenyQos              string
	DefaultTime          string
	DisableRootJobs      string
	GraceTime            string
	Hidden               string
	LLN                  string
	MaxCPUsPerNode       string
	MaxMemPerCPU         string
	MaxMemPerNode        string
	MaxNodes             string
	MaxTime              string
	MinNodes             string
	Nodes                string
	PreemptMode          string
	Priority             string
	ReqResv              string
	RootOnly             string
	SelectTypeParameters string
	Shared               string
	State                string
}

type Frontend struct {
	AllowGroups  string
	AllowUsers   string
	DenyGroups   string
	DenyUsers    string
	FrontendName string
	FrontendAddr string
	Port         string
	Reason       string
	State        string
}

type Config struct {
	Controller        Controller
	General           General
	Timers            Timers
	Scheduling        Scheduling
	JobPriority       JobPriority
	LoggingAccounting LoggingAccounting
	PowerSave         PowerSave
	Node              []Node
	Partition         []Partition
	Frontend          []Frontend
}

func main() {
	var out io.Writer
	if len(os.Args) > 1 {
		fi, err := os.Create(os.Args[1])
		if err != nil {
			log.Println(err)
		}
		out = fi
	} else {
		out = os.Stdout
	}
	conf := Config{}
	conf.FromEvn()
	conf.Write(out)
}

func (c *Config) Write(w io.Writer) {

	w.Write([]byte("#slurm.conf\n\n"))
	w.Write([]byte("#\n# Controll Machines\n#\n"))
	writeStructToConfig(c.Controller, w)
	w.Write([]byte("#\n# General\n#\n"))
	writeStructToConfig(c.General, w)
	w.Write([]byte("#\n# Timers\n#\n"))
	writeStructToConfig(c.Timers, w)
	w.Write([]byte("#\n# Scheduling\n#\n"))
	writeStructToConfig(c.Scheduling, w)
	w.Write([]byte("#\n# JobPriority\n#\n"))
	writeStructToConfig(c.JobPriority, w)
	w.Write([]byte("#\n# Logging and Accounting\n#\n"))
	writeStructToConfig(c.LoggingAccounting, w)
	w.Write([]byte("#\n# Power save\n#\n"))
	writeStructToConfig(c.PowerSave, w)
	w.Write([]byte("#\n# Nodes\n#\n"))
	for _, node := range c.Node {
		writeStructToConfig(node, w)
	}
	w.Write([]byte("#\n# Partitions\n#\n"))
	for _, partition := range c.Partition {
		writeStructToConfig(partition, w)
	}

}

func writeStructToConfig(st interface{}, w io.Writer) {

	dlim := []byte{'='}
	endl := []byte{'\n'}

	val := reflect.ValueOf(st)
	typ := val.Type()

	for i := 0; i < val.NumField(); i++ {
		configField := typ.Field(i)
		configValue := val.Field(i)
		if configField.Name == "_name" {
			endl = []byte{' '}
		}
		s := configValue.String()
		if s == "" {
			continue
		}

		w.Write([]byte(configField.Name))
		w.Write(dlim)
		w.Write([]byte(s))
		w.Write(endl)
	}
	w.Write([]byte{'\n'})
}

func (c *Config) FromEvn() {
	c.findNodes()
	c.findPartitions()

	val := reflect.ValueOf(c).Elem()
	for i := 0; i < val.NumField(); i++ {
		confVal := val.Field(i)
		switch confVal.Kind() {
		case reflect.Slice:
			for i := 0; i < confVal.Len(); i++ {
				fillStructFromEnv(confVal.Index(i))
			}
		case reflect.Struct:
			fillStructFromEnv(confVal)
		}
	}

}

func (c *Config) findNodes() {
	envNodes := os.Getenv("SLRM_NODE_NAMES")
	nodes := strings.Split(envNodes, ",")
	for _, name := range nodes {
		node := Node{_name: name}
		c.Node = append(c.Node, node)
	}
}

func (c *Config) findPartitions() {
	envPartitions := os.Getenv("SLRM_PARTITION_NAMES")
	partitions := strings.Split(envPartitions, ",")
	for _, name := range partitions {
		partition := Partition{_name: name}
		c.Partition = append(c.Partition, partition)
	}

}

func fillStructFromEnv(val reflect.Value) {
	prefix := "SLURM_"
	typ := val.Type()
	for i := 0; i < val.NumField(); i++ {
		for i := 0; i < val.NumField(); i++ {
			configField := typ.Field(i)
			configVal := val.Field(i)
			if configField.Name == "_name" {
				prefix += strings.ToUpper(configVal.String()) + "_"
				continue
			}
			envName := toEnvName(prefix, configField.Name)
			configVal.SetString(os.Getenv(envName))
		}
	}
}

func toEnvName(prefix string, name string) string {
	envName := []rune(prefix)
	for _, r := range name {
		if unicode.IsUpper(r) {
			envName = append(envName, '_', r)
		} else {
			envName = append(envName, unicode.ToUpper(r))
		}
	}
	return string(envName)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"v2ray.com/core"
	"v2ray.com/core/common/log"

	_ "v2ray.com/core/main/distro/all"
)

var (
	configFile string
	version    = flag.Bool("version", false, "Show current version of V2Ray.")
	test       = flag.Bool("test", false, "Test config file only, without launching V2Ray server.")
	format     = flag.String("format", "json", "Format of input file.")
)

func init() {
	defaultConfigFile := ""
	workingDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err == nil {
		defaultConfigFile = filepath.Join(workingDir, "config.json")
	}
	flag.StringVar(&configFile, "config", defaultConfigFile, "Config file for this Point server.")
}

func GetConfigFormat() core.ConfigFormat {
	switch strings.ToLower(*format) {
	case "json":
		return core.ConfigFormat_JSON
	case "pb", "protobuf":
		return core.ConfigFormat_Protobuf
	default:
		return core.ConfigFormat_JSON
	}
}

func startV2Ray() *core.Point {
	if len(configFile) == 0 {
		log.Error("Config file is not set.")
		return nil
	}
	var configInput io.Reader
	if configFile == "stdin:" {
		configInput = os.Stdin
	} else {
		fixedFile := os.ExpandEnv(configFile)
		file, err := os.Open(fixedFile)
		if err != nil {
			log.Error("Config file not readable: ", err)
			return nil
		}
		defer file.Close()
		configInput = file
	}
	config, err := core.LoadConfig(GetConfigFormat(), configInput)
	if err != nil {
		log.Error("Failed to read config file (", configFile, "): ", configFile, err)
		return nil
	}

	vPoint, err := core.NewPoint(config)
	if err != nil {
		log.Error("Failed to create Point server: ", err)
		return nil
	}

	if *test {
		fmt.Println("Configuration OK.")
		return nil
	}

	err = vPoint.Start()
	if err != nil {
		log.Error("Error starting Point server: ", err)
		return nil
	}

	return vPoint
}

func main() {
	flag.Parse()

	core.PrintVersion()

	if *version {
		return
	}

	if point := startV2Ray(); point != nil {
		osSignals := make(chan os.Signal, 1)
		signal.Notify(osSignals, os.Interrupt, os.Kill, syscall.SIGTERM)

		<-osSignals
		point.Close()
	}
	log.Close()
}
<commit_msg>fix test break<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"v2ray.com/core"
	"v2ray.com/core/app/log"

	_ "v2ray.com/core/main/distro/all"
)

var (
	configFile string
	version    = flag.Bool("version", false, "Show current version of V2Ray.")
	test       = flag.Bool("test", false, "Test config file only, without launching V2Ray server.")
	format     = flag.String("format", "json", "Format of input file.")
)

func init() {
	defaultConfigFile := ""
	workingDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err == nil {
		defaultConfigFile = filepath.Join(workingDir, "config.json")
	}
	flag.StringVar(&configFile, "config", defaultConfigFile, "Config file for this Point server.")
}

func GetConfigFormat() core.ConfigFormat {
	switch strings.ToLower(*format) {
	case "json":
		return core.ConfigFormat_JSON
	case "pb", "protobuf":
		return core.ConfigFormat_Protobuf
	default:
		return core.ConfigFormat_JSON
	}
}

func startV2Ray() *core.Point {
	if len(configFile) == 0 {
		log.Error("Config file is not set.")
		return nil
	}
	var configInput io.Reader
	if configFile == "stdin:" {
		configInput = os.Stdin
	} else {
		fixedFile := os.ExpandEnv(configFile)
		file, err := os.Open(fixedFile)
		if err != nil {
			log.Error("Config file not readable: ", err)
			return nil
		}
		defer file.Close()
		configInput = file
	}
	config, err := core.LoadConfig(GetConfigFormat(), configInput)
	if err != nil {
		log.Error("Failed to read config file (", configFile, "): ", configFile, err)
		return nil
	}

	vPoint, err := core.NewPoint(config)
	if err != nil {
		log.Error("Failed to create Point server: ", err)
		return nil
	}

	if *test {
		fmt.Println("Configuration OK.")
		return nil
	}

	err = vPoint.Start()
	if err != nil {
		log.Error("Error starting Point server: ", err)
		return nil
	}

	return vPoint
}

func main() {
	flag.Parse()

	core.PrintVersion()

	if *version {
		return
	}

	if point := startV2Ray(); point != nil {
		osSignals := make(chan os.Signal, 1)
		signal.Notify(osSignals, os.Interrupt, os.Kill, syscall.SIGTERM)

		<-osSignals
		point.Close()
	}
}
<|endoftext|>


<commit_before>package rest

import (
	"fmt"
	"sync"

	"github.com/velocity-ci/velocity/backend/pkg/domain/build"
	"github.com/velocity-ci/velocity/backend/pkg/domain/githistory"
	"github.com/velocity-ci/velocity/backend/pkg/domain/knownhost"
	"github.com/velocity-ci/velocity/backend/pkg/domain/project"
	"github.com/velocity-ci/velocity/backend/pkg/domain/user"

	"github.com/Sirupsen/logrus"
	"github.com/velocity-ci/velocity/backend/pkg/domain"
)

type broker struct {
	clients map[string]*Client
	lock    sync.RWMutex

	branchManager *githistory.BranchManager
	stepManager   *build.StepManager
	streamManager *build.StreamManager
}

func NewBroker(
	branchManager *githistory.BranchManager,
	stepManager *build.StepManager,
	streamManager *build.StreamManager,
) *broker {
	return &broker{
		clients:       map[string]*Client{},
		lock:          sync.RWMutex{},
		branchManager: branchManager,
		stepManager:   stepManager,
		streamManager: streamManager,
	}
}

func (m *broker) save(c *Client) {
	m.lock.Lock()
	m.clients[c.ID] = c
	m.lock.Unlock()
}

func (m *broker) remove(c *Client) {
	m.lock.Lock()
	delete(m.clients, c.ID)
	m.lock.Unlock()
}

func (m *broker) monitor(c *Client) {
	c.alive = true
	for {
		message := &PhoenixMessage{}
		err := c.ws.ReadJSON(message)
		if err != nil {
			c.alive = false
			logrus.Error(err)
			logrus.Infof("Closing Client WebSocket: %s", c.ID)
			c.ws.Close()
			m.remove(c)
			return
		}
		c.HandleMessage(message)
		m.save(c)
	}
}

func (m *broker) EmitAll(message *domain.Emit) {
	clientCount := 0
	mess := m.handleEmit(message)
	for _, c := range m.clients {
		if !c.alive {
			m.remove(c)
			break
		}
		for _, s := range c.subscriptions {
			if s == mess.Topic {
				err := c.WriteJSON(mess)
				clientCount++
				if err != nil {
					logrus.Println(err)
				}
			}
		}
	}
}

func (m *broker) handleEmit(em *domain.Emit) *PhoenixMessage {
	var payload interface{}
	var topic string

	switch v := em.Payload.(type) {
	case *user.User:
		break
	case *knownhost.KnownHost:
		topic = "knownhosts"
		payload = newKnownHostResponse(v)
		break
	case *project.Project:
		if em.Event == project.EventUpdate {
			topic = fmt.Sprintf("project:%s", v.Slug)
		} else {
			topic = "projects"
		}
		payload = newProjectResponse(v)
		break
	case *githistory.Branch:
		topic = fmt.Sprintf("project:%s", v.Project.Slug)
		payload = newBranchResponse(v)
		break
	case *githistory.Commit:
		topic = fmt.Sprintf("project:%s", v.Project.Slug)
		bs, _ := m.branchManager.GetAllForCommit(v, domain.NewPagingQuery())
		payload = newCommitResponse(v, bs)
		break
	// case *task.Task:
	// 	topic = fmt.Sprintf("project:%s", v.Commit.Project.Slug)
	// 	payload = newTaskResponse(v)
	// 	break
	case *build.Build:
		topic = fmt.Sprintf("project:%s", v.Task.Commit.Project.Slug)
		steps := m.stepManager.GetStepsForBuild(v)
		payload = newBuildResponse(v, stepsToStepResponse(steps, m.streamManager), m.branchManager)
		break
	case *build.Step:
		topic = fmt.Sprintf("project:%s", v.Build.Task.Commit.Project.Slug)
		steps := m.stepManager.GetStepsForBuild(v.Build)
		payload = newBuildResponse(v.Build, stepsToStepResponse(steps, m.streamManager), m.branchManager)
	case *build.StreamLine:
		topic = fmt.Sprintf("stream:%s", v.StreamID)
		payload = newStreamLineResponse(v)
		break
	default:
		logrus.Errorf("could not resolve websocket payload %+v", v)
	}

	// determine event
	var wsEvent string
	if val, ok := wsEventMapping[em.Event]; ok {
		wsEvent = val
	} else {
		logrus.Errorf("could not resolve event in websocket: %s", em.Event)
	}

	return &PhoenixMessage{
		Topic:   topic,
		Event:   wsEvent,
		Payload: payload,
	}
}

var wsEventMapping = map[string]string{
	project.EventCreate: "project:new",
	project.EventUpdate: "project:update",
	project.EventDelete: "project:delete",

	githistory.EventCommitCreate: "commit:new",
	githistory.EventCommitUpdate: "commit:update",

	githistory.EventBranchCreate: "branch:new",
	githistory.EventBranchUpdate: "branch:update",

	knownhost.EventCreate: "knownhost:new",
	knownhost.EventDelete: "knownhost:delete",

	build.EventBuildCreate:      "build:new",
	build.EventBuildUpdate:      "build:update",
	build.EventStepUpdate:       "step:update",
	build.EventStreamLineCreate: "streamLine:new",

	// "": "builder:new",
	// "": "builder:update",
	// "": "builder:delete",
}
<commit_msg>[backend] Using build:update event when step has been updated<commit_after>package rest

import (
	"fmt"
	"sync"

	"github.com/velocity-ci/velocity/backend/pkg/domain/build"
	"github.com/velocity-ci/velocity/backend/pkg/domain/githistory"
	"github.com/velocity-ci/velocity/backend/pkg/domain/knownhost"
	"github.com/velocity-ci/velocity/backend/pkg/domain/project"
	"github.com/velocity-ci/velocity/backend/pkg/domain/user"

	"github.com/Sirupsen/logrus"
	"github.com/velocity-ci/velocity/backend/pkg/domain"
)

type broker struct {
	clients map[string]*Client
	lock    sync.RWMutex

	branchManager *githistory.BranchManager
	stepManager   *build.StepManager
	streamManager *build.StreamManager
}

func NewBroker(
	branchManager *githistory.BranchManager,
	stepManager *build.StepManager,
	streamManager *build.StreamManager,
) *broker {
	return &broker{
		clients:       map[string]*Client{},
		lock:          sync.RWMutex{},
		branchManager: branchManager,
		stepManager:   stepManager,
		streamManager: streamManager,
	}
}

func (m *broker) save(c *Client) {
	m.lock.Lock()
	m.clients[c.ID] = c
	m.lock.Unlock()
}

func (m *broker) remove(c *Client) {
	m.lock.Lock()
	delete(m.clients, c.ID)
	m.lock.Unlock()
}

func (m *broker) monitor(c *Client) {
	c.alive = true
	for {
		message := &PhoenixMessage{}
		err := c.ws.ReadJSON(message)
		if err != nil {
			c.alive = false
			logrus.Error(err)
			logrus.Infof("Closing Client WebSocket: %s", c.ID)
			c.ws.Close()
			m.remove(c)
			return
		}
		c.HandleMessage(message)
		m.save(c)
	}
}

func (m *broker) EmitAll(message *domain.Emit) {
	clientCount := 0
	mess := m.handleEmit(message)
	for _, c := range m.clients {
		if !c.alive {
			m.remove(c)
			break
		}
		for _, s := range c.subscriptions {
			if s == mess.Topic {
				err := c.WriteJSON(mess)
				clientCount++
				if err != nil {
					logrus.Println(err)
				}
			}
		}
	}
}

func (m *broker) handleEmit(em *domain.Emit) *PhoenixMessage {
	var payload interface{}
	var topic string

	switch v := em.Payload.(type) {
	case *user.User:
		break
	case *knownhost.KnownHost:
		topic = "knownhosts"
		payload = newKnownHostResponse(v)
		break
	case *project.Project:
		if em.Event == project.EventUpdate {
			topic = fmt.Sprintf("project:%s", v.Slug)
		} else {
			topic = "projects"
		}
		payload = newProjectResponse(v)
		break
	case *githistory.Branch:
		topic = fmt.Sprintf("project:%s", v.Project.Slug)
		payload = newBranchResponse(v)
		break
	case *githistory.Commit:
		topic = fmt.Sprintf("project:%s", v.Project.Slug)
		bs, _ := m.branchManager.GetAllForCommit(v, domain.NewPagingQuery())
		payload = newCommitResponse(v, bs)
		break
	// case *task.Task:
	// 	topic = fmt.Sprintf("project:%s", v.Commit.Project.Slug)
	// 	payload = newTaskResponse(v)
	// 	break
	case *build.Build:
		topic = fmt.Sprintf("project:%s", v.Task.Commit.Project.Slug)
		steps := m.stepManager.GetStepsForBuild(v)
		payload = newBuildResponse(v, stepsToStepResponse(steps, m.streamManager), m.branchManager)
		break
	case *build.Step:
		topic = fmt.Sprintf("project:%s", v.Build.Task.Commit.Project.Slug)
		steps := m.stepManager.GetStepsForBuild(v.Build)
		payload = newBuildResponse(v.Build, stepsToStepResponse(steps, m.streamManager), m.branchManager)
	case *build.StreamLine:
		topic = fmt.Sprintf("stream:%s", v.StreamID)
		payload = newStreamLineResponse(v)
		break
	default:
		logrus.Errorf("could not resolve websocket payload %+v", v)
	}

	// determine event
	var wsEvent string
	if val, ok := wsEventMapping[em.Event]; ok {
		wsEvent = val
	} else {
		logrus.Errorf("could not resolve event in websocket: %s", em.Event)
	}

	return &PhoenixMessage{
		Topic:   topic,
		Event:   wsEvent,
		Payload: payload,
	}
}

var wsEventMapping = map[string]string{
	project.EventCreate: "project:new",
	project.EventUpdate: "project:update",
	project.EventDelete: "project:delete",

	githistory.EventCommitCreate: "commit:new",
	githistory.EventCommitUpdate: "commit:update",

	githistory.EventBranchCreate: "branch:new",
	githistory.EventBranchUpdate: "branch:update",

	knownhost.EventCreate: "knownhost:new",
	knownhost.EventDelete: "knownhost:delete",

	build.EventBuildCreate:      "build:new",
	build.EventBuildUpdate:      "build:update",
	build.EventStepUpdate:       "build:update",
	build.EventStreamLineCreate: "streamLine:new",

	// "": "builder:new",
	// "": "builder:update",
	// "": "builder:delete",
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/drone/drone-plugin-go/plugin"
)

// commits is a list of commits of different types (push, pull request, tag)
// to help us verify that this clone plugin can handle multiple commit types.
var commits = []struct {
	path   string
	clone  string
	event  string
	branch string
	commit string
	ref    string
	file   string
	data   string
	tags   []string
}{
	// first commit
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPush,
		branch: "master",
		commit: "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e",
		ref:    "refs/heads/master",
		file:   "README",
		data:   "Hello World!",
		tags:   nil,
	},
	// head commit
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPush,
		branch: "master",
		commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d",
		ref:    "refs/heads/master",
		file:   "README",
		data:   "Hello World!\n",
		tags:   nil,
	},
	// pull request commit
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPull,
		branch: "master",
		commit: "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e",
		ref:    "refs/pull/208/merge",
		file:   "README",
		data:   "Goodbye World!\n",
		tags:   nil,
	},
	// branch
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPush,
		branch: "test",
		commit: "b3cbd5bbd7e81436d2eee04537ea2b4c0cad4cdf",
		ref:    "refs/heads/test",
		file:   "CONTRIBUTING.md",
		data:   "## Contributing\n",
		tags:   nil,
	},
	// tags
	{
		path:   "github/mime-types",
		clone:  "https://github.com/github/mime-types.git",
		event:  plugin.EventTag,
		branch: "master",
		commit: "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e",
		ref:    "refs/tags/v1.17",
		file:   ".gitignore",
		data:   "*.swp\n*~\n.rake_tasks~\nhtml\ndoc\npkg\npublish\ncoverage\n",
		tags: []string{
			"v1.16",
			"v1.17",
			"v1.17.1",
			"v1.17.2",
			"v1.18",
			"v1.19",
			"v1.20",
			"v1.20.1",
			"v1.21",
			"v1.22",
			"v1.23",
		},
	},
}

// TestClone tests the ability to clone a specific commit into
// a fresh, empty directory every time.
func TestClone(t *testing.T) {

	for _, c := range commits {
		dir := setup()

		tags := false
		if c.tags != nil {
			tags = true
		}

		r := &plugin.Repo{Clone: c.clone}
		b := &plugin.Build{Commit: c.commit, Branch: c.branch, Ref: c.ref, Event: c.event}
		w := &plugin.Workspace{Path: dir}
		v := &Params{Tags: tags}
		if err := clone(r, b, w, v); err != nil {
			t.Errorf("Expected successful clone. Got error. %s.", err)
		}

		data := readFile(dir, c.file)
		if data != c.data {
			t.Errorf("Expected %s to contain [%s]. Got [%s].", c.file, c.data, data)
		}

		if c.tags != nil {
			tags, err := getTags(dir)
			if err != nil {
				t.Error(err)
			}
			for _, tag := range c.tags {
				if !tags[tag] {
					t.Errorf("Expected tag [%s] to exist.", tag)
				}
			}
		}

		teardown(dir)
	}
}

// TestCloneNonEmpty tests the ability to clone a specific commit into
// a non-empty directory. This is useful if the git workspace is cached
// and re-stored for every build.
func TestCloneNonEmpty(t *testing.T) {
	dir := setup()
	defer teardown(dir)

	for _, c := range commits {

		r := &plugin.Repo{Clone: c.clone}
		b := &plugin.Build{Commit: c.commit, Branch: c.branch, Ref: c.ref, Event: c.event}
		w := &plugin.Workspace{Path: filepath.Join(dir, c.path)}
		v := &Params{}
		if err := clone(r, b, w, v); err != nil {
			t.Errorf("Expected successful clone. Got error. %s.", err)
			break
		}

		data := readFile(w.Path, c.file)
		if data != c.data {
			t.Errorf("Expected %s to contain [%s]. Got [%s].", c.file, c.data, data)
			break
		}
	}
}

// TestClone tests if the arguments to `git fetch` are constructed properly.
func TestFetch(t *testing.T) {
	testdata := []struct {
		build *plugin.Build
		tags  bool
		depth int
		exp   []string
	}{
		{
			&plugin.Build{Ref: "refs/heads/master"},
			false,
			50,
			[]string{
				"git",
				"fetch",
				"--no-tags",
				"--depth=50",
				"origin",
				"+refs/heads/master:",
			},
		},
		{
			&plugin.Build{Ref: "refs/heads/master"},
			true,
			100,
			[]string{
				"git",
				"fetch",
				"--tags",
				"--depth=100",
				"origin",
				"+refs/heads/master:",
			},
		},
	}
	for _, td := range testdata {
		c := fetch(td.build, td.tags, td.depth)
		if len(c.Args) != len(td.exp) {
			t.Errorf("Expected: %s, got %s", td.exp, c.Args)
		}
		for i := range c.Args {
			if c.Args[i] != td.exp[i] {
				t.Errorf("Expected: %s, got %s", td.exp, c.Args)
			}
		}
	}
}

// helper function that will setup a temporary workspace.
// to which we can clone the repositroy
func setup() string {
	dir, _ := ioutil.TempDir("/tmp", "drone_git_test_")
	os.Mkdir(dir, 0777)
	return dir
}

// helper function to delete the temporary workspace.
func teardown(dir string) {
	os.RemoveAll(dir)
}

// helper function to read a file in the temporary worskapce.
func readFile(dir, file string) string {
	filename := filepath.Join(dir, file)
	data, _ := ioutil.ReadFile(filename)
	return string(data)
}

// getTags returns all of the tags in a git repository as a map.
func getTags(dir string) (map[string]bool, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	if err := os.Chdir(dir); err != nil {
		return nil, err
	}
	cmd := exec.Command("git", "tag")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	tags := make(map[string]bool)
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		tags[scanner.Text()] = true
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}
	if err := os.Chdir(cwd); err != nil {
		return nil, err
	}
	return tags, nil
}
<commit_msg>Use `cmd.Dir=dir` instead of `os.Chdir()`<commit_after>package main

import (
	"bufio"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/drone/drone-plugin-go/plugin"
)

// commits is a list of commits of different types (push, pull request, tag)
// to help us verify that this clone plugin can handle multiple commit types.
var commits = []struct {
	path   string
	clone  string
	event  string
	branch string
	commit string
	ref    string
	file   string
	data   string
	tags   []string
}{
	// first commit
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPush,
		branch: "master",
		commit: "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e",
		ref:    "refs/heads/master",
		file:   "README",
		data:   "Hello World!",
		tags:   nil,
	},
	// head commit
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPush,
		branch: "master",
		commit: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d",
		ref:    "refs/heads/master",
		file:   "README",
		data:   "Hello World!\n",
		tags:   nil,
	},
	// pull request commit
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPull,
		branch: "master",
		commit: "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e",
		ref:    "refs/pull/208/merge",
		file:   "README",
		data:   "Goodbye World!\n",
		tags:   nil,
	},
	// branch
	{
		path:   "octocat/Hello-World",
		clone:  "https://github.com/octocat/Hello-World.git",
		event:  plugin.EventPush,
		branch: "test",
		commit: "b3cbd5bbd7e81436d2eee04537ea2b4c0cad4cdf",
		ref:    "refs/heads/test",
		file:   "CONTRIBUTING.md",
		data:   "## Contributing\n",
		tags:   nil,
	},
	// tags
	{
		path:   "github/mime-types",
		clone:  "https://github.com/github/mime-types.git",
		event:  plugin.EventTag,
		branch: "master",
		commit: "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e",
		ref:    "refs/tags/v1.17",
		file:   ".gitignore",
		data:   "*.swp\n*~\n.rake_tasks~\nhtml\ndoc\npkg\npublish\ncoverage\n",
		tags: []string{
			"v1.16",
			"v1.17",
			"v1.17.1",
			"v1.17.2",
			"v1.18",
			"v1.19",
			"v1.20",
			"v1.20.1",
			"v1.21",
			"v1.22",
			"v1.23",
		},
	},
}

// TestClone tests the ability to clone a specific commit into
// a fresh, empty directory every time.
func TestClone(t *testing.T) {

	for _, c := range commits {
		dir := setup()

		tags := false
		if c.tags != nil {
			tags = true
		}

		r := &plugin.Repo{Clone: c.clone}
		b := &plugin.Build{Commit: c.commit, Branch: c.branch, Ref: c.ref, Event: c.event}
		w := &plugin.Workspace{Path: dir}
		v := &Params{Tags: tags}
		if err := clone(r, b, w, v); err != nil {
			t.Errorf("Expected successful clone. Got error. %s.", err)
		}

		data := readFile(dir, c.file)
		if data != c.data {
			t.Errorf("Expected %s to contain [%s]. Got [%s].", c.file, c.data, data)
		}

		if c.tags != nil {
			tags, err := getTags(dir)
			if err != nil {
				t.Error(err)
			}
			for _, tag := range c.tags {
				if !tags[tag] {
					t.Errorf("Expected tag [%s] to exist.", tag)
				}
			}
		}

		teardown(dir)
	}
}

// TestCloneNonEmpty tests the ability to clone a specific commit into
// a non-empty directory. This is useful if the git workspace is cached
// and re-stored for every build.
func TestCloneNonEmpty(t *testing.T) {
	dir := setup()
	defer teardown(dir)

	for _, c := range commits {

		r := &plugin.Repo{Clone: c.clone}
		b := &plugin.Build{Commit: c.commit, Branch: c.branch, Ref: c.ref, Event: c.event}
		w := &plugin.Workspace{Path: filepath.Join(dir, c.path)}
		v := &Params{}
		if err := clone(r, b, w, v); err != nil {
			t.Errorf("Expected successful clone. Got error. %s.", err)
			break
		}

		data := readFile(w.Path, c.file)
		if data != c.data {
			t.Errorf("Expected %s to contain [%s]. Got [%s].", c.file, c.data, data)
			break
		}
	}
}

// TestClone tests if the arguments to `git fetch` are constructed properly.
func TestFetch(t *testing.T) {
	testdata := []struct {
		build *plugin.Build
		tags  bool
		depth int
		exp   []string
	}{
		{
			&plugin.Build{Ref: "refs/heads/master"},
			false,
			50,
			[]string{
				"git",
				"fetch",
				"--no-tags",
				"--depth=50",
				"origin",
				"+refs/heads/master:",
			},
		},
		{
			&plugin.Build{Ref: "refs/heads/master"},
			true,
			100,
			[]string{
				"git",
				"fetch",
				"--tags",
				"--depth=100",
				"origin",
				"+refs/heads/master:",
			},
		},
	}
	for _, td := range testdata {
		c := fetch(td.build, td.tags, td.depth)
		if len(c.Args) != len(td.exp) {
			t.Errorf("Expected: %s, got %s", td.exp, c.Args)
		}
		for i := range c.Args {
			if c.Args[i] != td.exp[i] {
				t.Errorf("Expected: %s, got %s", td.exp, c.Args)
			}
		}
	}
}

// helper function that will setup a temporary workspace.
// to which we can clone the repositroy
func setup() string {
	dir, _ := ioutil.TempDir("/tmp", "drone_git_test_")
	os.Mkdir(dir, 0777)
	return dir
}

// helper function to delete the temporary workspace.
func teardown(dir string) {
	os.RemoveAll(dir)
}

// helper function to read a file in the temporary worskapce.
func readFile(dir, file string) string {
	filename := filepath.Join(dir, file)
	data, _ := ioutil.ReadFile(filename)
	return string(data)
}

// getTags returns all of the tags in a git repository as a map.
func getTags(dir string) (map[string]bool, error) {
	cmd := exec.Command("git", "tag")
	cmd.Dir = dir
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	tags := make(map[string]bool)
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		tags[scanner.Text()] = true
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	if err := cmd.Wait(); err != nil {
		return nil, err
	}
	return tags, nil
}
<|endoftext|>


<commit_before>package discovery

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/hashicorp/consul/api"
)

// ServiceDefinition is how a job communicates with the Consul service
// discovery backend.
type ServiceDefinition struct {
	ID                             string
	Name                           string
	Port                           int
	TTL                            int
	Tags                           []string
	IPAddress                      string
	EnableTagOverride              bool
	DeregisterCriticalServiceAfter string
	Consul                         Backend

	wasRegistered bool
}

// Deregister removes the service from Consul.
func (service *ServiceDefinition) Deregister() {
	service.MarkForMaintenance()
}

// MarkForMaintenance removes the service from Consul.
func (service *ServiceDefinition) MarkForMaintenance() {
	log.Debugf("deregistering: %s", service.ID)
	if err := service.Consul.ServiceDeregister(service.ID); err != nil {
		log.Infof("deregistering failed: %s", err)
	}
}

// SendHeartbeat writes a TTL check status=ok to the consul store.
// If consul has never seen this service, we register the service and
// its TTL check.
func (service *ServiceDefinition) SendHeartbeat() error {
	if !service.wasRegistered {
		if err := service.registerService(); err != nil {
			log.Warnf("service registration failed: %s", err)
			return err
		}
		service.wasRegistered = true
		return nil
	}
	if err := service.Consul.PassTTL(service.ID, "ok"); err != nil {
		log.Infof("service not registered: %v", err)
		if err = service.registerService(); err != nil {
			log.Warnf("service registration failed: %s", err)
			return err
		}
		log.Infof("Service registered: %v", service.Name)
	}
	return nil
}

// registers the service along with a check set to the passing state
func (service *ServiceDefinition) registerService() error {
	return service.Consul.ServiceRegister(
		&api.AgentServiceRegistration{
			ID:                service.ID,
			Name:              service.Name,
			Tags:              service.Tags,
			Port:              service.Port,
			Address:           service.IPAddress,
			EnableTagOverride: service.EnableTagOverride,
			Check: &api.AgentServiceCheck{
				TTL:    fmt.Sprintf("%ds", service.TTL),
				Status: api.HealthPassing,
				Notes:  fmt.Sprintf("TTL for %s set by containerpilot", service.Name),
				DeregisterCriticalServiceAfter: service.DeregisterCriticalServiceAfter,
			},
		},
	)
}
<commit_msg>fix failure to update TTL because checkID != service.ID (#405)<commit_after>package discovery

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/hashicorp/consul/api"
)

// ServiceDefinition is how a job communicates with the Consul service
// discovery backend.
type ServiceDefinition struct {
	ID                             string
	Name                           string
	Port                           int
	TTL                            int
	Tags                           []string
	IPAddress                      string
	EnableTagOverride              bool
	DeregisterCriticalServiceAfter string
	Consul                         Backend

	wasRegistered bool
}

// Deregister removes the service from Consul.
func (service *ServiceDefinition) Deregister() {
	service.MarkForMaintenance()
}

// MarkForMaintenance removes the service from Consul.
func (service *ServiceDefinition) MarkForMaintenance() {
	log.Debugf("deregistering: %s", service.ID)
	if err := service.Consul.ServiceDeregister(service.ID); err != nil {
		log.Infof("deregistering failed: %s", err)
	}
}

// SendHeartbeat writes a TTL check status=ok to the consul store.
// If consul has never seen this service, we register the service and
// its TTL check.
func (service *ServiceDefinition) SendHeartbeat() error {
	if !service.wasRegistered {
		if err := service.registerService(); err != nil {
			log.Warnf("service registration failed: %s", err)
			return err
		}
		service.wasRegistered = true
		return nil
	}
	checkID := fmt.Sprintf("service:%s", service.ID)
	if err := service.Consul.PassTTL(checkID, "ok"); err != nil {
		log.Infof("service not registered: %v", err)
		if err = service.registerService(); err != nil {
			log.Warnf("service registration failed: %s", err)
			return err
		}
		log.Infof("Service registered: %v", service.Name)
	}
	return nil
}

// registers the service along with a check set to the passing state
func (service *ServiceDefinition) registerService() error {
	return service.Consul.ServiceRegister(
		&api.AgentServiceRegistration{
			ID:                service.ID,
			Name:              service.Name,
			Tags:              service.Tags,
			Port:              service.Port,
			Address:           service.IPAddress,
			EnableTagOverride: service.EnableTagOverride,
			Check: &api.AgentServiceCheck{
				TTL:    fmt.Sprintf("%ds", service.TTL),
				Status: api.HealthPassing,
				Notes:  fmt.Sprintf("TTL for %s set by containerpilot", service.Name),
				DeregisterCriticalServiceAfter: service.DeregisterCriticalServiceAfter,
			},
		},
	)
}
<|endoftext|>


<commit_before>package main
<commit_msg>Add coverage around empty messages.<commit_after>package main

import (
	"github.com/Shopify/sarama"
	"os"
	"testing"
)

func TestInitConfig(t *testing.T) {
	os.Args = nil
	os.Args = append(os.Args, "gomkafka")
	os.Args = append(os.Args, "client_id")
	os.Args = append(os.Args, "localhost:8080")
	os.Args = append(os.Args, "foo_topic")
	_, err := initConfig()
	if err != nil {
		t.Errorf("error: failed to initialize gomkafka config.")
	}
}

func TestWork(t *testing.T) {
	mb := sarama.NewMockBroker(t, 1)
	mb.Returns(new(sarama.MetadataResponse))

	os.Args = nil
	os.Args = append(os.Args, "gomkafka")
	os.Args = append(os.Args, "client_id")
	os.Args = append(os.Args, mb.Addr())
	os.Args = append(os.Args, "foo_topic")

	err := work()
	// Test empty input case.
	if err == nil {
		t.Errorf("error. %+v", err)
	}
}

func assertEquals(t *testing.T, expected interface{}, actual interface{}) {
	if expected != actual {
		t.Errorf("error. got: %d, expected: %d.", actual, expected)
	}
}
<|endoftext|>


<commit_before>package asics_parser

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"
)

var (
	WirteDir = ""
)

var EuropeEcAsics []string
var NoEcAsics []string
var AllData = map[string]ParseData{}

type ParseData struct {
	Locale string
	Header []Link
	Footer []Link
}

type Link struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
	//the value of imgsize represent the type of the image, small | medium | big
	SingleRow bool
	//when the value of SingleRow is false, there will have two link in a row.
	Items []interface{}
}

type ItemLink struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
}

func ParseFile(dir string) map[string]ParseData {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	for _, file := range files {
		if strings.Contains(file.Name(), ".json") {
			openfile, err := os.Open(dir + file.Name())
			if err != nil {
				log.Fatal(err)
			}
			data, err := ioutil.ReadAll(openfile)
			if err != nil {
				log.Fatal(err)
			}
			var parsedate ParseData
			json.Unmarshal(data, &parsedate)
			AllData[parsedate.Locale] = parsedate

		}

	}
	return AllData

}

func ParseECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	parsedata := ParseData{
		Locale: strings.Split(target_url.Path, "/")[2],
	}

	//parse the header
	doc.Find("#main-menu").Children().Not(".mobile").Not("div").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Url:       paser_url(target_url, s.Children().First().AttrOr("href", "#")),
			Name:      s.Children().First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Find("ul").Each(func(i int, s *goquery.Selection) {
			if s.Children().Size() == 2 {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     1,
					SingleRow: false,
				}

				secondlink := Link{
					Title:     s.Find(".empty-nav-item").Last().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find(".empty-nav-item").Last().Text(),
					Index:     2,
					SingleRow: false,
				}

				is_first := true
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {
					// fmt.Println(i)
					// fmt.Println(s.Children().First().Html())
					if s.Children().First().Is("span") {
						if s.Children().First().Text() != "" {
							is_first = false
						}

					} else {
						itemlink := ItemLink{
							Title: s.Children().First().AttrOr("title", ""),
							Name:  s.Children().First().Text(),
							Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
							Index: i + 1,
						}
						if s.Children().First().Children().First().Is("img") {
							itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
						}

						if is_first {
							firstlink.Items = append(firstlink.Items, itemlink)
						} else {
							secondlink.Items = append(secondlink.Items, itemlink)

						}
					}
				})
				link.Items = append(link.Items, firstlink)
				link.Items = append(link.Items, secondlink)

			} else {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     i + 1,
					SingleRow: true,
				}
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {

					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Name:  s.Children().First().Text(),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Index: i + 1,
					}
					if s.Children().First().Children().First().Is("img") {
						itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
					}

					firstlink.Items = append(firstlink.Items, itemlink)

				})
				link.Items = append(link.Items, firstlink)
			}
		})
		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find("footer").Children().Filter(".tiger-clearfix-toggle").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().Filter("h4").First().AttrOr("title", ""),
			Url:       "#",
			Name:      s.Children().Filter("h4").First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Name:  s.Children().First().Text(),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	fmt.Printf("current url is %s", url_string)
	file, _ := os.OpenFile(WirteDir+strings.Split(target_url.Path, "/")[2]+".json", os.O_WRONLY|os.O_CREATE, 0666)
	file.Write(jsondata)

}

func ParseNoECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	locale := strings.Split(target_url.Host, ".")[0] + "-" + strings.Split(target_url.Host, ".")[2]
	parsedata := ParseData{
		Locale: locale,
	}

	//parse the header
	doc.Find("header #asicsAreas").Children().Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Children().First().Text(),
			SingleRow: true,
			Url:       "#",
			Index:     i + 1,
		}
		if i != 3 {
			doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsListing").Each(func(ii int, ss *goquery.Selection) {
				secend_link := Link{
					Title:     doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("title", ""),
					Name:      strings.TrimSpace(doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().Text()),
					Url:       doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("href", "#"),
					Img:       paser_url(target_url, doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().First().Children().First().AttrOr("src", "#")),
					ImgSize:   "small",
					Index:     i + 1,
					SingleRow: true,
				}
				ss.Children().Filter("ul").Children().Each(func(iii int, s *goquery.Selection) {
					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Name:  s.Children().First().Text(),
						Index: iii + 1,
					}

					secend_link.Items = append(secend_link.Items, itemlink)
				})

				link.Items = append(link.Items, secend_link)

			})

		}

		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find(".footer #tertiary section").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Text(),
			Url:       "#",
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Name:  s.Children().First().Text(),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	file, _ := os.OpenFile(WirteDir+locale+".json", os.O_WRONLY|os.O_CREATE, 0666)
	fmt.Printf("current url is %s", url_string)
	file.Write(jsondata)
}

func paser_url(target_url *url.URL, url string) (return_url string) {

	if matched, err := regexp.Match(`http`, []byte(url)); matched && err == nil {
		return_url = url
	} else {
		return_url = target_url.Scheme + "://" + target_url.Host + url
	}
	return
}
<commit_msg>modify the fmt<commit_after>package asics_parser

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"
)

var (
	WirteDir = ""
)

var EuropeEcAsics []string
var NoEcAsics []string
var AllData = map[string]ParseData{}

type ParseData struct {
	Locale string
	Header []Link
	Footer []Link
}

type Link struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
	//the value of imgsize represent the type of the image, small | medium | big
	SingleRow bool
	//when the value of SingleRow is false, there will have two link in a row.
	Items []interface{}
}

type ItemLink struct {
	Title   string
	Url     string
	Name    string
	Index   int
	Img     string `json:"Img,omitempty"`
	ImgSize string `json:"ImgSize,omitempty"`
}

func ParseFile(dir string) map[string]ParseData {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	for _, file := range files {
		if strings.Contains(file.Name(), ".json") {
			openfile, err := os.Open(dir + file.Name())
			if err != nil {
				log.Fatal(err)
			}
			data, err := ioutil.ReadAll(openfile)
			if err != nil {
				log.Fatal(err)
			}
			var parsedate ParseData
			json.Unmarshal(data, &parsedate)
			AllData[parsedate.Locale] = parsedate

		}

	}
	return AllData

}

func ParseECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	parsedata := ParseData{
		Locale: strings.Split(target_url.Path, "/")[2],
	}

	//parse the header
	doc.Find("#main-menu").Children().Not(".mobile").Not("div").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Url:       paser_url(target_url, s.Children().First().AttrOr("href", "#")),
			Name:      s.Children().First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Find("ul").Each(func(i int, s *goquery.Selection) {
			if s.Children().Size() == 2 {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     1,
					SingleRow: false,
				}

				secondlink := Link{
					Title:     s.Find(".empty-nav-item").Last().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find(".empty-nav-item").Last().Text(),
					Index:     2,
					SingleRow: false,
				}

				is_first := true
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {
					// fmt.Println(i)
					// fmt.Println(s.Children().First().Html())
					if s.Children().First().Is("span") {
						if s.Children().First().Text() != "" {
							is_first = false
						}

					} else {
						itemlink := ItemLink{
							Title: s.Children().First().AttrOr("title", ""),
							Name:  s.Children().First().Text(),
							Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
							Index: i + 1,
						}
						if s.Children().First().Children().First().Is("img") {
							itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
						}

						if is_first {
							firstlink.Items = append(firstlink.Items, itemlink)
						} else {
							secondlink.Items = append(secondlink.Items, itemlink)

						}
					}
				})
				link.Items = append(link.Items, firstlink)
				link.Items = append(link.Items, secondlink)

			} else {

				firstlink := Link{
					Title:     s.Find("h5").First().AttrOr("title", ""),
					Url:       "#",
					Name:      s.Find("h5").First().Text(),
					Index:     i + 1,
					SingleRow: true,
				}
				s.Find(".yCmsComponent").Each(func(i int, s *goquery.Selection) {

					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Name:  s.Children().First().Text(),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Index: i + 1,
					}
					if s.Children().First().Children().First().Is("img") {
						itemlink.Img = paser_url(target_url, s.Children().First().Children().First().AttrOr("src", "#"))
					}

					firstlink.Items = append(firstlink.Items, itemlink)

				})
				link.Items = append(link.Items, firstlink)
			}
		})
		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find("footer").Children().Filter(".tiger-clearfix-toggle").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().Filter("h4").First().AttrOr("title", ""),
			Url:       "#",
			Name:      s.Children().Filter("h4").First().Text(),
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Name:  s.Children().First().Text(),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	fmt.Printf("current url is %s \n", url_string)
	file, _ := os.OpenFile(WirteDir+strings.Split(target_url.Path, "/")[2]+".json", os.O_WRONLY|os.O_CREATE, 0666)
	file.Write(jsondata)

}

func ParseNoECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	locale := strings.Split(target_url.Host, ".")[0] + "-" + strings.Split(target_url.Host, ".")[2]
	parsedata := ParseData{
		Locale: locale,
	}

	//parse the header
	doc.Find("header #asicsAreas").Children().Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Children().First().Text(),
			SingleRow: true,
			Url:       "#",
			Index:     i + 1,
		}
		if i != 3 {
			doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsListing").Each(func(ii int, ss *goquery.Selection) {
				secend_link := Link{
					Title:     doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("title", ""),
					Name:      strings.TrimSpace(doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().Text()),
					Url:       doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().End().AttrOr("href", "#"),
					Img:       paser_url(target_url, doc.Find("header #asicsPanels").Children().Slice(i, i+1).Children().First().Children().Filter(".asicsFeatured").Children().First().Children().Slice(ii, ii+1).Children().First().Children().First().AttrOr("src", "#")),
					ImgSize:   "small",
					Index:     i + 1,
					SingleRow: true,
				}
				ss.Children().Filter("ul").Children().Each(func(iii int, s *goquery.Selection) {
					itemlink := ItemLink{
						Title: s.Children().First().AttrOr("title", ""),
						Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
						Name:  s.Children().First().Text(),
						Index: iii + 1,
					}

					secend_link.Items = append(secend_link.Items, itemlink)
				})

				link.Items = append(link.Items, secend_link)

			})

		}

		parsedata.Header = append(parsedata.Header, link)
	})

	//parse the footer
	doc.Find(".footer #tertiary section").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title:     s.Children().First().AttrOr("title", ""),
			Name:      s.Children().First().Text(),
			Url:       "#",
			Index:     i + 1,
			SingleRow: true,
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
				Title: s.Children().First().AttrOr("title", ""),
				Url:   paser_url(target_url, s.Children().First().AttrOr("href", "#")),
				Name:  s.Children().First().Text(),
				Index: i + 1,
			}
			link.Items = append(link.Items, itemlink)
		})
		parsedata.Footer = append(parsedata.Footer, link)
	})

	jsondata, _ := json.Marshal(parsedata)

	file, _ := os.OpenFile(WirteDir+locale+".json", os.O_WRONLY|os.O_CREATE, 0666)
	fmt.Printf("current url is %s \n", url_string)
	file.Write(jsondata)
}

func paser_url(target_url *url.URL, url string) (return_url string) {

	if matched, err := regexp.Match(`http`, []byte(url)); matched && err == nil {
		return_url = url
	} else {
		return_url = target_url.Scheme + "://" + target_url.Host + url
	}
	return
}
<|endoftext|>


<commit_before>package yext

import (
	"encoding/json"
	"net/http"
	"reflect"
	"strconv"
	"testing"
)

func parseAs(cftype string, rawval interface{}) (interface{}, error) {
	cfs := []*CustomField{
		&CustomField{
			Id:   String("123"),
			Type: cftype,
		},
	}

	cfsraw := map[string]interface{}{
		"123": rawval,
	}

	newcfs, err := ParseCustomFields(cfsraw, cfs)
	return newcfs["123"], err
}

type customFieldParseTest struct {
	TypeName     string
	Raw          interface{}
	ExpectedType reflect.Type
}

func runParseTest(t *testing.T, index int, c customFieldParseTest) {
	cf, err := parseAs(c.TypeName, c.Raw)
	if err != nil {
		t.Error(err)
		return
	}
	ty := reflect.TypeOf(cf).String()
	expect := c.ExpectedType.String()
	if ty != expect {
		t.Errorf("test #%d failed: expected type %s, got type %s", index, expect, ty)
	}
}

var (
	customPhoto = map[string]interface{}{
		"title":           "A great picture",
		"description":     "This is a picture of an awesome event",
		"clickthroughUrl": "https://yext.com/event",
		"url":             "https://mktgcdn.com/awesome.jpg",
	}
	hours = map[string]interface{}{
		"additionalHoursText": "This is an example of extra hours info",
		"hours":               "1:9:00:17:00,3:15:00:12:00,3:5:00:11:00,4:9:00:17:00,5:0:00:0:00,6:9:00:17:00,7:9:00:17:00",
		"holidayHours": []interface{}{
			map[string]interface{}{
				"date":  "2016-05-30",
				"hours": "",
			},
			map[string]interface{}{
				"date":  "2016-07-04",
				"hours": "0:00:0:00",
			},
			map[string]interface{}{
				"date":  "2016-09-05",
				"hours": "6:00:10:00",
			},
			map[string]interface{}{
				"date":  "2016-09-10",
				"hours": "9:00:17:00",
			},
		},
	}
	video = map[string]interface{}{
		"description": "An example caption for a video",
		"url":         "http://www.youtube.com/watch?v=M80FTIcEgZM",
	}
	dailyTimes = map[string]interface{}{
		"dailyTimes": "2:7:00,3:7:00,4:7:00,5:7:00,6:7:00,7:7:00,1:7:00",
	}
	parseTests = []customFieldParseTest{
		customFieldParseTest{"BOOLEAN", false, reflect.TypeOf(YesNo(false))},
		customFieldParseTest{"BOOLEAN", "false", reflect.TypeOf(YesNo(false))},
		customFieldParseTest{"NUMBER", "12345", reflect.TypeOf(Number(""))},
		customFieldParseTest{"TEXT", "foo", reflect.TypeOf(SingleLineText(""))},
		customFieldParseTest{"MULTILINE_TEXT", "foo", reflect.TypeOf(MultiLineText(""))},
		customFieldParseTest{"SINGLE_OPTION", "foo", reflect.TypeOf(GetSingleOptionPointer(SingleOption("")))},
		customFieldParseTest{"URL", "foo", reflect.TypeOf(Url(""))},
		customFieldParseTest{"DATE", "foo", reflect.TypeOf(Date(""))},
		customFieldParseTest{"TEXT_LIST", []string{"a", "b", "c"}, reflect.TypeOf(TextList([]string{}))},
		customFieldParseTest{"TEXT_LIST", []interface{}{"a", "b", "c"}, reflect.TypeOf(TextList([]string{}))},
		customFieldParseTest{"MULTI_OPTION", []string{"a", "b", "c"}, reflect.TypeOf(MultiOption([]string{}))},
		customFieldParseTest{"MULTI_OPTION", []interface{}{"a", "b", "c"}, reflect.TypeOf(MultiOption([]string{}))},
		customFieldParseTest{"PHOTO", customPhoto, reflect.TypeOf(Photo{})},
		customFieldParseTest{"GALLERY", []interface{}{customPhoto}, reflect.TypeOf(Gallery{})},
		customFieldParseTest{"VIDEO", video, reflect.TypeOf(Video{})},
		customFieldParseTest{"HOURS", hours, reflect.TypeOf(Hours{})},
		customFieldParseTest{"DAILY_TIMES", dailyTimes, reflect.TypeOf(DailyTimes{})},
		customFieldParseTest{"LOCATION_LIST", []string{"a", "b", "c"}, reflect.TypeOf(LocationList([]string{}))},
	}
)

func makeCustomFields(n int) []*CustomField {
	var cfs []*CustomField

	for i := 0; i < n; i++ {
		new := &CustomField{Id: String(strconv.Itoa(i))}
		cfs = append(cfs, new)
	}

	return cfs
}

func TestListAll(t *testing.T) {
	maxLimit := strconv.Itoa(CustomFieldListMaxLimit)

	type req struct {
		limit  string
		offset string
	}

	tests := []struct {
		count int
		reqs  []req
	}{
		{
			count: 0,
			reqs:  []req{{limit: maxLimit, offset: ""}},
		},
		{
			count: 1000,
			reqs:  []req{{limit: maxLimit, offset: ""}},
		},
		{
			count: 1001,
			reqs:  []req{{limit: maxLimit, offset: ""}, {limit: maxLimit, offset: "1000"}},
		},
		{
			count: 2000,
			reqs:  []req{{limit: maxLimit, offset: ""}, {limit: maxLimit, offset: "1000"}},
		},
	}

	for _, test := range tests {
		setup()
		reqs := 0
		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			reqs++
			if reqs > len(test.reqs) {
				t.Errorf("Too many requests sent to custom field list - got %d, expected %d", reqs, len(test.reqs))
			}

			expectedreq := test.reqs[reqs-1]

			if v := r.URL.Query().Get("limit"); v != expectedreq.limit {
				t.Errorf("Wanted limit %s, got %s", expectedreq.limit, v)
			}

			if v := r.URL.Query().Get("offset"); v != expectedreq.offset {
				t.Errorf("Wanted offset %s, got %s", expectedreq.offset, v)
			}

			cfs := []*CustomField{}
			remaining := test.count - ((reqs - 1) * CustomFieldListMaxLimit)
			if remaining > 0 {
				if remaining > CustomFieldListMaxLimit {
					remaining = CustomFieldListMaxLimit
				}
				cfs = makeCustomFields(remaining)
			}

			v := &mockResponse{Response: &CustomFieldResponse{Count: test.count, CustomFields: cfs}}
			data, _ := json.Marshal(v)
			w.Write(data)
		})

		client.CustomFieldService.ListAll()
		if reqs < len(test.reqs) {
			t.Errorf("Too few requests sent to custom field list - got %d, expected %d", reqs, len(test.reqs))
		}

		teardown()
	}
}

func TestListMismatchCount(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v := &mockResponse{Response: &CustomFieldResponse{Count: 25, CustomFields: makeCustomFields(24)}}
		data, _ := json.Marshal(v)
		w.Write(data)
	})

	rlr, err := client.CustomFieldService.ListAll()
	if rlr != nil {
		t.Error("Expected response to be nil when receiving mismatched count and number of custom fields")
	}
	if err == nil {
		t.Error("Expected error to be present when receiving mismatched count and number of custom fields")
	}
}

func TestMustCache(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v := &mockResponse{Response: &CustomFieldResponse{Count: 24, CustomFields: makeCustomFields(24)}}
		data, _ := json.Marshal(v)
		w.Write(data)
	})
	m := client.CustomFieldService.MustCacheCustomFields()
	n := makeCustomFields(24)
	for i := 0; i < 24; i++ {
		if m[i].Id != n[i].Id {
			t.Error("Must Cache Custom fields should return the same custom field slice as makeCustomFields")
		}
	}
}

func TestParsing(t *testing.T) {
	for i, testData := range parseTests {
		runParseTest(t, i, testData)
	}
}

func TestParseLeaveUnknownTypes(t *testing.T) {
	type peanut []string
	cf, err := parseAs("BLAH", peanut([]string{"a", "b", "c"}))
	if err != nil {
		t.Error(err)
	}

	if _, ok := cf.(peanut); !ok {
		t.Errorf("Expected type peanut, got type %T", cf)
	}
}
<commit_msg>changed to use getId so we have the number instead of the pointer<commit_after>package yext

import (
	"encoding/json"
	"net/http"
	"reflect"
	"strconv"
	"testing"
)

func parseAs(cftype string, rawval interface{}) (interface{}, error) {
	cfs := []*CustomField{
		&CustomField{
			Id:   String("123"),
			Type: cftype,
		},
	}

	cfsraw := map[string]interface{}{
		"123": rawval,
	}

	newcfs, err := ParseCustomFields(cfsraw, cfs)
	return newcfs["123"], err
}

type customFieldParseTest struct {
	TypeName     string
	Raw          interface{}
	ExpectedType reflect.Type
}

func runParseTest(t *testing.T, index int, c customFieldParseTest) {
	cf, err := parseAs(c.TypeName, c.Raw)
	if err != nil {
		t.Error(err)
		return
	}
	ty := reflect.TypeOf(cf).String()
	expect := c.ExpectedType.String()
	if ty != expect {
		t.Errorf("test #%d failed: expected type %s, got type %s", index, expect, ty)
	}
}

var (
	customPhoto = map[string]interface{}{
		"title":           "A great picture",
		"description":     "This is a picture of an awesome event",
		"clickthroughUrl": "https://yext.com/event",
		"url":             "https://mktgcdn.com/awesome.jpg",
	}
	hours = map[string]interface{}{
		"additionalHoursText": "This is an example of extra hours info",
		"hours":               "1:9:00:17:00,3:15:00:12:00,3:5:00:11:00,4:9:00:17:00,5:0:00:0:00,6:9:00:17:00,7:9:00:17:00",
		"holidayHours": []interface{}{
			map[string]interface{}{
				"date":  "2016-05-30",
				"hours": "",
			},
			map[string]interface{}{
				"date":  "2016-07-04",
				"hours": "0:00:0:00",
			},
			map[string]interface{}{
				"date":  "2016-09-05",
				"hours": "6:00:10:00",
			},
			map[string]interface{}{
				"date":  "2016-09-10",
				"hours": "9:00:17:00",
			},
		},
	}
	video = map[string]interface{}{
		"description": "An example caption for a video",
		"url":         "http://www.youtube.com/watch?v=M80FTIcEgZM",
	}
	dailyTimes = map[string]interface{}{
		"dailyTimes": "2:7:00,3:7:00,4:7:00,5:7:00,6:7:00,7:7:00,1:7:00",
	}
	parseTests = []customFieldParseTest{
		customFieldParseTest{"BOOLEAN", false, reflect.TypeOf(YesNo(false))},
		customFieldParseTest{"BOOLEAN", "false", reflect.TypeOf(YesNo(false))},
		customFieldParseTest{"NUMBER", "12345", reflect.TypeOf(Number(""))},
		customFieldParseTest{"TEXT", "foo", reflect.TypeOf(SingleLineText(""))},
		customFieldParseTest{"MULTILINE_TEXT", "foo", reflect.TypeOf(MultiLineText(""))},
		customFieldParseTest{"SINGLE_OPTION", "foo", reflect.TypeOf(GetSingleOptionPointer(SingleOption("")))},
		customFieldParseTest{"URL", "foo", reflect.TypeOf(Url(""))},
		customFieldParseTest{"DATE", "foo", reflect.TypeOf(Date(""))},
		customFieldParseTest{"TEXT_LIST", []string{"a", "b", "c"}, reflect.TypeOf(TextList([]string{}))},
		customFieldParseTest{"TEXT_LIST", []interface{}{"a", "b", "c"}, reflect.TypeOf(TextList([]string{}))},
		customFieldParseTest{"MULTI_OPTION", []string{"a", "b", "c"}, reflect.TypeOf(MultiOption([]string{}))},
		customFieldParseTest{"MULTI_OPTION", []interface{}{"a", "b", "c"}, reflect.TypeOf(MultiOption([]string{}))},
		customFieldParseTest{"PHOTO", customPhoto, reflect.TypeOf(Photo{})},
		customFieldParseTest{"GALLERY", []interface{}{customPhoto}, reflect.TypeOf(Gallery{})},
		customFieldParseTest{"VIDEO", video, reflect.TypeOf(Video{})},
		customFieldParseTest{"HOURS", hours, reflect.TypeOf(Hours{})},
		customFieldParseTest{"DAILY_TIMES", dailyTimes, reflect.TypeOf(DailyTimes{})},
		customFieldParseTest{"LOCATION_LIST", []string{"a", "b", "c"}, reflect.TypeOf(LocationList([]string{}))},
	}
)

func makeCustomFields(n int) []*CustomField {
	var cfs []*CustomField

	for i := 0; i < n; i++ {
		new := &CustomField{Id: String(strconv.Itoa(i))}
		cfs = append(cfs, new)
	}

	return cfs
}

func TestListAll(t *testing.T) {
	maxLimit := strconv.Itoa(CustomFieldListMaxLimit)

	type req struct {
		limit  string
		offset string
	}

	tests := []struct {
		count int
		reqs  []req
	}{
		{
			count: 0,
			reqs:  []req{{limit: maxLimit, offset: ""}},
		},
		{
			count: 1000,
			reqs:  []req{{limit: maxLimit, offset: ""}},
		},
		{
			count: 1001,
			reqs:  []req{{limit: maxLimit, offset: ""}, {limit: maxLimit, offset: "1000"}},
		},
		{
			count: 2000,
			reqs:  []req{{limit: maxLimit, offset: ""}, {limit: maxLimit, offset: "1000"}},
		},
	}

	for _, test := range tests {
		setup()
		reqs := 0
		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			reqs++
			if reqs > len(test.reqs) {
				t.Errorf("Too many requests sent to custom field list - got %d, expected %d", reqs, len(test.reqs))
			}

			expectedreq := test.reqs[reqs-1]

			if v := r.URL.Query().Get("limit"); v != expectedreq.limit {
				t.Errorf("Wanted limit %s, got %s", expectedreq.limit, v)
			}

			if v := r.URL.Query().Get("offset"); v != expectedreq.offset {
				t.Errorf("Wanted offset %s, got %s", expectedreq.offset, v)
			}

			cfs := []*CustomField{}
			remaining := test.count - ((reqs - 1) * CustomFieldListMaxLimit)
			if remaining > 0 {
				if remaining > CustomFieldListMaxLimit {
					remaining = CustomFieldListMaxLimit
				}
				cfs = makeCustomFields(remaining)
			}

			v := &mockResponse{Response: &CustomFieldResponse{Count: test.count, CustomFields: cfs}}
			data, _ := json.Marshal(v)
			w.Write(data)
		})

		client.CustomFieldService.ListAll()
		if reqs < len(test.reqs) {
			t.Errorf("Too few requests sent to custom field list - got %d, expected %d", reqs, len(test.reqs))
		}

		teardown()
	}
}

func TestListMismatchCount(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v := &mockResponse{Response: &CustomFieldResponse{Count: 25, CustomFields: makeCustomFields(24)}}
		data, _ := json.Marshal(v)
		w.Write(data)
	})

	rlr, err := client.CustomFieldService.ListAll()
	if rlr != nil {
		t.Error("Expected response to be nil when receiving mismatched count and number of custom fields")
	}
	if err == nil {
		t.Error("Expected error to be present when receiving mismatched count and number of custom fields")
	}
}

func TestMustCache(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v := &mockResponse{Response: &CustomFieldResponse{Count: 24, CustomFields: makeCustomFields(24)}}
		data, _ := json.Marshal(v)
		w.Write(data)
	})
	m := client.CustomFieldService.MustCacheCustomFields()
	n := makeCustomFields(24)
	for i := 0; i < 24; i++ {
		if m[i].GetId() != n[i].GetId() {
			t.Error("Must Cache Custom fields should return the same custom field slice as makeCustomFields")
		}
	}
}

func TestParsing(t *testing.T) {
	for i, testData := range parseTests {
		runParseTest(t, i, testData)
	}
}

func TestParseLeaveUnknownTypes(t *testing.T) {
	type peanut []string
	cf, err := parseAs("BLAH", peanut([]string{"a", "b", "c"}))
	if err != nil {
		t.Error(err)
	}

	if _, ok := cf.(peanut); !ok {
		t.Errorf("Expected type peanut, got type %T", cf)
	}
}
<|endoftext|>