<commit_before>package types

import (
	"time"
)

// UnsupportedProperties not yet supported by this implementation of the compose file
var UnsupportedProperties = []string{
	"build",
	"cap_add",
	"cap_drop",
	"cgroup_parent",
	"devices",
	"dns",
	"dns_search",
	"domainname",
	"external_links",
	"ipc",
	"links",
	"mac_address",
	"network_mode",
	"privileged",
	"read_only",
	"restart",
	"security_opt",
	"shm_size",
	"stop_signal",
	"tmpfs",
}

// DeprecatedProperties that were removed from the v3 format, but their
// use should not impact the behaviour of the application.
var DeprecatedProperties = map[string]string{
	"container_name": "Setting the container name is not supported.",
	"expose":         "Exposing ports is unnecessary - services on the same network can access each other's containers on any port.",
}

// ForbiddenProperties that are not supported in this implementation of the
// compose file.
var ForbiddenProperties = map[string]string{
	"extends":       "Support for `extends` is not implemented yet. Use `docker-compose config` to generate a configuration with all `extends` options resolved, and deploy from that.",
	"volume_driver": "Instead of setting the volume driver on the service, define a volume using the top-level `volumes` option and specify the driver there.",
	"volumes_from":  "To share a volume between services, define it using the top-level `volumes` option and reference it from each service that shares it using the service-level `volumes` option.",
	"cpu_quota":     "Set resource limits using deploy.resources",
	"cpu_shares":    "Set resource limits using deploy.resources",
	"cpuset":        "Set resource limits using deploy.resources",
	"mem_limit":     "Set resource limits using deploy.resources",
	"memswap_limit": "Set resource limits using deploy.resources",
}

// Dict is a mapping of strings to interface{}
type Dict map[string]interface{}

// ConfigFile is a filename and the contents of the file as a Dict
type ConfigFile struct {
	Filename string
	Config   Dict
}

// ConfigDetails are the details about a group of ConfigFiles
type ConfigDetails struct {
	WorkingDir  string
	ConfigFiles []ConfigFile
	Environment map[string]string
}

// Config is a full compose file configuration
type Config struct {
	Services []ServiceConfig
	Networks map[string]NetworkConfig
	Volumes  map[string]VolumeConfig
}

// ServiceConfig is the configuration of one service
type ServiceConfig struct {
	Name string

	CapAdd          []string `mapstructure:"cap_add"`
	CapDrop         []string `mapstructure:"cap_drop"`
	CgroupParent    string   `mapstructure:"cgroup_parent"`
	Command         []string `compose:"shell_command"`
	ContainerName   string   `mapstructure:"container_name"`
	DependsOn       []string `mapstructure:"depends_on"`
	Deploy          DeployConfig
	Devices         []string
	DNS             []string          `compose:"string_or_list"`
	DNSSearch       []string          `mapstructure:"dns_search" compose:"string_or_list"`
	DomainName      string            `mapstructure:"domainname"`
	Entrypoint      []string          `compose:"shell_command"`
	Environment     map[string]string `compose:"list_or_dict_equals"`
	Expose          []string          `compose:"list_of_strings_or_numbers"`
	ExternalLinks   []string          `mapstructure:"external_links"`
	ExtraHosts      map[string]string `mapstructure:"extra_hosts" compose:"list_or_dict_colon"`
	Hostname        string
	HealthCheck     *HealthCheckConfig
	Image           string
	Ipc             string
	Labels          map[string]string `compose:"list_or_dict_equals"`
	Links           []string
	Logging         *LoggingConfig
	MacAddress      string                           `mapstructure:"mac_address"`
	NetworkMode     string                           `mapstructure:"network_mode"`
	Networks        map[string]*ServiceNetworkConfig `compose:"list_or_struct_map"`
	Pid             string
	Ports           []string `compose:"list_of_strings_or_numbers"`
	Privileged      bool
	ReadOnly        bool `mapstructure:"read_only"`
	Restart         string
	SecurityOpt     []string       `mapstructure:"security_opt"`
	StdinOpen       bool           `mapstructure:"stdin_open"`
	StopGracePeriod *time.Duration `mapstructure:"stop_grace_period"`
	StopSignal      string         `mapstructure:"stop_signal"`
	Tmpfs           []string       `compose:"string_or_list"`
	Tty             bool           `mapstructure:"tty"`
	Ulimits         map[string]*UlimitsConfig
	User            string
	Volumes         []string
	WorkingDir      string `mapstructure:"working_dir"`
}

// LoggingConfig the logging configuration for a service
type LoggingConfig struct {
	Driver  string
	Options map[string]string
}

// DeployConfig the deployment configuration for a service
type DeployConfig struct {
	Mode          string
	Replicas      *uint64
	Labels        map[string]string `compose:"list_or_dict_equals"`
	UpdateConfig  *UpdateConfig     `mapstructure:"update_config"`
	Resources     Resources
	RestartPolicy *RestartPolicy `mapstructure:"restart_policy"`
	Placement     Placement
}

// HealthCheckConfig the healthcheck configuration for a service
type HealthCheckConfig struct {
	Test     []string `compose:"healthcheck"`
	Timeout  string
	Interval string
	Retries  *uint64
	Disable  bool
}

// UpdateConfig the service update configuration
type UpdateConfig struct {
	Parallelism     *uint64
	Delay           time.Duration
	FailureAction   string `mapstructure:"failure_action"`
	Monitor         time.Duration
	MaxFailureRatio float32 `mapstructure:"max_failure_ratio"`
}

// Resources the resource limits and reservations
type Resources struct {
	Limits       *Resource
	Reservations *Resource
}

// Resource is a resource to be limited or reserved
type Resource struct {
	// TODO: types to convert from units and ratios
	NanoCPUs    string    `mapstructure:"cpus"`
	MemoryBytes UnitBytes `mapstructure:"memory"`
}

// UnitBytes is the bytes type
type UnitBytes int64

// RestartPolicy the service restart policy
type RestartPolicy struct {
	Condition   string
	Delay       *time.Duration
	MaxAttempts *uint64 `mapstructure:"max_attempts"`
	Window      *time.Duration
}

// Placement constraints for the service
type Placement struct {
	Constraints []string
}

// ServiceNetworkConfig is the network configuration for a service
type ServiceNetworkConfig struct {
	Aliases     []string
	Ipv4Address string `mapstructure:"ipv4_address"`
	Ipv6Address string `mapstructure:"ipv6_address"`
}

// UlimitsConfig the ulimit configuration
type UlimitsConfig struct {
	Single int
	Soft   int
	Hard   int
}

// NetworkConfig for a network
type NetworkConfig struct {
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	Ipam       IPAMConfig
	External   External
	Labels     map[string]string `compose:"list_or_dict_equals"`
}

// IPAMConfig for a network
type IPAMConfig struct {
	Driver string
	Config []*IPAMPool
}

// IPAMPool for a network
type IPAMPool struct {
	Subnet string
}

// VolumeConfig for a volume
type VolumeConfig struct {
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	External   External
	Labels     map[string]string `compose:"list_or_dict_equals"`
}

// External identifies a Volume or Network as a reference to a resource that is
// not managed, and should already exist.
type External struct {
	Name     string
	External bool
}
<commit_msg>Improve the error message for extends in stack deploy.<commit_after>package types

import (
	"time"
)

// UnsupportedProperties not yet supported by this implementation of the compose file
var UnsupportedProperties = []string{
	"build",
	"cap_add",
	"cap_drop",
	"cgroup_parent",
	"devices",
	"dns",
	"dns_search",
	"domainname",
	"external_links",
	"ipc",
	"links",
	"mac_address",
	"network_mode",
	"privileged",
	"read_only",
	"restart",
	"security_opt",
	"shm_size",
	"stop_signal",
	"tmpfs",
}

// DeprecatedProperties that were removed from the v3 format, but their
// use should not impact the behaviour of the application.
var DeprecatedProperties = map[string]string{
	"container_name": "Setting the container name is not supported.",
	"expose":         "Exposing ports is unnecessary - services on the same network can access each other's containers on any port.",
}

// ForbiddenProperties that are not supported in this implementation of the
// compose file.
var ForbiddenProperties = map[string]string{
	"extends":       "Support for `extends` is not implemented yet.",
	"volume_driver": "Instead of setting the volume driver on the service, define a volume using the top-level `volumes` option and specify the driver there.",
	"volumes_from":  "To share a volume between services, define it using the top-level `volumes` option and reference it from each service that shares it using the service-level `volumes` option.",
	"cpu_quota":     "Set resource limits using deploy.resources",
	"cpu_shares":    "Set resource limits using deploy.resources",
	"cpuset":        "Set resource limits using deploy.resources",
	"mem_limit":     "Set resource limits using deploy.resources",
	"memswap_limit": "Set resource limits using deploy.resources",
}

// Dict is a mapping of strings to interface{}
type Dict map[string]interface{}

// ConfigFile is a filename and the contents of the file as a Dict
type ConfigFile struct {
	Filename string
	Config   Dict
}

// ConfigDetails are the details about a group of ConfigFiles
type ConfigDetails struct {
	WorkingDir  string
	ConfigFiles []ConfigFile
	Environment map[string]string
}

// Config is a full compose file configuration
type Config struct {
	Services []ServiceConfig
	Networks map[string]NetworkConfig
	Volumes  map[string]VolumeConfig
}

// ServiceConfig is the configuration of one service
type ServiceConfig struct {
	Name string

	CapAdd          []string `mapstructure:"cap_add"`
	CapDrop         []string `mapstructure:"cap_drop"`
	CgroupParent    string   `mapstructure:"cgroup_parent"`
	Command         []string `compose:"shell_command"`
	ContainerName   string   `mapstructure:"container_name"`
	DependsOn       []string `mapstructure:"depends_on"`
	Deploy          DeployConfig
	Devices         []string
	DNS             []string          `compose:"string_or_list"`
	DNSSearch       []string          `mapstructure:"dns_search" compose:"string_or_list"`
	DomainName      string            `mapstructure:"domainname"`
	Entrypoint      []string          `compose:"shell_command"`
	Environment     map[string]string `compose:"list_or_dict_equals"`
	Expose          []string          `compose:"list_of_strings_or_numbers"`
	ExternalLinks   []string          `mapstructure:"external_links"`
	ExtraHosts      map[string]string `mapstructure:"extra_hosts" compose:"list_or_dict_colon"`
	Hostname        string
	HealthCheck     *HealthCheckConfig
	Image           string
	Ipc             string
	Labels          map[string]string `compose:"list_or_dict_equals"`
	Links           []string
	Logging         *LoggingConfig
	MacAddress      string                           `mapstructure:"mac_address"`
	NetworkMode     string                           `mapstructure:"network_mode"`
	Networks        map[string]*ServiceNetworkConfig `compose:"list_or_struct_map"`
	Pid             string
	Ports           []string `compose:"list_of_strings_or_numbers"`
	Privileged      bool
	ReadOnly        bool `mapstructure:"read_only"`
	Restart         string
	SecurityOpt     []string       `mapstructure:"security_opt"`
	StdinOpen       bool           `mapstructure:"stdin_open"`
	StopGracePeriod *time.Duration `mapstructure:"stop_grace_period"`
	StopSignal      string         `mapstructure:"stop_signal"`
	Tmpfs           []string       `compose:"string_or_list"`
	Tty             bool           `mapstructure:"tty"`
	Ulimits         map[string]*UlimitsConfig
	User            string
	Volumes         []string
	WorkingDir      string `mapstructure:"working_dir"`
}

// LoggingConfig the logging configuration for a service
type LoggingConfig struct {
	Driver  string
	Options map[string]string
}

// DeployConfig the deployment configuration for a service
type DeployConfig struct {
	Mode          string
	Replicas      *uint64
	Labels        map[string]string `compose:"list_or_dict_equals"`
	UpdateConfig  *UpdateConfig     `mapstructure:"update_config"`
	Resources     Resources
	RestartPolicy *RestartPolicy `mapstructure:"restart_policy"`
	Placement     Placement
}

// HealthCheckConfig the healthcheck configuration for a service
type HealthCheckConfig struct {
	Test     []string `compose:"healthcheck"`
	Timeout  string
	Interval string
	Retries  *uint64
	Disable  bool
}

// UpdateConfig the service update configuration
type UpdateConfig struct {
	Parallelism     *uint64
	Delay           time.Duration
	FailureAction   string `mapstructure:"failure_action"`
	Monitor         time.Duration
	MaxFailureRatio float32 `mapstructure:"max_failure_ratio"`
}

// Resources the resource limits and reservations
type Resources struct {
	Limits       *Resource
	Reservations *Resource
}

// Resource is a resource to be limited or reserved
type Resource struct {
	// TODO: types to convert from units and ratios
	NanoCPUs    string    `mapstructure:"cpus"`
	MemoryBytes UnitBytes `mapstructure:"memory"`
}

// UnitBytes is the bytes type
type UnitBytes int64

// RestartPolicy the service restart policy
type RestartPolicy struct {
	Condition   string
	Delay       *time.Duration
	MaxAttempts *uint64 `mapstructure:"max_attempts"`
	Window      *time.Duration
}

// Placement constraints for the service
type Placement struct {
	Constraints []string
}

// ServiceNetworkConfig is the network configuration for a service
type ServiceNetworkConfig struct {
	Aliases     []string
	Ipv4Address string `mapstructure:"ipv4_address"`
	Ipv6Address string `mapstructure:"ipv6_address"`
}

// UlimitsConfig the ulimit configuration
type UlimitsConfig struct {
	Single int
	Soft   int
	Hard   int
}

// NetworkConfig for a network
type NetworkConfig struct {
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	Ipam       IPAMConfig
	External   External
	Labels     map[string]string `compose:"list_or_dict_equals"`
}

// IPAMConfig for a network
type IPAMConfig struct {
	Driver string
	Config []*IPAMPool
}

// IPAMPool for a network
type IPAMPool struct {
	Subnet string
}

// VolumeConfig for a volume
type VolumeConfig struct {
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	External   External
	Labels     map[string]string `compose:"list_or_dict_equals"`
}

// External identifies a Volume or Network as a reference to a resource that is
// not managed, and should already exist.
type External struct {
	Name     string
	External bool
}
<|endoftext|>


<commit_before>package types

import (
	"time"
)

// UnsupportedProperties not yet supported by this implementation of the compose file
var UnsupportedProperties = []string{
	"build",
	"cap_add",
	"cap_drop",
	"cgroup_parent",
	"devices",
	"domainname",
	"external_links",
	"ipc",
	"links",
	"mac_address",
	"network_mode",
	"privileged",
	"restart",
	"security_opt",
	"shm_size",
	"sysctls",
	"tmpfs",
	"userns_mode",
}

// DeprecatedProperties that were removed from the v3 format, but their
// use should not impact the behaviour of the application.
var DeprecatedProperties = map[string]string{
	"container_name": "Setting the container name is not supported.",
	"expose":         "Exposing ports is unnecessary - services on the same network can access each other's containers on any port.",
}

// ForbiddenProperties that are not supported in this implementation of the
// compose file.
var ForbiddenProperties = map[string]string{
	"extends":       "Support for `extends` is not implemented yet.",
	"volume_driver": "Instead of setting the volume driver on the service, define a volume using the top-level `volumes` option and specify the driver there.",
	"volumes_from":  "To share a volume between services, define it using the top-level `volumes` option and reference it from each service that shares it using the service-level `volumes` option.",
	"cpu_quota":     "Set resource limits using deploy.resources",
	"cpu_shares":    "Set resource limits using deploy.resources",
	"cpuset":        "Set resource limits using deploy.resources",
	"mem_limit":     "Set resource limits using deploy.resources",
	"memswap_limit": "Set resource limits using deploy.resources",
}

// ConfigFile is a filename and the contents of the file as a Dict
type ConfigFile struct {
	Filename string
	Config   map[string]interface{}
}

// ConfigDetails are the details about a group of ConfigFiles
type ConfigDetails struct {
	WorkingDir  string
	ConfigFiles []ConfigFile
	Environment map[string]string
}

// LookupEnv provides a lookup function for environment variables
func (cd ConfigDetails) LookupEnv(key string) (string, bool) {
	v, ok := cd.Environment[key]
	return v, ok
}

// Config is a full compose file configuration
type Config struct {
	Services []ServiceConfig
	Networks map[string]NetworkConfig
	Volumes  map[string]VolumeConfig
	Secrets  map[string]SecretConfig
	Configs  map[string]ConfigObjConfig
}

// ServiceConfig is the configuration of one service
type ServiceConfig struct {
	Name string

	CapAdd          []string `mapstructure:"cap_add"`
	CapDrop         []string `mapstructure:"cap_drop"`
	CgroupParent    string   `mapstructure:"cgroup_parent"`
	Command         ShellCommand
	Configs         []ServiceConfigObjConfig
	ContainerName   string               `mapstructure:"container_name"`
	CredentialSpec  CredentialSpecConfig `mapstructure:"credential_spec"`
	DependsOn       []string             `mapstructure:"depends_on"`
	Deploy          DeployConfig
	Devices         []string
	DNS             StringList
	DNSSearch       StringList `mapstructure:"dns_search"`
	DomainName      string     `mapstructure:"domainname"`
	Entrypoint      ShellCommand
	Environment     MappingWithEquals
	EnvFile         StringList `mapstructure:"env_file"`
	Expose          StringOrNumberList
	ExternalLinks   []string         `mapstructure:"external_links"`
	ExtraHosts      MappingWithColon `mapstructure:"extra_hosts"`
	Hostname        string
	HealthCheck     *HealthCheckConfig
	Image           string
	Ipc             string
	Labels          Labels
	Links           []string
	Logging         *LoggingConfig
	MacAddress      string `mapstructure:"mac_address"`
	NetworkMode     string `mapstructure:"network_mode"`
	Networks        map[string]*ServiceNetworkConfig
	Pid             string
	Ports           []ServicePortConfig
	Privileged      bool
	ReadOnly        bool `mapstructure:"read_only"`
	Restart         string
	Secrets         []ServiceSecretConfig
	SecurityOpt     []string       `mapstructure:"security_opt"`
	StdinOpen       bool           `mapstructure:"stdin_open"`
	StopGracePeriod *time.Duration `mapstructure:"stop_grace_period"`
	StopSignal      string         `mapstructure:"stop_signal"`
	Tmpfs           StringList
	Tty             bool `mapstructure:"tty"`
	Ulimits         map[string]*UlimitsConfig
	User            string
	Volumes         []ServiceVolumeConfig
	WorkingDir      string `mapstructure:"working_dir"`
}

// ShellCommand is a string or list of string args
type ShellCommand []string

// StringList is a type for fields that can be a string or list of strings
type StringList []string

// StringOrNumberList is a type for fields that can be a list of strings or
// numbers
type StringOrNumberList []string

// MappingWithEquals is a mapping type that can be converted from a list of
// key[=value] strings.
// For the key with an empty value (`key=`), the mapped value is set to a pointer to `""`.
// For the key without value (`key`), the mapped value is set to nil.
type MappingWithEquals map[string]*string

// Labels is a mapping type for labels
type Labels map[string]string

// MappingWithColon is a mapping type that can be converted from a list of
// 'key: value' strings
type MappingWithColon map[string]string

// LoggingConfig the logging configuration for a service
type LoggingConfig struct {
	Driver  string
	Options map[string]string
}

// DeployConfig the deployment configuration for a service
type DeployConfig struct {
	Mode          string
	Replicas      *uint64
	Labels        Labels
	UpdateConfig  *UpdateConfig `mapstructure:"update_config"`
	Resources     Resources
	RestartPolicy *RestartPolicy `mapstructure:"restart_policy"`
	Placement     Placement
	EndpointMode  string `mapstructure:"endpoint_mode"`
}

// HealthCheckConfig the healthcheck configuration for a service
type HealthCheckConfig struct {
	Test        HealthCheckTest
	Timeout     string
	Interval    string
	Retries     *uint64
	StartPeriod string
	Disable     bool
}

// HealthCheckTest is the command run to test the health of a service
type HealthCheckTest []string

// UpdateConfig the service update configuration
type UpdateConfig struct {
	Parallelism     *uint64
	Delay           time.Duration
	FailureAction   string `mapstructure:"failure_action"`
	Monitor         time.Duration
	MaxFailureRatio float32 `mapstructure:"max_failure_ratio"`
	Order           string
}

// Resources the resource limits and reservations
type Resources struct {
	Limits       *Resource
	Reservations *Resource
}

// Resource is a resource to be limited or reserved
type Resource struct {
	// TODO: types to convert from units and ratios
	NanoCPUs    string    `mapstructure:"cpus"`
	MemoryBytes UnitBytes `mapstructure:"memory"`
}

// UnitBytes is the bytes type
type UnitBytes int64

// RestartPolicy the service restart policy
type RestartPolicy struct {
	Condition   string
	Delay       *time.Duration
	MaxAttempts *uint64 `mapstructure:"max_attempts"`
	Window      *time.Duration
}

// Placement constraints for the service
type Placement struct {
	Constraints []string
	Preferences []PlacementPreferences
}

// PlacementPreferences is the preferences for a service placement
type PlacementPreferences struct {
	Spread string
}

// ServiceNetworkConfig is the network configuration for a service
type ServiceNetworkConfig struct {
	Aliases     []string
	Ipv4Address string `mapstructure:"ipv4_address"`
	Ipv6Address string `mapstructure:"ipv6_address"`
}

// ServicePortConfig is the port configuration for a service
type ServicePortConfig struct {
	Mode      string
	Target    uint32
	Published uint32
	Protocol  string
}

// ServiceVolumeConfig are references to a volume used by a service
type ServiceVolumeConfig struct {
	Type        string
	Source      string
	Target      string
	ReadOnly    bool `mapstructure:"read_only"`
	Consistency string
	Bind        *ServiceVolumeBind
	Volume      *ServiceVolumeVolume
}

// ServiceVolumeBind are options for a service volume of type bind
type ServiceVolumeBind struct {
	Propagation string
}

// ServiceVolumeVolume are options for a service volume of type volume
type ServiceVolumeVolume struct {
	NoCopy bool `mapstructure:"nocopy"`
}

type fileReferenceConfig struct {
	Source string
	Target string
	UID    string
	GID    string
	Mode   *uint32
}

// ServiceConfigObjConfig is the config obj configuration for a service
type ServiceConfigObjConfig fileReferenceConfig

// ServiceSecretConfig is the secret configuration for a service
type ServiceSecretConfig fileReferenceConfig

// UlimitsConfig the ulimit configuration
type UlimitsConfig struct {
	Single int
	Soft   int
	Hard   int
}

// NetworkConfig for a network
type NetworkConfig struct {
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	Ipam       IPAMConfig
	External   External
	Internal   bool
	Attachable bool
	Labels     Labels
}

// IPAMConfig for a network
type IPAMConfig struct {
	Driver string
	Config []*IPAMPool
}

// IPAMPool for a network
type IPAMPool struct {
	Subnet string
}

// VolumeConfig for a volume
type VolumeConfig struct {
	Name       string
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	External   External
	Labels     Labels
}

// External identifies a Volume or Network as a reference to a resource that is
// not managed, and should already exist.
// External.name is deprecated and replaced by Volume.name
type External struct {
	Name     string
	External bool
}

// CredentialSpecConfig for credential spec on Windows
type CredentialSpecConfig struct {
	File     string
	Registry string
}

type fileObjectConfig struct {
	File     string
	External External
	Labels   Labels
}

// SecretConfig for a secret
type SecretConfig fileObjectConfig

// ConfigObjConfig is the config for the swarm "Config" object
type ConfigObjConfig fileObjectConfig
<commit_msg>Add ulimits to unsupported compose fields.<commit_after>package types

import (
	"time"
)

// UnsupportedProperties not yet supported by this implementation of the compose file
var UnsupportedProperties = []string{
	"build",
	"cap_add",
	"cap_drop",
	"cgroup_parent",
	"devices",
	"domainname",
	"external_links",
	"ipc",
	"links",
	"mac_address",
	"network_mode",
	"privileged",
	"restart",
	"security_opt",
	"shm_size",
	"sysctls",
	"tmpfs",
	"ulimits",
	"userns_mode",
}

// DeprecatedProperties that were removed from the v3 format, but their
// use should not impact the behaviour of the application.
var DeprecatedProperties = map[string]string{
	"container_name": "Setting the container name is not supported.",
	"expose":         "Exposing ports is unnecessary - services on the same network can access each other's containers on any port.",
}

// ForbiddenProperties that are not supported in this implementation of the
// compose file.
var ForbiddenProperties = map[string]string{
	"extends":       "Support for `extends` is not implemented yet.",
	"volume_driver": "Instead of setting the volume driver on the service, define a volume using the top-level `volumes` option and specify the driver there.",
	"volumes_from":  "To share a volume between services, define it using the top-level `volumes` option and reference it from each service that shares it using the service-level `volumes` option.",
	"cpu_quota":     "Set resource limits using deploy.resources",
	"cpu_shares":    "Set resource limits using deploy.resources",
	"cpuset":        "Set resource limits using deploy.resources",
	"mem_limit":     "Set resource limits using deploy.resources",
	"memswap_limit": "Set resource limits using deploy.resources",
}

// ConfigFile is a filename and the contents of the file as a Dict
type ConfigFile struct {
	Filename string
	Config   map[string]interface{}
}

// ConfigDetails are the details about a group of ConfigFiles
type ConfigDetails struct {
	WorkingDir  string
	ConfigFiles []ConfigFile
	Environment map[string]string
}

// LookupEnv provides a lookup function for environment variables
func (cd ConfigDetails) LookupEnv(key string) (string, bool) {
	v, ok := cd.Environment[key]
	return v, ok
}

// Config is a full compose file configuration
type Config struct {
	Services []ServiceConfig
	Networks map[string]NetworkConfig
	Volumes  map[string]VolumeConfig
	Secrets  map[string]SecretConfig
	Configs  map[string]ConfigObjConfig
}

// ServiceConfig is the configuration of one service
type ServiceConfig struct {
	Name string

	CapAdd          []string `mapstructure:"cap_add"`
	CapDrop         []string `mapstructure:"cap_drop"`
	CgroupParent    string   `mapstructure:"cgroup_parent"`
	Command         ShellCommand
	Configs         []ServiceConfigObjConfig
	ContainerName   string               `mapstructure:"container_name"`
	CredentialSpec  CredentialSpecConfig `mapstructure:"credential_spec"`
	DependsOn       []string             `mapstructure:"depends_on"`
	Deploy          DeployConfig
	Devices         []string
	DNS             StringList
	DNSSearch       StringList `mapstructure:"dns_search"`
	DomainName      string     `mapstructure:"domainname"`
	Entrypoint      ShellCommand
	Environment     MappingWithEquals
	EnvFile         StringList `mapstructure:"env_file"`
	Expose          StringOrNumberList
	ExternalLinks   []string         `mapstructure:"external_links"`
	ExtraHosts      MappingWithColon `mapstructure:"extra_hosts"`
	Hostname        string
	HealthCheck     *HealthCheckConfig
	Image           string
	Ipc             string
	Labels          Labels
	Links           []string
	Logging         *LoggingConfig
	MacAddress      string `mapstructure:"mac_address"`
	NetworkMode     string `mapstructure:"network_mode"`
	Networks        map[string]*ServiceNetworkConfig
	Pid             string
	Ports           []ServicePortConfig
	Privileged      bool
	ReadOnly        bool `mapstructure:"read_only"`
	Restart         string
	Secrets         []ServiceSecretConfig
	SecurityOpt     []string       `mapstructure:"security_opt"`
	StdinOpen       bool           `mapstructure:"stdin_open"`
	StopGracePeriod *time.Duration `mapstructure:"stop_grace_period"`
	StopSignal      string         `mapstructure:"stop_signal"`
	Tmpfs           StringList
	Tty             bool `mapstructure:"tty"`
	Ulimits         map[string]*UlimitsConfig
	User            string
	Volumes         []ServiceVolumeConfig
	WorkingDir      string `mapstructure:"working_dir"`
}

// ShellCommand is a string or list of string args
type ShellCommand []string

// StringList is a type for fields that can be a string or list of strings
type StringList []string

// StringOrNumberList is a type for fields that can be a list of strings or
// numbers
type StringOrNumberList []string

// MappingWithEquals is a mapping type that can be converted from a list of
// key[=value] strings.
// For the key with an empty value (`key=`), the mapped value is set to a pointer to `""`.
// For the key without value (`key`), the mapped value is set to nil.
type MappingWithEquals map[string]*string

// Labels is a mapping type for labels
type Labels map[string]string

// MappingWithColon is a mapping type that can be converted from a list of
// 'key: value' strings
type MappingWithColon map[string]string

// LoggingConfig the logging configuration for a service
type LoggingConfig struct {
	Driver  string
	Options map[string]string
}

// DeployConfig the deployment configuration for a service
type DeployConfig struct {
	Mode          string
	Replicas      *uint64
	Labels        Labels
	UpdateConfig  *UpdateConfig `mapstructure:"update_config"`
	Resources     Resources
	RestartPolicy *RestartPolicy `mapstructure:"restart_policy"`
	Placement     Placement
	EndpointMode  string `mapstructure:"endpoint_mode"`
}

// HealthCheckConfig the healthcheck configuration for a service
type HealthCheckConfig struct {
	Test        HealthCheckTest
	Timeout     string
	Interval    string
	Retries     *uint64
	StartPeriod string
	Disable     bool
}

// HealthCheckTest is the command run to test the health of a service
type HealthCheckTest []string

// UpdateConfig the service update configuration
type UpdateConfig struct {
	Parallelism     *uint64
	Delay           time.Duration
	FailureAction   string `mapstructure:"failure_action"`
	Monitor         time.Duration
	MaxFailureRatio float32 `mapstructure:"max_failure_ratio"`
	Order           string
}

// Resources the resource limits and reservations
type Resources struct {
	Limits       *Resource
	Reservations *Resource
}

// Resource is a resource to be limited or reserved
type Resource struct {
	// TODO: types to convert from units and ratios
	NanoCPUs    string    `mapstructure:"cpus"`
	MemoryBytes UnitBytes `mapstructure:"memory"`
}

// UnitBytes is the bytes type
type UnitBytes int64

// RestartPolicy the service restart policy
type RestartPolicy struct {
	Condition   string
	Delay       *time.Duration
	MaxAttempts *uint64 `mapstructure:"max_attempts"`
	Window      *time.Duration
}

// Placement constraints for the service
type Placement struct {
	Constraints []string
	Preferences []PlacementPreferences
}

// PlacementPreferences is the preferences for a service placement
type PlacementPreferences struct {
	Spread string
}

// ServiceNetworkConfig is the network configuration for a service
type ServiceNetworkConfig struct {
	Aliases     []string
	Ipv4Address string `mapstructure:"ipv4_address"`
	Ipv6Address string `mapstructure:"ipv6_address"`
}

// ServicePortConfig is the port configuration for a service
type ServicePortConfig struct {
	Mode      string
	Target    uint32
	Published uint32
	Protocol  string
}

// ServiceVolumeConfig are references to a volume used by a service
type ServiceVolumeConfig struct {
	Type        string
	Source      string
	Target      string
	ReadOnly    bool `mapstructure:"read_only"`
	Consistency string
	Bind        *ServiceVolumeBind
	Volume      *ServiceVolumeVolume
}

// ServiceVolumeBind are options for a service volume of type bind
type ServiceVolumeBind struct {
	Propagation string
}

// ServiceVolumeVolume are options for a service volume of type volume
type ServiceVolumeVolume struct {
	NoCopy bool `mapstructure:"nocopy"`
}

type fileReferenceConfig struct {
	Source string
	Target string
	UID    string
	GID    string
	Mode   *uint32
}

// ServiceConfigObjConfig is the config obj configuration for a service
type ServiceConfigObjConfig fileReferenceConfig

// ServiceSecretConfig is the secret configuration for a service
type ServiceSecretConfig fileReferenceConfig

// UlimitsConfig the ulimit configuration
type UlimitsConfig struct {
	Single int
	Soft   int
	Hard   int
}

// NetworkConfig for a network
type NetworkConfig struct {
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	Ipam       IPAMConfig
	External   External
	Internal   bool
	Attachable bool
	Labels     Labels
}

// IPAMConfig for a network
type IPAMConfig struct {
	Driver string
	Config []*IPAMPool
}

// IPAMPool for a network
type IPAMPool struct {
	Subnet string
}

// VolumeConfig for a volume
type VolumeConfig struct {
	Name       string
	Driver     string
	DriverOpts map[string]string `mapstructure:"driver_opts"`
	External   External
	Labels     Labels
}

// External identifies a Volume or Network as a reference to a resource that is
// not managed, and should already exist.
// External.name is deprecated and replaced by Volume.name
type External struct {
	Name     string
	External bool
}

// CredentialSpecConfig for credential spec on Windows
type CredentialSpecConfig struct {
	File     string
	Registry string
}

type fileObjectConfig struct {
	File     string
	External External
	Labels   Labels
}

// SecretConfig for a secret
type SecretConfig fileObjectConfig

// ConfigObjConfig is the config for the swarm "Config" object
type ConfigObjConfig fileObjectConfig
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"os"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/client/versioncli"
	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/data/rand/mathrand"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// version must be updated whenever functional change (behavior, arguments,
// supported commands) is done.
const version = "0.1"

func GetApplication(defaultAuthOpts auth.Options) *subcommands.DefaultApplication {
	defaultAuthOpts.Scopes = []string{auth.OAuthScopeEmail, gitiles.OAuthScope}
	return &subcommands.DefaultApplication{
		Name:  "gitiles",
		Title: "gitiles client",
		// Keep in alphabetical order of their name.
		Commands: []*subcommands.Command{
			cmdArchive(defaultAuthOpts),
			cmdLog(defaultAuthOpts),
			cmdRefs(defaultAuthOpts),
			subcommands.CmdHelp,
			authcli.SubcommandInfo(defaultAuthOpts, "whoami", false),
			authcli.SubcommandLogin(defaultAuthOpts, "login", false),
			authcli.SubcommandLogout(defaultAuthOpts, "logout", false),
			versioncli.CmdVersion(version),
		},
	}
}

func main() {
	mathrand.SeedRandomly()
	app := GetApplication(chromeinfra.DefaultAuthOptions())
	os.Exit(subcommands.Run(app, nil))
}
<commit_msg>gitiles: Fix lint error<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"os"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/client/versioncli"
	"go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/data/rand/mathrand"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// version must be updated whenever functional change (behavior, arguments,
// supported commands) is done.
const version = "0.1"

func getApplication(defaultAuthOpts auth.Options) *subcommands.DefaultApplication {
	defaultAuthOpts.Scopes = []string{auth.OAuthScopeEmail, gitiles.OAuthScope}
	return &subcommands.DefaultApplication{
		Name:  "gitiles",
		Title: "gitiles client",
		// Keep in alphabetical order of their name.
		Commands: []*subcommands.Command{
			cmdArchive(defaultAuthOpts),
			cmdLog(defaultAuthOpts),
			cmdRefs(defaultAuthOpts),
			subcommands.CmdHelp,
			authcli.SubcommandInfo(defaultAuthOpts, "whoami", false),
			authcli.SubcommandLogin(defaultAuthOpts, "login", false),
			authcli.SubcommandLogout(defaultAuthOpts, "logout", false),
			versioncli.CmdVersion(version),
		},
	}
}

func main() {
	mathrand.SeedRandomly()
	app := getApplication(chromeinfra.DefaultAuthOptions())
	os.Exit(subcommands.Run(app, nil))
}
<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// test command tests
// Author: jonmv

package cmd

import (
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSuite(t *testing.T) {
	client := &mockHttpClient{}
	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
	client.NextStatus(200)
	client.NextStatus(200)
	for i := 0; i < 9; i++ {
		client.NextResponse(200, string(searchResponse))
	}

	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected-suite.out")
	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test"}}, t, client)
	assert.Equal(t, string(expectedBytes), outBytes)
	assert.Equal(t, "", errBytes)

	baseUrl := "http://127.0.0.1:8080"
	urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
	requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)}
	for i := 0; i < 7; i++ {
		requests = append(requests, createSearchRequest(baseUrl+"/search/"))
	}
	assertRequests(requests, client, t)
}

func TestTestWithoutAssertions(t *testing.T) {
	client := &mockHttpClient{}
	_, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client)
	assert.Equal(t, "a test must have at least one assertion, but none were found in 'testdata/tests/system-test/foo/query.json'\n", errBytes)
}

func TestSingleTest(t *testing.T) {
	client := &mockHttpClient{}
	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
	client.NextStatus(200)
	client.NextStatus(200)
	client.NextResponse(200, string(searchResponse))
	client.NextResponse(200, string(searchResponse))

	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out")
	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/test.json"}}, t, client)
	assert.Equal(t, string(expectedBytes), outBytes)
	assert.Equal(t, "", errBytes)

	baseUrl := "http://127.0.0.1:8080"
	rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
	assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t)
}

func TestSingleTestWithCloudAndEndpoints(t *testing.T) {
	cmd := command{args: []string{"test", "testdata/tests/system-test/test.json", "-t", "cloud", "-a", "t.a.i"}}
	cmd.homeDir = filepath.Join(t.TempDir(), ".vespa")
	os.MkdirAll(cmd.homeDir, 0700)
	keyFile := filepath.Join(cmd.homeDir, "key")
	certFile := filepath.Join(cmd.homeDir, "cert")

	os.Setenv("VESPA_CLI_DATA_PLANE_KEY_FILE", keyFile)
	os.Setenv("VESPA_CLI_DATA_PLANE_CERT_FILE", certFile)
	os.Setenv("VESPA_CLI_ENDPOINTS", "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"https://url\"}]}")

	kp, _ := vespa.CreateKeyPair()
	ioutil.WriteFile(keyFile, kp.PrivateKey, 0600)
	ioutil.WriteFile(certFile, kp.Certificate, 0600)

	client := &mockHttpClient{}
	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
	client.NextStatus(200)
	client.NextStatus(200)
	client.NextResponse(200, string(searchResponse))
	client.NextResponse(200, string(searchResponse))

	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out")
	outBytes, errBytes := execute(cmd, t, client)
	assert.Equal(t, string(expectedBytes), outBytes)
	assert.Equal(t, "", errBytes)

	baseUrl := "https://url"
	rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
	assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t)
}

func createFeedRequest(urlPrefix string) *http.Request {
	return createRequest("POST",
		urlPrefix+"/document/v1/test/music/docid/doc?timeout=3.4s",
		"{\"fields\":{\"artist\":\"Foo Fighters\"}}")
}

func createSearchRequest(rawUrl string) *http.Request {
	return createRequest("GET", rawUrl, "")
}

func createRequest(method string, uri string, body string) *http.Request {
	requestUrl, _ := url.ParseRequestURI(uri)
	return &http.Request{
		URL:    requestUrl,
		Method: method,
		Header: nil,
		Body:   ioutil.NopCloser(strings.NewReader(body)),
	}
}

func assertRequests(requests []*http.Request, client *mockHttpClient, t *testing.T) {
	if assert.Equal(t, len(requests), len(client.requests)) {
		for i, e := range requests {
			a := client.requests[i]
			assert.Equal(t, e.URL.String(), a.URL.String())
			assert.Equal(t, e.Method, a.Method)
			assert.Equal(t, util.ReaderToJSON(e.Body), util.ReaderToJSON(a.Body))
		}
	}
}
<commit_msg>Test no tests found<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// test command tests
// Author: jonmv

package cmd

import (
	"github.com/vespa-engine/vespa/client/go/util"
	"github.com/vespa-engine/vespa/client/go/vespa"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSuite(t *testing.T) {
	client := &mockHttpClient{}
	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
	client.NextStatus(200)
	client.NextStatus(200)
	for i := 0; i < 9; i++ {
		client.NextResponse(200, string(searchResponse))
	}

	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected-suite.out")
	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test"}}, t, client)
	assert.Equal(t, string(expectedBytes), outBytes)
	assert.Equal(t, "", errBytes)

	baseUrl := "http://127.0.0.1:8080"
	urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
	requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)}
	for i := 0; i < 7; i++ {
		requests = append(requests, createSearchRequest(baseUrl+"/search/"))
	}
	assertRequests(requests, client, t)
}

func TestTestWithoutAssertions(t *testing.T) {
	client := &mockHttpClient{}
	_, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client)
	assert.Equal(t, "a test must have at least one assertion, but none were found in 'testdata/tests/system-test/foo/query.json'\n", errBytes)
}

func TestSuiteWithoutTests(t *testing.T) {
	client := &mockHttpClient{}
	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client)
	assert.Equal(t, "Failed to find any tests at 'testdata/tests/staging-test'\n", outBytes)
	assert.Equal(t, "", errBytes)
}

func TestSingleTest(t *testing.T) {
	client := &mockHttpClient{}
	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
	client.NextStatus(200)
	client.NextStatus(200)
	client.NextResponse(200, string(searchResponse))
	client.NextResponse(200, string(searchResponse))

	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out")
	outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/test.json"}}, t, client)
	assert.Equal(t, string(expectedBytes), outBytes)
	assert.Equal(t, "", errBytes)

	baseUrl := "http://127.0.0.1:8080"
	rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
	assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t)
}

func TestSingleTestWithCloudAndEndpoints(t *testing.T) {
	cmd := command{args: []string{"test", "testdata/tests/system-test/test.json", "-t", "cloud", "-a", "t.a.i"}}
	cmd.homeDir = filepath.Join(t.TempDir(), ".vespa")
	os.MkdirAll(cmd.homeDir, 0700)
	keyFile := filepath.Join(cmd.homeDir, "key")
	certFile := filepath.Join(cmd.homeDir, "cert")

	os.Setenv("VESPA_CLI_DATA_PLANE_KEY_FILE", keyFile)
	os.Setenv("VESPA_CLI_DATA_PLANE_CERT_FILE", certFile)
	os.Setenv("VESPA_CLI_ENDPOINTS", "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"https://url\"}]}")

	kp, _ := vespa.CreateKeyPair()
	ioutil.WriteFile(keyFile, kp.PrivateKey, 0600)
	ioutil.WriteFile(certFile, kp.Certificate, 0600)

	client := &mockHttpClient{}
	searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json")
	client.NextStatus(200)
	client.NextStatus(200)
	client.NextResponse(200, string(searchResponse))
	client.NextResponse(200, string(searchResponse))

	expectedBytes, _ := ioutil.ReadFile("testdata/tests/expected.out")
	outBytes, errBytes := execute(cmd, t, client)
	assert.Equal(t, string(expectedBytes), outBytes)
	assert.Equal(t, "", errBytes)

	baseUrl := "https://url"
	rawUrl := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s"
	assertRequests([]*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(rawUrl), createSearchRequest(rawUrl)}, client, t)
}

func createFeedRequest(urlPrefix string) *http.Request {
	return createRequest("POST",
		urlPrefix+"/document/v1/test/music/docid/doc?timeout=3.4s",
		"{\"fields\":{\"artist\":\"Foo Fighters\"}}")
}

func createSearchRequest(rawUrl string) *http.Request {
	return createRequest("GET", rawUrl, "")
}

func createRequest(method string, uri string, body string) *http.Request {
	requestUrl, _ := url.ParseRequestURI(uri)
	return &http.Request{
		URL:    requestUrl,
		Method: method,
		Header: nil,
		Body:   ioutil.NopCloser(strings.NewReader(body)),
	}
}

func assertRequests(requests []*http.Request, client *mockHttpClient, t *testing.T) {
	if assert.Equal(t, len(requests), len(client.requests)) {
		for i, e := range requests {
			a := client.requests[i]
			assert.Equal(t, e.URL.String(), a.URL.String())
			assert.Equal(t, e.Method, a.Method)
			assert.Equal(t, util.ReaderToJSON(e.Body), util.ReaderToJSON(a.Body))
		}
	}
}
<|endoftext|>


<commit_before>package openvswitch

import (
	"os/exec"

	"github.com/lxc/lxd/shared"
)

// NewOVS initialises new OVS wrapper.
func NewOVS() *OVS {
	return &OVS{}
}

// OVS command wrapper.
type OVS struct{}

// Installed returns true if OVS tools are installed.
func (o *OVS) Installed() bool {
	_, err := exec.LookPath("ovs-vsctl")
	if err != nil {
		return false
	}

	return true
}

// BridgeExists returns true if OVS bridge exists.
func (o *OVS) BridgeExists(bridgeName string) (bool, error) {
	_, err := shared.RunCommand("ovs-vsctl", "br-exists", bridgeName)
	if err != nil {
		runErr, ok := err.(shared.RunError)
		if ok {
			exitError, ok := runErr.Err.(*exec.ExitError)

			// ovs-vsctl manpage says that br-exists exits with code 2 if bridge doesn't exist.
			if ok && exitError.ExitCode() == 2 {
				return false, nil
			}
		}

		return false, err
	}

	return true, nil
}

// BridgeAdd adds an OVS bridge.
func (o *OVS) BridgeAdd(bridgeName string) error {
	_, err := shared.RunCommand("ovs-vsctl", "add-br", bridgeName)
	if err != nil {
		return err
	}

	return nil
}

// BridgeDelete deletes an OVS bridge.
func (o *OVS) BridgeDelete(bridgeName string) error {
	_, err := shared.RunCommand("ovs-vsctl", "del-br", bridgeName)
	if err != nil {
		return err
	}

	return nil
}

// BridgeAddPort adds a port to the bridge (if already attached does nothing).
func (o *OVS) BridgeAddPort(bridgeName string, portName string) error {
	// Check if interface is already connected to a bridge, if not, connect it to the specified bridge.
	_, err := shared.RunCommand("ovs-vsctl", "port-to-br", portName)
	if err != nil {
		_, err := shared.RunCommand("ovs-vsctl", "add-port", bridgeName, portName)
		if err != nil {
			return err
		}
	}

	return nil
}

// BridgeDeletePort deletes a port from the bridge (if already deteached does nothing).
func (o *OVS) BridgeDeletePort(bridgeName string, portName string) error {
	// Check if interface is connected to a bridge, if so, then remove it from the bridge.
	_, err := shared.RunCommand("ovs-vsctl", "port-to-br", portName)
	if err == nil {
		_, err := shared.RunCommand("ovs-vsctl", "del-port", bridgeName, portName)
		if err != nil {
			return err
		}
	}

	return nil
}

// PortSet sets port options.
func (o *OVS) PortSet(portName string, options ...string) error {
	_, err := shared.RunCommand("ovs-vsctl", append([]string{"set", "port", portName}, options...)...)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/network/openvswitch: Name functions consistently using ObjectAction format<commit_after>package openvswitch

import (
	"os/exec"

	"github.com/lxc/lxd/shared"
)

// NewOVS initialises new OVS wrapper.
func NewOVS() *OVS {
	return &OVS{}
}

// OVS command wrapper.
type OVS struct{}

// Installed returns true if OVS tools are installed.
func (o *OVS) Installed() bool {
	_, err := exec.LookPath("ovs-vsctl")
	if err != nil {
		return false
	}

	return true
}

// BridgeExists returns true if OVS bridge exists.
func (o *OVS) BridgeExists(bridgeName string) (bool, error) {
	_, err := shared.RunCommand("ovs-vsctl", "br-exists", bridgeName)
	if err != nil {
		runErr, ok := err.(shared.RunError)
		if ok {
			exitError, ok := runErr.Err.(*exec.ExitError)

			// ovs-vsctl manpage says that br-exists exits with code 2 if bridge doesn't exist.
			if ok && exitError.ExitCode() == 2 {
				return false, nil
			}
		}

		return false, err
	}

	return true, nil
}

// BridgeAdd adds an OVS bridge.
func (o *OVS) BridgeAdd(bridgeName string, mayExist bool) error {
	args := []string{}

	if mayExist {
		args = append(args, "--may-exist")
	}

	args = append(args, "add-br", bridgeName)

	_, err := shared.RunCommand("ovs-vsctl", args...)
	if err != nil {
		return err
	}

	return nil
}

// BridgeDelete deletes an OVS bridge.
func (o *OVS) BridgeDelete(bridgeName string) error {
	_, err := shared.RunCommand("ovs-vsctl", "del-br", bridgeName)
	if err != nil {
		return err
	}

	return nil
}

// BridgePortAdd adds a port to the bridge (if already attached does nothing).
func (o *OVS) BridgePortAdd(bridgeName string, portName string, mayExist bool) error {
	args := []string{}

	if mayExist {
		args = append(args, "--may-exist")
	}

	args = append(args, "add-port", bridgeName, portName)

	_, err := shared.RunCommand("ovs-vsctl", args...)
	if err != nil {
		return err
	}

	return nil
}

// BridgePortDelete deletes a port from the bridge (if already detached does nothing).
func (o *OVS) BridgePortDelete(bridgeName string, portName string) error {
	_, err := shared.RunCommand("ovs-vsctl", "--if-exists", "del-port", bridgeName, portName)
	if err != nil {
		return err
	}

	return nil
}

// BridgePortSet sets port options.
func (o *OVS) BridgePortSet(portName string, options ...string) error {
	_, err := shared.RunCommand("ovs-vsctl", append([]string{"set", "port", portName}, options...)...)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"

	"golang.org/x/sync/errgroup"

	"github.com/Debian/debiman/internal/archive"
	ptarchive "pault.ag/go/archive"
	"pault.ag/go/debian/control"
)

type contentEntry struct {
	suite     string
	arch      string
	binarypkg string
	filename  string
}

var manPrefix = []byte("usr/share/man/")

func parseContentsEntry(scanner *bufio.Scanner) ([]*contentEntry, error) {
	for scanner.Scan() {
		text := scanner.Bytes()
		if !bytes.HasPrefix(text, manPrefix) {
			continue
		}

		idx := bytes.LastIndex(text, []byte{' '})
		if idx == -1 {
			continue
		}
		parts := bytes.Split(text[idx:], []byte{','})
		entries := make([]*contentEntry, 0, len(parts))
		for _, part := range parts {
			idx2 := bytes.IndexByte(part, '/')
			if idx2 == -1 {
				continue
			}
			entries = append(entries, &contentEntry{
				binarypkg: string(part[idx2+1:]),
				filename:  string(bytes.TrimSpace(text[len(manPrefix):idx])),
			})
		}
		if len(entries) > 0 {
			return entries, nil
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return nil, io.EOF
}

func getContents(ar *archive.Getter, suite string, component string, archs []string, hashByFilename map[string]*control.SHA256FileHash) ([]*contentEntry, error) {
	files := make([]*os.File, len(archs))
	scanners := make([]*bufio.Scanner, len(archs))
	contents := make([][]*contentEntry, len(archs))
	advance := make([]bool, len(archs))
	exhausted := make([]bool, len(archs))
	var eg errgroup.Group
	for idx, arch := range archs {
		idx := idx   // copy
		arch := arch // copy
		eg.Go(func() error {
			path := component + "/Contents-" + arch + ".gz"
			fh, ok := hashByFilename[path]
			if !ok {
				return fmt.Errorf("ERROR: expected path %q not found in Release file", path)
			}

			h, err := hex.DecodeString(fh.Hash)
			if err != nil {
				return err
			}

			log.Printf("getting %q (hash %v)", suite+"/"+path, fh.Hash)
			r, err := ar.Get("dists/"+suite+"/"+path, h)
			if err != nil {
				return err
			}

			files[idx] = r
			scanners[idx] = bufio.NewScanner(r)
			contents[idx], err = parseContentsEntry(scanners[idx])
			if err != nil {
				return err
			}
			advance[idx] = false
			return nil
		})
	}
	defer func() {
		for _, f := range files {
			if f != nil {
				f.Close()
			}
		}
	}()
	if err := eg.Wait(); err != nil {
		return nil, err
	}

	var entries []*contentEntry
	for {
		for idx, move := range advance {
			if !move {
				continue
			}
			var err error
			contents[idx], err = parseContentsEntry(scanners[idx])
			if err != nil {
				if err == io.EOF {
					exhausted[idx] = true
				} else {
					return nil, err
				}
			}
		}
		// TODO: unit test for edge cases: can this loop indefinitely or can packages be skipped here?
		if done(exhausted) {
			break
		}

		// find the filename which is the least advanced in the sort order
		var lowest int
		var sum int
		for idx := range archs {
			sum += len(contents[idx])
			if exhausted[idx] {
				continue
			}
			if contents[idx][0].filename < contents[lowest][0].filename {
				lowest = idx
			}
		}

		for idx := range advance {
			advance[idx] = !exhausted[idx] && contents[lowest][0].filename == contents[idx][0].filename
		}

		binarypkgs := make(map[string]string, sum)
		for idx := range archs {
			if !advance[idx] {
				continue
			}

			for _, e := range contents[idx] {
				// first arch (amd64) wins
				if _, ok := binarypkgs[e.binarypkg]; !ok {
					binarypkgs[e.binarypkg] = archs[idx]
				}
			}
		}

		for pkg, arch := range binarypkgs {
			entries = append(entries, &contentEntry{
				binarypkg: pkg,
				arch:      arch,
				filename:  contents[lowest][0].filename,
				suite:     suite,
			})
		}
	}

	return entries, nil
}

func getAllContents(ar *archive.Getter, suite string, release *ptarchive.Release, hashByFilename map[string]*control.SHA256FileHash) ([]*contentEntry, error) {
	// We skip archAll, because there is no Contents-all file. The
	// contents of Architecture: all packages are included in the
	// architecture-specific Contents-* files.

	// TODO(later): make this code work with all components once it’s
	// confirmed that we are interested in serving more than just
	// main.
	for _, component := range []string{"main"} {
		archs := make([]string, len(release.Architectures))
		for idx, arch := range release.Architectures {
			archs[idx] = arch.String()
		}

		return getContents(ar, suite, component, archs, hashByFilename)
	}

	return nil, nil
}
<commit_msg>bugfix: correctly split Debian sections containing slashes<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"

	"golang.org/x/sync/errgroup"

	"github.com/Debian/debiman/internal/archive"
	ptarchive "pault.ag/go/archive"
	"pault.ag/go/debian/control"
)

type contentEntry struct {
	suite     string
	arch      string
	binarypkg string
	filename  string
}

var manPrefix = []byte("usr/share/man/")

func parseContentsEntry(scanner *bufio.Scanner) ([]*contentEntry, error) {
	for scanner.Scan() {
		text := scanner.Bytes()
		if !bytes.HasPrefix(text, manPrefix) {
			continue
		}

		idx := bytes.LastIndex(text, []byte{' '})
		if idx == -1 {
			continue
		}
		parts := bytes.Split(text[idx:], []byte{','})
		entries := make([]*contentEntry, 0, len(parts))
		for _, part := range parts {
			idx2 := bytes.LastIndex(part, []byte{'/'})
			if idx2 == -1 {
				continue
			}
			entries = append(entries, &contentEntry{
				binarypkg: string(part[idx2+1:]),
				filename:  string(bytes.TrimSpace(text[len(manPrefix):idx])),
			})
		}
		if len(entries) > 0 {
			return entries, nil
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return nil, io.EOF
}

func getContents(ar *archive.Getter, suite string, component string, archs []string, hashByFilename map[string]*control.SHA256FileHash) ([]*contentEntry, error) {
	files := make([]*os.File, len(archs))
	scanners := make([]*bufio.Scanner, len(archs))
	contents := make([][]*contentEntry, len(archs))
	advance := make([]bool, len(archs))
	exhausted := make([]bool, len(archs))
	var eg errgroup.Group
	for idx, arch := range archs {
		idx := idx   // copy
		arch := arch // copy
		eg.Go(func() error {
			path := component + "/Contents-" + arch + ".gz"
			fh, ok := hashByFilename[path]
			if !ok {
				return fmt.Errorf("ERROR: expected path %q not found in Release file", path)
			}

			h, err := hex.DecodeString(fh.Hash)
			if err != nil {
				return err
			}

			log.Printf("getting %q (hash %v)", suite+"/"+path, fh.Hash)
			r, err := ar.Get("dists/"+suite+"/"+path, h)
			if err != nil {
				return err
			}

			files[idx] = r
			scanners[idx] = bufio.NewScanner(r)
			contents[idx], err = parseContentsEntry(scanners[idx])
			if err != nil {
				return err
			}
			advance[idx] = false
			return nil
		})
	}
	defer func() {
		for _, f := range files {
			if f != nil {
				f.Close()
			}
		}
	}()
	if err := eg.Wait(); err != nil {
		return nil, err
	}

	var entries []*contentEntry
	for {
		for idx, move := range advance {
			if !move {
				continue
			}
			var err error
			contents[idx], err = parseContentsEntry(scanners[idx])
			if err != nil {
				if err == io.EOF {
					exhausted[idx] = true
				} else {
					return nil, err
				}
			}
		}
		// TODO: unit test for edge cases: can this loop indefinitely or can packages be skipped here?
		if done(exhausted) {
			break
		}

		// find the filename which is the least advanced in the sort order
		var lowest int
		var sum int
		for idx := range archs {
			sum += len(contents[idx])
			if exhausted[idx] {
				continue
			}
			if contents[idx][0].filename < contents[lowest][0].filename {
				lowest = idx
			}
		}

		for idx := range advance {
			advance[idx] = !exhausted[idx] && contents[lowest][0].filename == contents[idx][0].filename
		}

		binarypkgs := make(map[string]string, sum)
		for idx := range archs {
			if !advance[idx] {
				continue
			}

			for _, e := range contents[idx] {
				// first arch (amd64) wins
				if _, ok := binarypkgs[e.binarypkg]; !ok {
					binarypkgs[e.binarypkg] = archs[idx]
				}
			}
		}

		for pkg, arch := range binarypkgs {
			entries = append(entries, &contentEntry{
				binarypkg: pkg,
				arch:      arch,
				filename:  contents[lowest][0].filename,
				suite:     suite,
			})
		}
	}

	return entries, nil
}

func getAllContents(ar *archive.Getter, suite string, release *ptarchive.Release, hashByFilename map[string]*control.SHA256FileHash) ([]*contentEntry, error) {
	// We skip archAll, because there is no Contents-all file. The
	// contents of Architecture: all packages are included in the
	// architecture-specific Contents-* files.

	// TODO(later): make this code work with all components once it’s
	// confirmed that we are interested in serving more than just
	// main.
	for _, component := range []string{"main"} {
		archs := make([]string, len(release.Architectures))
		for idx, arch := range release.Architectures {
			archs[idx] = arch.String()
		}

		return getContents(ar, suite, component, archs, hashByFilename)
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package tunnel

import (
	"bytes"
	"io"
	"sync"
	"time"

	"github.com/google/uuid"
	"github.com/micro/go-micro/transport"
	"github.com/micro/go-micro/util/log"
)

type link struct {
	transport.Socket

	sync.RWMutex
	// stops the link
	closed chan bool
	// link state channel for testing link
	state chan *packet
	// send queue for sending packets
	sendQueue chan *packet
	// receive queue for receiving packets
	recvQueue chan *packet
	// unique id of this link e.g uuid
	// which we define for ourselves
	id string
	// whether its a loopback connection
	// this flag is used by the transport listener
	// which accepts inbound quic connections
	loopback bool
	// whether its actually connected
	// dialled side sets it to connected
	// after sending the message. the
	// listener waits for the connect
	connected bool
	// the last time we received a keepalive
	// on this link from the remote side
	lastKeepAlive time.Time
	// channels keeps a mapping of channels and last seen
	channels map[string]time.Time
	// the weighted moving average roundtrip
	length int64
	// weighted moving average of bits flowing
	rate float64
	// keep an error count on the link
	errCount int
}

// packet send over link
type packet struct {
	// message to send or received
	message *transport.Message

	// status returned when sent
	status chan error

	// receive related error
	err error
}

var (
	// the 4 byte 0 packet sent to determine the link state
	linkRequest = []byte{0, 0, 0, 0}
	// the 4 byte 1 filled packet sent to determine link state
	linkResponse = []byte{1, 1, 1, 1}
)

func newLink(s transport.Socket) *link {
	l := &link{
		Socket:        s,
		id:            uuid.New().String(),
		lastKeepAlive: time.Now(),
		channels:      make(map[string]time.Time),
		closed:        make(chan bool),
		state:         make(chan *packet, 64),
		sendQueue:     make(chan *packet, 128),
		recvQueue:     make(chan *packet, 128),
	}

	// process inbound/outbound packets
	go l.process()
	// manage the link state
	go l.manage()

	return l
}

// setRate sets the bits per second rate as a float64
func (l *link) setRate(bits int64, delta time.Duration) {
	// rate of send in bits per nanosecond
	rate := float64(bits) / float64(delta.Nanoseconds())

	// default the rate if its zero
	if l.rate == 0 {
		// rate per second
		l.rate = rate * 1e9
	} else {
		// set new rate per second
		l.rate = 0.8*l.rate + 0.2*(rate*1e9)
	}
}

// setRTT sets a nanosecond based moving average roundtrip time for the link
func (l *link) setRTT(d time.Duration) {
	l.Lock()
	defer l.Unlock()

	if l.length <= 0 {
		l.length = d.Nanoseconds()
		return
	}

	// https://fishi.devtail.io/weblog/2015/04/12/measuring-bandwidth-and-round-trip-time-tcp-connection-inside-application-layer/
	length := 0.8*float64(l.length) + 0.2*float64(d.Nanoseconds())
	// set new length
	l.length = int64(length)
}

func (l *link) delChannel(ch string) {
	l.Lock()
	delete(l.channels, ch)
	l.Unlock()
}

func (l *link) setChannel(channels ...string) {
	l.Lock()
	for _, ch := range channels {
		l.channels[ch] = time.Now()
	}
	l.Unlock()
}

func (l *link) getChannel(ch string) time.Time {
	l.RLock()
	defer l.RUnlock()
	return l.channels[ch]
}

// set the keepalive time
func (l *link) keepalive() {
	l.Lock()
	l.lastKeepAlive = time.Now()
	l.Unlock()
}

// process deals with the send queue
func (l *link) process() {
	// receive messages
	go func() {
		for {
			m := new(transport.Message)
			err := l.recv(m)
			if err != nil {
				l.Lock()
				l.errCount++
				l.Unlock()
			}

			// process new received message

			pk := &packet{message: m, err: err}

			// this is our link state packet
			if m.Header["Micro-Method"] == "link" {
				// process link state message
				select {
				case l.state <- pk:
				default:
				}
				continue
			}

			// process all messages as is

			select {
			case l.recvQueue <- pk:
			case <-l.closed:
				return
			}
		}
	}()

	// send messages

	for {
		select {
		case pk := <-l.sendQueue:
			// send the message
			pk.status <- l.send(pk.message)
		case <-l.closed:
			return
		}
	}
}

// manage manages the link state including rtt packets and channel mapping expiry
func (l *link) manage() {
	// tick over every minute to expire and fire rtt packets
	t := time.NewTicker(time.Minute)
	defer t.Stop()

	// used to send link state packets
	send := func(b []byte) error {
		return l.Send(&transport.Message{
			Header: map[string]string{
				"Micro-Method": "link",
			}, Body: b,
		})
	}

	// set time now
	now := time.Now()

	// send the initial rtt request packet
	send(linkRequest)

	for {
		select {
		// exit if closed
		case <-l.closed:
			return
		// process link state rtt packets
		case p := <-l.state:
			if p.err != nil {
				continue
			}
			// check the type of message
			switch {
			case bytes.Compare(p.message.Body, linkRequest) == 0:
				log.Tracef("Link %s received link request %v", l.id, p.message.Body)
				// send response
				if err := send(linkResponse); err != nil {
					l.Lock()
					l.errCount++
					l.Unlock()
				}
			case bytes.Compare(p.message.Body, linkResponse) == 0:
				// set round trip time
				d := time.Since(now)
				log.Tracef("Link %s received link response in %v", p.message.Body, d)
				l.setRTT(d)
			}
		case <-t.C:
			// drop any channel mappings older than 2 minutes
			var kill []string
			killTime := time.Minute * 2

			l.RLock()
			for ch, t := range l.channels {
				if d := time.Since(t); d > killTime {
					kill = append(kill, ch)
				}
			}
			l.RUnlock()

			// if nothing to kill don't bother with a wasted lock
			if len(kill) == 0 {
				continue
			}

			// kill the channels!
			l.Lock()
			for _, ch := range kill {
				delete(l.channels, ch)
			}
			l.Unlock()

			// fire off a link state rtt packet
			now = time.Now()
			send(linkRequest)
		}
	}
}

func (l *link) send(m *transport.Message) error {
	if m.Header == nil {
		m.Header = make(map[string]string)
	}
	// send the message
	return l.Socket.Send(m)
}

// recv a message on the link
func (l *link) recv(m *transport.Message) error {
	if m.Header == nil {
		m.Header = make(map[string]string)
	}
	// receive the transport message
	return l.Socket.Recv(m)
}

// Delay is the current load on the link
func (l *link) Delay() int64 {
	return int64(len(l.sendQueue) + len(l.recvQueue))
}

// Current transfer rate as bits per second (lower is better)
func (l *link) Rate() float64 {
	l.RLock()
	defer l.RUnlock()
	return l.rate
}

// Length returns the roundtrip time as nanoseconds (lower is better).
// Returns 0 where no measurement has been taken.
func (l *link) Length() int64 {
	l.RLock()
	defer l.RUnlock()
	return l.length
}

func (l *link) Id() string {
	l.RLock()
	defer l.RUnlock()

	return l.id
}

func (l *link) Close() error {
	l.Lock()
	defer l.Unlock()

	select {
	case <-l.closed:
		return nil
	default:
		l.Socket.Close()
		close(l.closed)
	}

	return nil
}

// Send sencs a message on the link
func (l *link) Send(m *transport.Message) error {
	// create a new packet to send over the link
	p := &packet{
		message: m,
		status:  make(chan error, 1),
	}

	// get time now
	now := time.Now()

	// check if its closed first
	select {
	case <-l.closed:
		return io.EOF
	default:
	}

	// queue the message
	select {
	case l.sendQueue <- p:
		// in the send queue
	case <-l.closed:
		return io.EOF
	}

	// error to use
	var err error

	// wait for response
	select {
	case <-l.closed:
		return io.EOF
	case err = <-p.status:
	}

	l.Lock()
	defer l.Unlock()

	// there's an error increment the counter and bail
	if err != nil {
		l.errCount++
		return err
	}

	// reset the counter
	l.errCount = 0

	// calculate the data sent
	dataSent := len(m.Body)

	// set header length
	for k, v := range m.Header {
		dataSent += (len(k) + len(v))
	}

	// calculate based on data
	if dataSent > 0 {
		// bit sent
		bits := dataSent * 1024

		// set the rate
		l.setRate(int64(bits), time.Since(now))
	}

	return nil
}

// Accept accepts a message on the socket
func (l *link) Recv(m *transport.Message) error {
	select {
	case <-l.closed:
		// check if there's any messages left
		select {
		case pk := <-l.recvQueue:
			// check the packet receive error
			if pk.err != nil {
				return pk.err
			}
			*m = *pk.message
		default:
			return io.EOF
		}
	case pk := <-l.recvQueue:
		// check the packet receive error
		if pk.err != nil {
			return pk.err
		}
		*m = *pk.message
	}
	return nil
}

// State can return connected, closed, error
func (l *link) State() string {
	select {
	case <-l.closed:
		return "closed"
	default:
		l.RLock()
		defer l.RUnlock()
		if l.errCount > 3 {
			return "error"
		}
		return "connected"
	}
}
<commit_msg>Reorder setChannel method<commit_after>package tunnel

import (
	"bytes"
	"io"
	"sync"
	"time"

	"github.com/google/uuid"
	"github.com/micro/go-micro/transport"
	"github.com/micro/go-micro/util/log"
)

type link struct {
	transport.Socket

	sync.RWMutex
	// stops the link
	closed chan bool
	// link state channel for testing link
	state chan *packet
	// send queue for sending packets
	sendQueue chan *packet
	// receive queue for receiving packets
	recvQueue chan *packet
	// unique id of this link e.g uuid
	// which we define for ourselves
	id string
	// whether its a loopback connection
	// this flag is used by the transport listener
	// which accepts inbound quic connections
	loopback bool
	// whether its actually connected
	// dialled side sets it to connected
	// after sending the message. the
	// listener waits for the connect
	connected bool
	// the last time we received a keepalive
	// on this link from the remote side
	lastKeepAlive time.Time
	// channels keeps a mapping of channels and last seen
	channels map[string]time.Time
	// the weighted moving average roundtrip
	length int64
	// weighted moving average of bits flowing
	rate float64
	// keep an error count on the link
	errCount int
}

// packet send over link
type packet struct {
	// message to send or received
	message *transport.Message

	// status returned when sent
	status chan error

	// receive related error
	err error
}

var (
	// the 4 byte 0 packet sent to determine the link state
	linkRequest = []byte{0, 0, 0, 0}
	// the 4 byte 1 filled packet sent to determine link state
	linkResponse = []byte{1, 1, 1, 1}
)

func newLink(s transport.Socket) *link {
	l := &link{
		Socket:        s,
		id:            uuid.New().String(),
		lastKeepAlive: time.Now(),
		channels:      make(map[string]time.Time),
		closed:        make(chan bool),
		state:         make(chan *packet, 64),
		sendQueue:     make(chan *packet, 128),
		recvQueue:     make(chan *packet, 128),
	}

	// process inbound/outbound packets
	go l.process()
	// manage the link state
	go l.manage()

	return l
}

// setRate sets the bits per second rate as a float64
func (l *link) setRate(bits int64, delta time.Duration) {
	// rate of send in bits per nanosecond
	rate := float64(bits) / float64(delta.Nanoseconds())

	// default the rate if its zero
	if l.rate == 0 {
		// rate per second
		l.rate = rate * 1e9
	} else {
		// set new rate per second
		l.rate = 0.8*l.rate + 0.2*(rate*1e9)
	}
}

// setRTT sets a nanosecond based moving average roundtrip time for the link
func (l *link) setRTT(d time.Duration) {
	l.Lock()
	defer l.Unlock()

	if l.length <= 0 {
		l.length = d.Nanoseconds()
		return
	}

	// https://fishi.devtail.io/weblog/2015/04/12/measuring-bandwidth-and-round-trip-time-tcp-connection-inside-application-layer/
	length := 0.8*float64(l.length) + 0.2*float64(d.Nanoseconds())
	// set new length
	l.length = int64(length)
}

func (l *link) delChannel(ch string) {
	l.Lock()
	delete(l.channels, ch)
	l.Unlock()
}

func (l *link) getChannel(ch string) time.Time {
	l.RLock()
	defer l.RUnlock()
	return l.channels[ch]
}

func (l *link) setChannel(channels ...string) {
	l.Lock()
	for _, ch := range channels {
		l.channels[ch] = time.Now()
	}
	l.Unlock()
}

// set the keepalive time
func (l *link) keepalive() {
	l.Lock()
	l.lastKeepAlive = time.Now()
	l.Unlock()
}

// process deals with the send queue
func (l *link) process() {
	// receive messages
	go func() {
		for {
			m := new(transport.Message)
			err := l.recv(m)
			if err != nil {
				l.Lock()
				l.errCount++
				l.Unlock()
			}

			// process new received message

			pk := &packet{message: m, err: err}

			// this is our link state packet
			if m.Header["Micro-Method"] == "link" {
				// process link state message
				select {
				case l.state <- pk:
				default:
				}
				continue
			}

			// process all messages as is

			select {
			case l.recvQueue <- pk:
			case <-l.closed:
				return
			}
		}
	}()

	// send messages

	for {
		select {
		case pk := <-l.sendQueue:
			// send the message
			pk.status <- l.send(pk.message)
		case <-l.closed:
			return
		}
	}
}

// manage manages the link state including rtt packets and channel mapping expiry
func (l *link) manage() {
	// tick over every minute to expire and fire rtt packets
	t := time.NewTicker(time.Minute)
	defer t.Stop()

	// used to send link state packets
	send := func(b []byte) error {
		return l.Send(&transport.Message{
			Header: map[string]string{
				"Micro-Method": "link",
			}, Body: b,
		})
	}

	// set time now
	now := time.Now()

	// send the initial rtt request packet
	send(linkRequest)

	for {
		select {
		// exit if closed
		case <-l.closed:
			return
		// process link state rtt packets
		case p := <-l.state:
			if p.err != nil {
				continue
			}
			// check the type of message
			switch {
			case bytes.Compare(p.message.Body, linkRequest) == 0:
				log.Tracef("Link %s received link request %v", l.id, p.message.Body)
				// send response
				if err := send(linkResponse); err != nil {
					l.Lock()
					l.errCount++
					l.Unlock()
				}
			case bytes.Compare(p.message.Body, linkResponse) == 0:
				// set round trip time
				d := time.Since(now)
				log.Tracef("Link %s received link response in %v", p.message.Body, d)
				l.setRTT(d)
			}
		case <-t.C:
			// drop any channel mappings older than 2 minutes
			var kill []string
			killTime := time.Minute * 2

			l.RLock()
			for ch, t := range l.channels {
				if d := time.Since(t); d > killTime {
					kill = append(kill, ch)
				}
			}
			l.RUnlock()

			// if nothing to kill don't bother with a wasted lock
			if len(kill) == 0 {
				continue
			}

			// kill the channels!
			l.Lock()
			for _, ch := range kill {
				delete(l.channels, ch)
			}
			l.Unlock()

			// fire off a link state rtt packet
			now = time.Now()
			send(linkRequest)
		}
	}
}

func (l *link) send(m *transport.Message) error {
	if m.Header == nil {
		m.Header = make(map[string]string)
	}
	// send the message
	return l.Socket.Send(m)
}

// recv a message on the link
func (l *link) recv(m *transport.Message) error {
	if m.Header == nil {
		m.Header = make(map[string]string)
	}
	// receive the transport message
	return l.Socket.Recv(m)
}

// Delay is the current load on the link
func (l *link) Delay() int64 {
	return int64(len(l.sendQueue) + len(l.recvQueue))
}

// Current transfer rate as bits per second (lower is better)
func (l *link) Rate() float64 {
	l.RLock()
	defer l.RUnlock()
	return l.rate
}

// Length returns the roundtrip time as nanoseconds (lower is better).
// Returns 0 where no measurement has been taken.
func (l *link) Length() int64 {
	l.RLock()
	defer l.RUnlock()
	return l.length
}

func (l *link) Id() string {
	l.RLock()
	defer l.RUnlock()

	return l.id
}

func (l *link) Close() error {
	l.Lock()
	defer l.Unlock()

	select {
	case <-l.closed:
		return nil
	default:
		l.Socket.Close()
		close(l.closed)
	}

	return nil
}

// Send sencs a message on the link
func (l *link) Send(m *transport.Message) error {
	// create a new packet to send over the link
	p := &packet{
		message: m,
		status:  make(chan error, 1),
	}

	// get time now
	now := time.Now()

	// check if its closed first
	select {
	case <-l.closed:
		return io.EOF
	default:
	}

	// queue the message
	select {
	case l.sendQueue <- p:
		// in the send queue
	case <-l.closed:
		return io.EOF
	}

	// error to use
	var err error

	// wait for response
	select {
	case <-l.closed:
		return io.EOF
	case err = <-p.status:
	}

	l.Lock()
	defer l.Unlock()

	// there's an error increment the counter and bail
	if err != nil {
		l.errCount++
		return err
	}

	// reset the counter
	l.errCount = 0

	// calculate the data sent
	dataSent := len(m.Body)

	// set header length
	for k, v := range m.Header {
		dataSent += (len(k) + len(v))
	}

	// calculate based on data
	if dataSent > 0 {
		// bit sent
		bits := dataSent * 1024

		// set the rate
		l.setRate(int64(bits), time.Since(now))
	}

	return nil
}

// Accept accepts a message on the socket
func (l *link) Recv(m *transport.Message) error {
	select {
	case <-l.closed:
		// check if there's any messages left
		select {
		case pk := <-l.recvQueue:
			// check the packet receive error
			if pk.err != nil {
				return pk.err
			}
			*m = *pk.message
		default:
			return io.EOF
		}
	case pk := <-l.recvQueue:
		// check the packet receive error
		if pk.err != nil {
			return pk.err
		}
		*m = *pk.message
	}
	return nil
}

// State can return connected, closed, error
func (l *link) State() string {
	select {
	case <-l.closed:
		return "closed"
	default:
		l.RLock()
		defer l.RUnlock()
		if l.errCount > 3 {
			return "error"
		}
		return "connected"
	}
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/funkygao/gocli"
)

type Logstash struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Logstash) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("logstash", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.showSampleConfig()

	return
}

func (this *Logstash) showSampleConfig() {
	config := `
input {
    file {
        path => "/var/wd/ehaproxy/ehaproxy.log"
        type => "ehaproxy"
    }
    file {
        path => "/var/wd/ehaproxy/panic"
        type => "ehaproxy_panic"
    }
    file {
        path => "/var/wd/ehaproxy/logs/haproxy.log"
        type => "haproxy"
    }
    file {
        path => "/var/wd/kateway/kateway.log"
        type => "kateway"
    }
    file {
        path => "/var/wd/kateway/panic"
        type => "kateway_panic"
    }
}

filter {
    multiline {
        pattern => "^201" # e,g. this line begins with 2017-01-22
        what => "previous"
        negate => true
    }
}

output {
    if [type] == "pubsub" {
         http {
             http_method => "post"
             url => "http://pub.test.mycorp.com:10191/v1/raw/msgs/cluster/topic"
             workers => 8 # each worker has 25 http connection, totals: 200
             headers => {
                "User-Agent" => "logstash"
             }
         }
    } else {
        kafka {
            bootstrap_servers => "k11003a.mycorp.kfk.com:11003,k11003b.mycorp.kfk.com:11003"
            topic_id => "pubsub_log"
            metadata_max_age_ms => 300000
            workers => 1
            retries => 1
        }
    }
}`
	this.Ui.Output(strings.TrimSpace(config))
}

func (*Logstash) Synopsis() string {
	return "Sample configuration for logstash"
}

func (this *Logstash) Help() string {
	help := fmt.Sprintf(`
Usage: %s logstash [options]

    %s

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>inject audit intro central repo<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/funkygao/gocli"
)

type Logstash struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Logstash) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("logstash", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.showSampleConfig()

	return
}

func (this *Logstash) showSampleConfig() {
	config := `
input {
    file {
        path => "/var/wd/ehaproxy/ehaproxy.log"
        type => "ehaproxy"
    }
    file {
        path => "/var/wd/ehaproxy/panic"
        type => "ehaproxy_panic"
    }
    file {
        path => "/var/wd/ehaproxy/logs/haproxy.log"
        type => "haproxy"
    }
    file {
        path => "/var/wd/kateway/kateway.log"
        type => "kateway"
    }
    file {
        path => "/var/wd/kateway/panic"
        type => "kateway_panic"
    }
    file {
        path => "/var/wd/kateway/audit/pub_audit.log"
        type => "pubaudit"
    }
    file {
        path => "/var/wd/kateway/audit/sub_audit.log"
        type => "subaudit"
    }
}

filter {
    multiline {
        pattern => "^201" # e,g. this line begins with 2017-01-22
        what => "previous"
        negate => true
    }
}

output {
    if [type] == "pubsub" {
         http {
             http_method => "post"
             url => "http://pub.test.mycorp.com:10191/v1/raw/msgs/cluster/topic"
             workers => 8 # each worker has 25 http connection, totals: 200
             headers => {
                "User-Agent" => "logstash"
             }
         }
    } else if [type] == "pubaudit" {
        kafka {
            bootstrap_servers => "k11003a.mycorp.kfk.com:11003,k11003b.mycorp.kfk.com:11003"
            topic_id => "pubaudit"
        }
    } else if [type] == "subaudit" {
        kafka {
            bootstrap_servers => "k11003a.mycorp.kfk.com:11003,k11003b.mycorp.kfk.com:11003"
            topic_id => "subaudit"
        }
    } else {
        kafka {
            bootstrap_servers => "k11003a.mycorp.kfk.com:11003,k11003b.mycorp.kfk.com:11003"
            topic_id => "pubsub_log"
            metadata_max_age_ms => 300000
            workers => 1
            retries => 1
        }
    }
}`
	this.Ui.Output(strings.TrimSpace(config))
}

func (*Logstash) Synopsis() string {
	return "Sample configuration for logstash"
}

func (this *Logstash) Help() string {
	help := fmt.Sprintf(`
Usage: %s logstash [options]

    %s

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type Topology struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Topology) Run(args []string) (exitCode int) {
	if len(args) == 0 {
		this.Ui.Error("zone required")
		this.Ui.Output(this.Help())
		return 2
	}

	for _, zone := range args {
		ensureZoneValid(zone)

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZonePath(zone)))
		this.displayZoneTopology(zone, zkzone)
	}

	return
}

type brokerHostInfo struct {
	ports    []int
	leadingN int // being leader of how many partitions
}

func newBrokerHostInfo() *brokerHostInfo {
	return &brokerHostInfo{
		ports: make([]int, 0),
	}
}

func (this *brokerHostInfo) addPort(port int) {
	this.ports = append(this.ports, port)
}

func (this *Topology) swallow(err error) {
	if err != nil {
		panic(err)
	}
}

func (this *Topology) displayZoneTopology(zone string, zkzone *zk.ZkZone) {
	this.Ui.Output(zone)

	instances := make(map[string]*brokerHostInfo)
	zkzone.WithinBrokers(func(cluster string, brokers map[string]*zk.Broker) {
		if len(brokers) == 0 {
			return
		}

		for _, broker := range brokers {
			if _, present := instances[broker.Host]; !present {
				instances[broker.Host] = newBrokerHostInfo()
			}
			instances[broker.Host].addPort(broker.Port)
		}

		// find how many partitions a broker is leading
		zkcluster := zkzone.NewCluster(cluster)
		kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
		this.swallow(err)
		topics, err := kfk.Topics()
		this.swallow(err)
		for _, topic := range topics {
			partions, err := kfk.WritablePartitions(topic)
			this.swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				this.swallow(err)
				host, _, err := net.SplitHostPort(leader.Addr())
				this.swallow(err)
				instances[host].leadingN++
			}
		}
	})

	// sort by host ip
	sortedHosts := make([]string, 0, len(instances))
	for host, _ := range instances {
		sortedHosts = append(sortedHosts, host)
	}
	sort.Strings(sortedHosts)

	for _, host := range sortedHosts {
		this.Ui.Output(fmt.Sprintf("\t%s ports %d:%+v leading:%sP",
			color.Green(host),
			len(instances[host].ports),
			instances[host].ports,
			color.Magenta("%d", instances[host].leadingN)))
	}
}

func (*Topology) Synopsis() string {
	return "Print server topology and balancing stats of kafka clusters"
}

func (this *Topology) Help() string {
	help := fmt.Sprintf(`
Usage: %s topology [zone ...]

	Print server topology and balancing stats of kafka clusters
`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>format of output<commit_after>package command

import (
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type Topology struct {
	Ui  cli.Ui
	Cmd string
}

func (this *Topology) Run(args []string) (exitCode int) {
	if len(args) == 0 {
		this.Ui.Error("zone required")
		this.Ui.Output(this.Help())
		return 2
	}

	for _, zone := range args {
		ensureZoneValid(zone)

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZonePath(zone)))
		this.displayZoneTopology(zone, zkzone)
	}

	return
}

type brokerHostInfo struct {
	ports    []int
	leadingN int // being leader of how many partitions
}

func newBrokerHostInfo() *brokerHostInfo {
	return &brokerHostInfo{
		ports: make([]int, 0),
	}
}

func (this *brokerHostInfo) addPort(port int) {
	this.ports = append(this.ports, port)
}

func (this *Topology) swallow(err error) {
	if err != nil {
		panic(err)
	}
}

func (this *Topology) displayZoneTopology(zone string, zkzone *zk.ZkZone) {
	this.Ui.Output(zone)

	instances := make(map[string]*brokerHostInfo)
	zkzone.WithinBrokers(func(cluster string, brokers map[string]*zk.Broker) {
		if len(brokers) == 0 {
			return
		}

		for _, broker := range brokers {
			if _, present := instances[broker.Host]; !present {
				instances[broker.Host] = newBrokerHostInfo()
			}
			instances[broker.Host].addPort(broker.Port)
		}

		// find how many partitions a broker is leading
		zkcluster := zkzone.NewCluster(cluster)
		kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
		this.swallow(err)
		topics, err := kfk.Topics()
		this.swallow(err)
		for _, topic := range topics {
			partions, err := kfk.WritablePartitions(topic)
			this.swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				this.swallow(err)
				host, _, err := net.SplitHostPort(leader.Addr())
				this.swallow(err)
				instances[host].leadingN++
			}
		}
	})

	// sort by host ip
	sortedHosts := make([]string, 0, len(instances))
	for host, _ := range instances {
		sortedHosts = append(sortedHosts, host)
	}
	sort.Strings(sortedHosts)

	for _, host := range sortedHosts {
		this.Ui.Output(fmt.Sprintf("    %s leading: %3dP ports %2d:%+v",
			color.Green("%15s", host),
			instances[host].leadingN,
			len(instances[host].ports),
			instances[host].ports))
	}
}

func (*Topology) Synopsis() string {
	return "Print server topology and balancing stats of kafka clusters"
}

func (this *Topology) Help() string {
	help := fmt.Sprintf(`
Usage: %s topology [zone ...]

	Print server topology and balancing stats of kafka clusters
`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>