def _decorate_fn_or_cls(decorator, fn_or_cls, subclass=False):
  """Decorate a function or class with the given decorator.

  When `fn_or_cls` is a function, applies `decorator` to the function and
  returns the (decorated) result.

  When `fn_or_cls` is a class and the `subclass` parameter is `False`, this will
  replace `fn_or_cls.__init__` with the result of applying `decorator` to it.

  When `fn_or_cls` is a class and `subclass` is `True`, this will subclass the
  class, but with `__init__` defined to be the result of applying `decorator` to
  `fn_or_cls.__init__`. The decorated class has metadata (docstring, name, and
  module information) copied over from `fn_or_cls`. The goal is to provide a
  decorated class the behaves as much like the original as possible, without
  modifying it (for example, inspection operations using `isinstance` or
  `issubclass` should behave the same way as on the original class).

  Args:
    decorator: The decorator to use.
    fn_or_cls: The function or class to decorate.
    subclass: Whether to decorate classes by subclassing. This argument is
      ignored if `fn_or_cls` is not a class.

  Returns:
    The decorated function or class.
  """
  if not inspect.isclass(fn_or_cls):
    return decorator(_ensure_wrappability(fn_or_cls))

  construction_fn = _find_class_construction_fn(fn_or_cls)

  if subclass:
    class DecoratedClass(fn_or_cls):
      __doc__ = fn_or_cls.__doc__
      __module__ = fn_or_cls.__module__
    DecoratedClass.__name__ = fn_or_cls.__name__
    if six.PY3:
      DecoratedClass.__qualname__ = fn_or_cls.__qualname__
    cls = DecoratedClass
  else:
    cls = fn_or_cls

  decorated_fn = decorator(_ensure_wrappability(construction_fn))
  if construction_fn.__name__ == '__new__':
    decorated_fn = staticmethod(decorated_fn)
  setattr(cls, construction_fn.__name__, decorated_fn)
  return cls

def _decorate_fn_or_cls ( decorator , fn_or_cls , subclass = False ) : if not inspect . isclass ( fn_or_cls ) : return decorator ( _ensure_wrappability ( fn_or_cls ) ) construction_fn = _find_class_construction_fn ( fn_or_cls ) if subclass : class DecoratedClass ( fn_or_cls ) : __doc__ = fn_or_cls . __doc__ __module__ = fn_or_cls . __module__ DecoratedClass . __name__ = fn_or_cls . __name__ if six . PY3 : DecoratedClass . __qualname__ = fn_or_cls . __qualname__ cls = DecoratedClass else : cls = fn_or_cls decorated_fn = decorator ( _ensure_wrappability ( construction_fn ) ) if construction_fn . __name__ == '__new__' : decorated_fn = staticmethod ( decorated_fn ) setattr ( cls , construction_fn . __name__ , decorated_fn ) return cls

Decorate a function or class with the given decorator .


def _format_value(value):
  """Returns `value` in a format parseable by `parse_value`, or `None`.

  Simply put, This function ensures that when it returns a string value, the
  following will hold:

      parse_value(_format_value(value)) == value

  Args:
    value: The value to format.

  Returns:
    A string representation of `value` when `value` is literally representable,
    or `None`.
  """
  literal = repr(value)
  try:
    if parse_value(literal) == value:
      return literal
  except SyntaxError:
    pass
  return None

def _format_value ( value ) : literal = repr ( value ) try : if parse_value ( literal ) == value : return literal except SyntaxError : pass return None

Returns value in a format parseable by parse_value or None .


def clear_config(clear_constants=False):
  """Clears the global configuration.

  This clears any parameter values set by `bind_parameter` or `parse_config`, as
  well as the set of dynamically imported modules. It does not remove any
  configurable functions or classes from the registry of configurables.

  Args:
    clear_constants: Whether to clear constants created by `constant`. Defaults
      to False.
  """
  _set_config_is_locked(False)
  _CONFIG.clear()
  _SINGLETONS.clear()
  if clear_constants:
    _CONSTANTS.clear()
  else:
    saved_constants = _CONSTANTS.copy()
    _CONSTANTS.clear()  # Clear then redefine constants (re-adding bindings).
    for name, value in six.iteritems(saved_constants):
      constant(name, value)
  _IMPORTED_MODULES.clear()
  _OPERATIVE_CONFIG.clear()

def clear_config ( clear_constants = False ) : _set_config_is_locked ( False ) _CONFIG . clear ( ) _SINGLETONS . clear ( ) if clear_constants : _CONSTANTS . clear ( ) else : saved_constants = _CONSTANTS . copy ( ) _CONSTANTS . clear ( ) for name , value in six . iteritems ( saved_constants ) : constant ( name , value ) _IMPORTED_MODULES . clear ( ) _OPERATIVE_CONFIG . clear ( )


def bind_parameter(binding_key, value):
  """Binds the parameter value specified by `binding_key` to `value`.

  The `binding_key` argument should either be a string of the form
  `maybe/scope/optional.module.names.configurable_name.parameter_name`, or a
  list or tuple of `(scope, selector, parameter_name)`, where `selector`
  corresponds to `optional.module.names.configurable_name`. Once this function
  has been called, subsequent calls (in the specified scope) to the specified
  configurable function will have `value` supplied to their `parameter_name`
  parameter.

  Example:

      @configurable('fully_connected_network')
      def network_fn(num_layers=5, units_per_layer=1024):
         ...

      def main(_):
        config.bind_parameter('fully_connected_network.num_layers', 3)
        network_fn()  # Called with num_layers == 3, not the default of 5.

  Args:
    binding_key: The parameter whose value should be set. This can either be a
      string, or a tuple of the form `(scope, selector, parameter)`.
    value: The desired value.

  Raises:
    RuntimeError: If the config is locked.
    ValueError: If no function can be found matching the configurable name
      specified by `binding_key`, or if the specified parameter name is
      blacklisted or not in the function's whitelist (if present).
  """
  if config_is_locked():
    raise RuntimeError('Attempted to modify locked Gin config.')

  pbk = ParsedBindingKey(binding_key)
  fn_dict = _CONFIG.setdefault(pbk.config_key, {})
  fn_dict[pbk.arg_name] = value

Binds the parameter value specified by binding_key to value .


def query_parameter(binding_key):
  """Returns the currently bound value to the specified `binding_key`.

  The `binding_key` argument should look like
  'maybe/some/scope/maybe.moduels.configurable_name.parameter_name'. Note that
  this will not include default parameters.

  Args:
    binding_key: The parameter whose value should be set.

  Returns:
    The value bound to the configurable/parameter combination given in
    `binding_key`.

  Raises:
    ValueError: If no function can be found matching the configurable name
      specified by `biding_key`, or if the specified parameter name is
      blacklisted or not in the function's whitelist (if present) or if there is
      no value bound for the queried parameter or configurable.
  """
  pbk = ParsedBindingKey(binding_key)
  if pbk.config_key not in _CONFIG:
    err_str = "Configurable '{}' has no bound parameters."
    raise ValueError(err_str.format(pbk.given_selector))
  if pbk.arg_name not in _CONFIG[pbk.config_key]:
    err_str = "Configurable '{}' has no value bound for parameter '{}'."
    raise ValueError(err_str.format(pbk.given_selector, pbk.arg_name))
  return _CONFIG[pbk.config_key][pbk.arg_name]

Returns the currently bound value to the specified binding_key .


def _might_have_parameter(fn_or_cls, arg_name):
  """Returns True if `arg_name` might be a valid parameter for `fn_or_cls`.

  Specifically, this means that `fn_or_cls` either has a parameter named
  `arg_name`, or has a `**kwargs` parameter.

  Args:
    fn_or_cls: The function or class to check.
    arg_name: The name fo the parameter.

  Returns:
    Whether `arg_name` might be a valid argument of `fn`.
  """
  if inspect.isclass(fn_or_cls):
    fn = _find_class_construction_fn(fn_or_cls)
  else:
    fn = fn_or_cls

  while hasattr(fn, '__wrapped__'):
    fn = fn.__wrapped__
  arg_spec = _get_cached_arg_spec(fn)
  if six.PY3:
    if arg_spec.varkw:
      return True
    return arg_name in arg_spec.args or arg_name in arg_spec.kwonlyargs
  else:
    if arg_spec.keywords:
      return True
    return arg_name in arg_spec.args

def _might_have_parameter ( fn_or_cls , arg_name ) : if inspect . isclass ( fn_or_cls ) : fn = _find_class_construction_fn ( fn_or_cls ) else : fn = fn_or_cls while hasattr ( fn , '__wrapped__' ) : fn = fn . __wrapped__ arg_spec = _get_cached_arg_spec ( fn ) if six . PY3 : if arg_spec . varkw : return True return arg_name in arg_spec . args or arg_name in arg_spec . kwonlyargs else : if arg_spec . keywords : return True return arg_name in arg_spec . args

Returns True if arg_name might be a valid parameter for fn_or_cls .


def _get_cached_arg_spec(fn):
  """Gets cached argspec for `fn`."""

  arg_spec = _ARG_SPEC_CACHE.get(fn)
  if arg_spec is None:
    arg_spec_fn = inspect.getfullargspec if six.PY3 else inspect.getargspec
    try:
      arg_spec = arg_spec_fn(fn)
    except TypeError:
      # `fn` might be a callable object.
      arg_spec = arg_spec_fn(fn.__call__)
    _ARG_SPEC_CACHE[fn] = arg_spec
  return arg_spec

def _get_cached_arg_spec ( fn ) : arg_spec = _ARG_SPEC_CACHE . get ( fn ) if arg_spec is None : arg_spec_fn = inspect . getfullargspec if six . PY3 else inspect . getargspec try : arg_spec = arg_spec_fn ( fn ) except TypeError : arg_spec = arg_spec_fn ( fn . __call__ ) _ARG_SPEC_CACHE [ fn ] = arg_spec return arg_spec


def _get_supplied_positional_parameter_names(fn, args):
  """Returns the names of the supplied arguments to the given function."""
  arg_spec = _get_cached_arg_spec(fn)
  # May be shorter than len(args) if args contains vararg (*args) arguments.
  return arg_spec.args[:len(args)]

Returns the names of the supplied arguments to the given function .


def _get_all_positional_parameter_names(fn):
  """Returns the names of all positional arguments to the given function."""
  arg_spec = _get_cached_arg_spec(fn)
  args = arg_spec.args
  if arg_spec.defaults:
    args = args[:-len(arg_spec.defaults)]
  return args

def _get_all_positional_parameter_names ( fn ) : arg_spec = _get_cached_arg_spec ( fn ) args = arg_spec . args if arg_spec . defaults : args = args [ : - len ( arg_spec . defaults ) ] return args

Returns the names of all positional arguments to the given function .


def _get_default_configurable_parameter_values(fn, whitelist, blacklist):
  """Retrieve all default values for configurable parameters of a function.

  Any parameters included in the supplied blacklist, or not included in the
  supplied whitelist, are excluded.

  Args:
    fn: The function whose parameter values should be retrieved.
    whitelist: The whitelist (or `None`) associated with the function.
    blacklist: The blacklist (or `None`) associated with the function.

  Returns:
    A dictionary mapping configurable parameter names to their default values.
  """
  arg_vals = _ARG_DEFAULTS_CACHE.get(fn)
  if arg_vals is not None:
    return arg_vals.copy()

  # First, grab any default values not captured in the kwargs var.
  arg_spec = _get_cached_arg_spec(fn)
  if arg_spec.defaults:
    default_kwarg_names = arg_spec.args[-len(arg_spec.defaults):]
    arg_vals = dict(zip(default_kwarg_names, arg_spec.defaults))
  else:
    arg_vals = {}

  if six.PY3 and arg_spec.kwonlydefaults:
    arg_vals.update(arg_spec.kwonlydefaults)

  # Now, eliminate keywords that are blacklisted, or aren't whitelisted (if
  # there's a whitelist), or aren't representable as a literal value.
  for k in list(six.iterkeys(arg_vals)):
    whitelist_fail = whitelist and k not in whitelist
    blacklist_fail = blacklist and k in blacklist
    representable = _is_literally_representable(arg_vals[k])
    if whitelist_fail or blacklist_fail or not representable:
      del arg_vals[k]

  _ARG_DEFAULTS_CACHE[fn] = arg_vals
  return arg_vals.copy()

Retrieve all default values for configurable parameters of a function .


def config_scope(name_or_scope):
  """Opens a new configuration scope.

  Provides a context manager that opens a new explicit configuration
  scope. Explicit configuration scopes restrict parameter bindings to only
  certain sections of code that run within the scope. Scopes can be nested to
  arbitrary depth; any configurable functions called within a scope inherit
  parameters defined by higher level scopes.

  For example, suppose a function named `preprocess_images` is called in two
  places in a codebase: Once when loading data for a training task, and once
  when loading data for an evaluation task:

      def load_training_data():
        ...
        with gin.config_scope('train'):
          images = preprocess_images(images)
        ...

      def load_eval_data():
        ...
        with gin.config_scope('eval'):
          images = preprocess_images(images)
        ...

  By using a `config_scope` to wrap each invocation of `preprocess_images` as
  above, it is possible to use Gin to supply specific parameters to each. Here
  is a possible configuration for the above example:

      preprocess_images.crop_size = [64, 64]
      preprocess_images.normalize_image = True

      train/preprocess_images.crop_location = 'random'
      train/preprocess_images.random_flip_lr = True

      eval/preprocess_images.crop_location = 'center'

  The `crop_size` and `normalize_image` parameters above will be shared by both
  the `train` and `eval` invocations; only `train` will receive
  `random_flip_lr`, and the two invocations receive different values for
  `crop_location`.

  Passing `None` or `''` to `config_scope` will temporarily clear all currently
  active scopes (within the `with` block; they will be restored afterwards).

  Args:
    name_or_scope: A name for the config scope, or an existing scope (e.g.,
      captured from `with gin.config_scope(...) as scope`), or `None` to clear
      currently active scopes.

  Raises:
    ValueError: If `name_or_scope` is not a list, string, or None.

  Yields:
    The resulting config scope (a list of all active scope names, ordered from
    outermost to innermost).
  """
  try:
    valid_value = True
    if isinstance(name_or_scope, list):
      new_scope = name_or_scope
    elif name_or_scope and isinstance(name_or_scope, six.string_types):
      new_scope = current_scope()  # Returns a copy.
      new_scope.extend(name_or_scope.split('/'))
    else:
      valid_value = name_or_scope in (None, '')
      new_scope = []

    # Append new_scope first. It will be popped in the finally block if an
    # exception is raised below.
    _ACTIVE_SCOPES.append(new_scope)

    scopes_are_valid = map(config_parser.MODULE_RE.match, new_scope)
    if not valid_value or not all(scopes_are_valid):
      err_str = 'Invalid value for `name_or_scope`: {}.'
      raise ValueError(err_str.format(name_or_scope))

    yield new_scope
  finally:
    _ACTIVE_SCOPES.pop()


def configurable(name_or_fn=None, module=None, whitelist=None, blacklist=None):
  """Decorator to make a function or class configurable.

  This decorator registers the decorated function/class as configurable, which
  allows its parameters to be supplied from the global configuration (i.e., set
  through `bind_parameter` or `parse_config`). The decorated function is
  associated with a name in the global configuration, which by default is simply
  the name of the function or class, but can be specified explicitly to avoid
  naming collisions or improve clarity.

  If some parameters should not be configurable, they can be specified in
  `blacklist`. If only a restricted set of parameters should be configurable,
  they can be specified in `whitelist`.

  The decorator can be used without any parameters as follows:

      @config.configurable
      def some_configurable_function(param1, param2='a default value'):
        ...

  In this case, the function is associated with the name
  `'some_configurable_function'` in the global configuration, and both `param1`
  and `param2` are configurable.

  The decorator can be supplied with parameters to specify the configurable name
  or supply a whitelist/blacklist:

      @config.configurable('explicit_configurable_name', whitelist='param2')
      def some_configurable_function(param1, param2='a default value'):
        ...

  In this case, the configurable is associated with the name
  `'explicit_configurable_name'` in the global configuration, and only `param2`
  is configurable.

  Classes can be decorated as well, in which case parameters of their
  constructors are made configurable:

      @config.configurable
      class SomeClass(object):
        def __init__(self, param1, param2='a default value'):
          ...

  In this case, the name of the configurable is `'SomeClass'`, and both `param1`
  and `param2` are configurable.

  Args:
    name_or_fn: A name for this configurable, or a function to decorate (in
      which case the name will be taken from that function). If not set,
      defaults to the name of the function/class that is being made
      configurable. If a name is provided, it may also include module components
      to be used for disambiguation (these will be appended to any components
      explicitly specified by `module`).
    module: The module to associate with the configurable, to help handle naming
      collisions. By default, the module of the function or class being made
      configurable will be used (if no module is specified as part of the name).
    whitelist: A whitelisted set of kwargs that should be configurable. All
      other kwargs will not be configurable. Only one of `whitelist` or
      `blacklist` should be specified.
    blacklist: A blacklisted set of kwargs that should not be configurable. All
      other kwargs will be configurable. Only one of `whitelist` or `blacklist`
      should be specified.

  Returns:
    When used with no parameters (or with a function/class supplied as the first
    parameter), it returns the decorated function or class. When used with
    parameters, it returns a function that can be applied to decorate the target
    function or class.
  """
  decoration_target = None
  if callable(name_or_fn):
    decoration_target = name_or_fn
    name = None
  else:
    name = name_or_fn

  def perform_decoration(fn_or_cls):
    return _make_configurable(fn_or_cls, name, module, whitelist, blacklist)

  if decoration_target:
    return perform_decoration(decoration_target)
  return perform_decoration

def configurable ( name_or_fn = None , module = None , whitelist = None , blacklist = None ) : decoration_target = None if callable ( name_or_fn ) : decoration_target = name_or_fn name = None else : name = name_or_fn def perform_decoration ( fn_or_cls ) : return _make_configurable ( fn_or_cls , name , module , whitelist , blacklist ) if decoration_target : return perform_decoration ( decoration_target ) return perform_decoration

Decorator to make a function or class configurable .


def operative_config_str(max_line_length=80, continuation_indent=4):
  """Retrieve the "operative" configuration as a config string.

  The operative configuration consists of all parameter values used by
  configurable functions that are actually called during execution of the
  current program. Parameters associated with configurable functions that are
  not called (and so can have no effect on program execution) won't be included.

  The goal of the function is to return a config that captures the full set of
  relevant configurable "hyperparameters" used by a program. As such, the
  returned configuration will include the default values of arguments from
  configurable functions (as long as the arguments aren't blacklisted or missing
  from a supplied whitelist), as well as any parameter values overridden via
  `bind_parameter` or through `parse_config`.

  Any parameters that can't be represented as literals (capable of being parsed
  by `parse_config`) are excluded. The resulting config string is sorted
  lexicographically and grouped by configurable name.

  Args:
    max_line_length: A (soft) constraint on the maximum length of a line in the
      formatted string. Large nested structures will be split across lines, but
      e.g. long strings won't be split into a concatenation of shorter strings.
    continuation_indent: The indentation for continued lines.

  Returns:
    A config string capturing all parameter values used by the current program.
  """
  def format_binding(key, value):
    """Pretty print the given key/value pair."""
    formatted_val = pprint.pformat(
        value, width=(max_line_length - continuation_indent))
    formatted_val_lines = formatted_val.split('\n')
    if (len(formatted_val_lines) == 1 and
        len(key + formatted_val) <= max_line_length):
      output = '{} = {}'.format(key, formatted_val)
    else:
      indented_formatted_val = '\n'.join(
          [' ' * continuation_indent + line for line in formatted_val_lines])
      output = '{} = \\\n{}'.format(key, indented_formatted_val)
    return output

  def sort_key(key_tuple):
    """Sort configurable selector/innermost scopes, ignoring case."""
    scope, selector = key_tuple[0]
    parts = selector.lower().split('.')[::-1] + scope.lower().split('/')[::-1]
    return '/'.join(parts)

  # Build the output as an array of formatted Gin statements. Each statement may
  # span multiple lines. Imports are first, followed by macros, and finally all
  # other bindings sorted in alphabetical order by configurable name.
  formatted_statements = [
      'import {}'.format(module) for module in sorted(_IMPORTED_MODULES)
  ]
  if formatted_statements:
    formatted_statements.append('')

  macros = {}
  for (scope, selector), config in six.iteritems(_OPERATIVE_CONFIG):
    if _REGISTRY[selector].fn_or_cls == macro:
      macros[scope, selector] = config
  if macros:
    formatted_statements.append('# Macros:')
    formatted_statements.append('# ' + '=' * (max_line_length - 2))
  for (name, _), config in sorted(macros.items(), key=sort_key):
    binding = format_binding(name, config['value'])
    formatted_statements.append(binding)
  if macros:
    formatted_statements.append('')

  sorted_items = sorted(_OPERATIVE_CONFIG.items(), key=sort_key)
  for (scope, selector), config in sorted_items:
    configurable_ = _REGISTRY[selector]

    fn = configurable_.fn_or_cls
    if fn == macro or fn == _retrieve_constant:
      continue

    minimal_selector = _REGISTRY.minimal_selector(configurable_.selector)
    scoped_selector = (scope + '/' if scope else '') + minimal_selector
    parameters = [(k, v) for k, v in six.iteritems(config)
                  if _is_literally_representable(v)]
    formatted_statements.append('# Parameters for {}:'.format(scoped_selector))
    formatted_statements.append('# ' + '=' * (max_line_length - 2))
    for arg, val in sorted(parameters):
      binding = format_binding('{}.{}'.format(scoped_selector, arg), val)
      formatted_statements.append(binding)
    if not parameters:
      formatted_statements.append('# None.')
    formatted_statements.append('')

  return '\n'.join(formatted_statements)

def
operative_config_str
(
max_line_length
=
80
,
continuation_indent
=
4
)
:
def
format_binding
(
key
,
value
)
:
formatted_val
=
pprint
.
pformat
(
value
,
width
=
(
max_line_length
continuation_indent
)
)
formatted_val_lines
=
formatted_val
.
split
(
'\n'
)
if
(
len
(
formatted_val_lines
)
==
1
and
len
(
key
+
formatted_val
)
<=
max_line_length
)
:
output
=
'{} = {}'
.
format
(
key
,
formatted_val
)
else
:
indented_formatted_val
=
'\n'
.
join
(
[
' '
*
continuation_indent
+
line
for
line
in
formatted_val_lines
]
)
output
=
'{} = \\\n{}'
.
format
(
key
,
indented_formatted_val
)
return
output
def
sort_key
(
key_tuple
)
:
scope
,
selector
=
key_tuple
[
0
]
parts
=
selector
.
lower
(
)
.
split
(
'.'
)
[
:
:
1
]
+
scope
.
lower
(
)
.
split
(
'/'
)
[
:
:
1
]
return
'/'
.
join
(
parts
)
formatted_statements
=
[
'import {}'
.
format
(
module
)
for
module
in
sorted
(
_IMPORTED_MODULES
)
]
if
formatted_statements
:
formatted_statements
.
append
(
''
)
macros
=
{
}
for
(
scope
,
selector
)
,
config
in
six
.
iteritems
(
_OPERATIVE_CONFIG
)
:
if
_REGISTRY
[
selector
]
.
fn_or_cls
==
macro
:
macros
[
scope
,
selector
]
=
config
if
macros
:
formatted_statements
.
append
(
'# Macros:'
)
formatted_statements
.
append
(
'# '
+
'='
*
(
max_line_length
2
)
)
for
(
name
,
_
)
,
config
in
sorted
(
macros
.
items
(
)
,
key
=
sort_key
)
:
binding
=
format_binding
(
name
,
config
[
'value'
]
)
formatted_statements
.
append
(
binding
)
if
macros
:
formatted_statements
.
append
(
''
)
sorted_items
=
sorted
(
_OPERATIVE_CONFIG
.
items
(
)
,
key
=
sort_key
)
for
(
scope
,
selector
)
,
config
in
sorted_items
:
configurable_
=
_REGISTRY
[
selector
]
fn
=
configurable_
.
fn_or_cls
if
fn
==
macro
or
fn
==
_retrieve_constant
:
continue
minimal_selector
=
_REGISTRY
.
minimal_selector
(
configurable_
.
selector
)
scoped_selector
=
(
scope
+
'/'
if
scope
else
''
)
+
minimal_selector
parameters
=
[
(
k
,
v
)
for
k
,
v
in
six
.
iteritems
(
config
)
if
_is_literally_representable
(
v
)
]
formatted_statements
.
append
(
'# Parameters for {}:'
.
format
(
scoped_selector
)
)
formatted_statements
.
append
(
'# '
+
'='
*
(
max_line_length
2
)
)
for
arg
,
val
in
sorted
(
parameters
)
:
binding
=
format_binding
(
'{}.{}'
.
format
(
scoped_selector
,
arg
)
,
val
)
formatted_statements
.
append
(
binding
)
if
not
parameters
:
formatted_statements
.
append
(
'# None.'
)
formatted_statements
.
append
(
''
)
return
'\n'
.
join
(
formatted_statements
)

Retrieve the operative configuration as a config string .


def parse_config(bindings, skip_unknown=False):
  """Parse a file, string, or list of strings containing parameter bindings.

  Parses parameter binding strings to set up the global configuration.  Once
  `parse_config` has been called, any calls to configurable functions will have
  parameter values set according to the values specified by the parameter
  bindings in `bindings`.

  An individual parameter binding has the format

      maybe/some/scopes/configurable_name.parameter_name = value

  Multiple binding strings can be passed either in the form of a file-like
  object supporting the `readline` method, a single string with each individual
  parameter binding separated by a newline, or as a list of individual parameter
  binding strings.

  Any Python literal (lists, tuples, dicts, strings, etc.) is acceptable to the
  right of the equals sign, and follows standard Python rules for line
  continuation. Additionally, a value starting with '@' is interpreted as a
  (possibly scoped) reference to another configurable function, in which case
  this value is replaced by a reference to that function. If the value
  furthermore ends in `()` (e.g., `@configurable_name()`), then the value
  returned when calling the function is used (it will be called *just before*
  the function consuming the output is called).

  See the module documentation for a more detailed description of scoping
  mechanisms and a complete example.

  Reading from a file could be done as follows:

      with open('/path/to/file.config') as bindings:
        gin.parse_config(bindings)

  Passing a newline separated string of parameter bindings might look like:

      bindings = '''
          my_class.param_one = 'asdf'
          my_class_param_two = 9.7
      '''
      gin.parse_config(bindings)

  Alternatively, one can declare a list of parameter bindings and pass it in:

      bindings = [
          'my_class.param_one = "asdf"',
          'my_class.param_two = 9.7',
      ]
      gin.parse_config(bindings)

  Can skip unknown configurables. For example, if no module containing a
  'training' configurable was imported, errors can be avoided by specifying
  `skip_unknown=True`:

      bindings = [
          'my_class.param_one = "asdf"',
          'my_class.param_two = 9.7',
          'training.learning_rate = 0.1',
      ]
      gin.parse_config(bindings, skip_unknown=True)

  Args:
    bindings: A file-like object supporting the readline method, a newline
      separated string of parameter bindings, or a list of individual parameter
      binding strings.
    skip_unknown: A boolean indicating whether unknown configurables and imports
      should be skipped (instead of causing an error). Configurable references
      to unknown configurables will cause errors if they are present in a
      binding that is not itself skipped due to an unknown configurable. This
      can also be a list of configurable names: any unknown configurables that
      do not match an item in the list will still cause errors. Note that
      bindings for known configurables will always be parsed.
  """
  if isinstance(bindings, (list, tuple)):
    bindings = '\n'.join(bindings)

  _validate_skip_unknown(skip_unknown)
  if isinstance(skip_unknown, (list, tuple)):
    skip_unknown = set(skip_unknown)

  parser = config_parser.ConfigParser(bindings, ParserDelegate(skip_unknown))
  for statement in parser:
    if isinstance(statement, config_parser.BindingStatement):
      scope, selector, arg_name, value, location = statement
      if not arg_name:
        macro_name = '{}/{}'.format(scope, selector) if scope else selector
        with utils.try_with_location(location):
          bind_parameter((macro_name, 'gin.macro', 'value'), value)
        continue
      if not _should_skip(selector, skip_unknown):
        with utils.try_with_location(location):
          bind_parameter((scope, selector, arg_name), value)
    elif isinstance(statement, config_parser.ImportStatement):
      if skip_unknown:
        try:
          __import__(statement.module)
          _IMPORTED_MODULES.add(statement.module)
        except ImportError:
          log_str = 'Skipping import of unknown module `%s` (skip_unknown=%r).'
          logging.info(log_str, statement.module, skip_unknown)
      else:
        with utils.try_with_location(statement.location):
          __import__(statement.module)
        _IMPORTED_MODULES.add(statement.module)
    elif isinstance(statement, config_parser.IncludeStatement):
      with utils.try_with_location(statement.location):
        parse_config_file(statement.filename, skip_unknown)
    else:
      raise AssertionError('Unrecognized statement type {}.'.format(statement))

Parse a file string or list of strings containing parameter bindings .


def register_file_reader(*args):
  """Register a file reader for use in parse_config_file.

  Registered file readers will be used to try reading files passed to
  `parse_config_file`. All file readers (beginning with the default `open`) will
  be tried until one of them succeeds at opening the file.

  This function may also be be used used as a decorator. For example:

      @register_file_reader(IOError)
      def exotic_data_source(filename):
        ...

  Args:
    *args: (When used as a decorator, only the existence check is supplied.)
      - file_reader_fn: The file reader function to register. This should be a
        function that can be used as a context manager to open a file and
        provide a file-like object, similar to Python's built-in `open`.
      - is_readable_fn: A function taking the file path and returning a boolean
        indicating whether the file can be read by `file_reader_fn`.

  Returns:
    `None`, or when used as a decorator, a function that will perform the
    registration using the supplied readability predicate.
  """
  def do_registration(file_reader_fn, is_readable_fn):
    if file_reader_fn not in list(zip(*_FILE_READERS))[0]:
      _FILE_READERS.append((file_reader_fn, is_readable_fn))

  if len(args) == 1:  # It's a decorator.
    return functools.partial(do_registration, is_readable_fn=args[0])
  elif len(args) == 2:
    do_registration(*args)
  else:  # 0 or > 2 arguments supplied.
    err_str = 'register_file_reader() takes 1 or 2 arguments ({} given)'
    raise TypeError(err_str.format(len(args)))

Register a file reader for use in parse_config_file .


def parse_config_file(config_file, skip_unknown=False):
  """Parse a Gin config file.

  Args:
    config_file: The path to a Gin config file.
    skip_unknown: A boolean indicating whether unknown configurables and imports
      should be skipped instead of causing errors (alternatively a list of
      configurable names to skip if unknown). See `parse_config` for additional
      details.

  Raises:
    IOError: If `config_file` cannot be read using any register file reader.
  """
  for reader, existence_check in _FILE_READERS:
    if existence_check(config_file):
      with reader(config_file) as f:
        parse_config(f, skip_unknown=skip_unknown)
        return
  raise IOError('Unable to open file: {}'.format(config_file))

def
parse_config_file
(
config_file
,
skip_unknown
=
False
)
:
for
reader
,
existence_check
in
_FILE_READERS
:
if
existence_check
(
config_file
)
:
with
reader
(
config_file
)
as
f
:
parse_config
(
f
,
skip_unknown
=
skip_unknown
)
return
raise
IOError
(
'Unable to open file: {}'
.
format
(
config_file
)
)


def parse_config_files_and_bindings(config_files,
                                    bindings,
                                    finalize_config=True,
                                    skip_unknown=False):
  """Parse a list of config files followed by extra Gin bindings.

  This function is equivalent to:

      for config_file in config_files:
        gin.parse_config_file(config_file, skip_configurables)
      gin.parse_config(bindings, skip_configurables)
      if finalize_config:
        gin.finalize()

  Args:
    config_files: A list of paths to the Gin config files.
    bindings: A list of individual parameter binding strings.
    finalize_config: Whether to finalize the config after parsing and binding
      (defaults to True).
    skip_unknown: A boolean indicating whether unknown configurables and imports
      should be skipped instead of causing errors (alternatively a list of
      configurable names to skip if unknown). See `parse_config` for additional
      details.
  """
  if config_files is None:
    config_files = []
  if bindings is None:
    bindings = ''
  for config_file in config_files:
    parse_config_file(config_file, skip_unknown)
  parse_config(bindings, skip_unknown)
  if finalize_config:
    finalize()

def parse_config_files_and_bindings ( config_files , bindings , finalize_config = True , skip_unknown = False ) : if config_files is None : config_files = [ ] if bindings is None : bindings = '' for config_file in config_files : parse_config_file ( config_file , skip_unknown ) parse_config ( bindings , skip_unknown ) if finalize_config : finalize ( )

Parse a list of config files followed by extra Gin bindings .


def parse_value(value):
  """Parse and return a single Gin value."""
  if not isinstance(value, six.string_types):
    raise ValueError('value ({}) should be a string type.'.format(value))
  return config_parser.ConfigParser(value, ParserDelegate()).parse_value()

def
parse_value
(
value
)
:
if
not
isinstance
(
value
,
six
.
string_types
)
:
raise
ValueError
(
'value ({}) should be a string type.'
.
format
(
value
)
)
return
config_parser
.
ConfigParser
(
value
,
ParserDelegate
(
)
)
.
parse_value
(
)


def finalize():
  """A function that should be called after parsing all Gin config files.

  Calling this function allows registered "finalize hooks" to inspect (and
  potentially modify) the Gin config, to provide additional functionality. Hooks
  should not modify the configuration object they receive directly; instead,
  they should return a dictionary mapping Gin binding keys to (new or updated)
  values. This way, all hooks see the config as originally parsed.

  Raises:
    RuntimeError: If the config is already locked.
    ValueError: If two or more hooks attempt to modify or introduce bindings for
      the same key. Since it is difficult to control the order in which hooks
      are registered, allowing this could yield unpredictable behavior.
  """
  if config_is_locked():
    raise RuntimeError('Finalize called twice (config already locked).')

  bindings = {}
  for hook in _FINALIZE_HOOKS:
    new_bindings = hook(_CONFIG)
    if new_bindings is not None:
      for key, value in six.iteritems(new_bindings):
        pbk = ParsedBindingKey(key)
        if pbk in bindings:
          err_str = 'Received conflicting updates when running {}.'
          raise ValueError(err_str.format(hook))
        bindings[pbk] = value

  for pbk, value in six.iteritems(bindings):
    bind_parameter(pbk, value)

  _set_config_is_locked(True)

A function that should be called after parsing all Gin config files .


def _iterate_flattened_values(value):
  """Provides an iterator over all values in a nested structure."""
  if isinstance(value, six.string_types):
    yield value
    return

  if isinstance(value, collections.Mapping):
    value = collections.ValuesView(value)

  if isinstance(value, collections.Iterable):
    for nested_value in value:
      for nested_nested_value in _iterate_flattened_values(nested_value):
        yield nested_nested_value

  yield value

def _iterate_flattened_values ( value ) : if isinstance ( value , six . string_types ) : yield value return if isinstance ( value , collections . Mapping ) : value = collections . ValuesView ( value ) if isinstance ( value , collections . Iterable ) : for nested_value in value : for nested_nested_value in _iterate_flattened_values ( nested_value ) : yield nested_nested_value yield value

Provides an iterator over all values in a nested structure .


def iterate_references(config, to=None):
  """Provides an iterator over references in the given config.

  Args:
    config: A dictionary mapping scoped configurable names to argument bindings.
    to: If supplied, only yield references whose `configurable_fn` matches `to`.

  Yields:
    `ConfigurableReference` instances within `config`, maybe restricted to those
    matching the `to` parameter if it is supplied.
  """
  for value in _iterate_flattened_values(config):
    if isinstance(value, ConfigurableReference):
      if to is None or value.configurable.fn_or_cls == to:
        yield value

def iterate_references ( config , to = None ) : for value in _iterate_flattened_values ( config ) : if isinstance ( value , ConfigurableReference ) : if to is None or value . configurable . fn_or_cls == to : yield value

Provides an iterator over references in the given config .


def constant(name, value):
  """Creates a constant that can be referenced from gin config files.

  After calling this function in Python, the constant can be referenced from
  within a Gin config file using the macro syntax. For example, in Python:

      gin.constant('THE_ANSWER', 42)

  Then, in a Gin config file:

      meaning.of_life = %THE_ANSWER

  Note that any Python object can be used as the value of a constant (including
  objects not representable as Gin literals). Values will be stored until
  program termination in a Gin-internal dictionary, so avoid creating constants
  with values that should have a limited lifetime.

  Optionally, a disambiguating module may be prefixed onto the constant
  name. For instance:

      gin.constant('some.modules.PI', 3.14159)

  Args:
    name: The name of the constant, possibly prepended by one or more
      disambiguating module components separated by periods. An macro with this
      name (including the modules) will be created.
    value: The value of the constant. This can be anything (including objects
      not representable as Gin literals). The value will be stored and returned
      whenever the constant is referenced.

  Raises:
    ValueError: If the constant's selector is invalid, or a constant with the
      given selector already exists.
  """
  if not config_parser.MODULE_RE.match(name):
    raise ValueError("Invalid constant selector '{}'.".format(name))

  if _CONSTANTS.matching_selectors(name):
    err_str = "Constants matching selector '{}' already exist ({})."
    raise ValueError(err_str.format(name, _CONSTANTS.matching_selectors(name)))

  _CONSTANTS[name] = value

Creates a constant that can be referenced from gin config files .


def constants_from_enum(cls, module=None):
  """Decorator for an enum class that generates Gin constants from values.

  Generated constants have format `module.ClassName.ENUM_VALUE`. The module
  name is optional when using the constant.

  Args:
    cls: Class type.
    module: The module to associate with the constants, to help handle naming
      collisions. If `None`, `cls.__module__` will be used.

  Returns:
    Class type (identity function).

  Raises:
    TypeError: When applied to a non-enum class.
  """
  if not issubclass(cls, enum.Enum):
    raise TypeError("Class '{}' is not subclass of enum.".format(cls.__name__))

  if module is None:
    module = cls.__module__
  for value in cls:
    constant('{}.{}'.format(module, str(value)), value)
  return cls

def
constants_from_enum
(
cls
,
module
=
)
:
if
not
issubclass
(
cls
,
enum
.
Enum
)
:
raise
TypeError
(
"Class '{}' is not subclass of enum."
.
format
(
cls
.
__name__
)
)
if
module
is
:
module
=
cls
.
__module__
for
value
in
cls
:
constant
(
'{}.{}'
.
format
(
module
,
str
(
value
)
)
,
value
)
return
cls

Decorator for an enum class that generates Gin constants from values .


def matching_selectors(self, partial_selector):
    """Retrieves all selectors matching `partial_selector`.

    For instance, if "one.a.b" and "two.a.b" are stored in a `SelectorMap`, both
    `matching_selectors('b')` and `matching_selectors('a.b')` will return them.

    In the event that `partial_selector` exactly matches an existing complete
    selector, only that complete selector is returned. For instance, if
    "a.b.c.d" and "c.d" are stored, `matching_selectors('c.d')` will return only
    `['c.d']`, while `matching_selectors('d')` will return both.

    Args:
      partial_selector: The partial selector to find matches for.

    Returns:
      A list of selectors matching `partial_selector`.
    """
    if partial_selector in self._selector_map:
      return [partial_selector]

    selector_components = partial_selector.split('.')
    node = self._selector_tree

    for component in reversed(selector_components):
      if component not in node:
        return []
      node = node[component]

    selectors = []
    dfs_stack = [node]
    while dfs_stack:
      node = dfs_stack.pop().copy()
      selector = node.pop(_TERMINAL_KEY, None)
      dfs_stack.extend(node.values())
      if selector:
        selectors.append(selector)

    return selectors

def matching_selectors ( self , partial_selector ) : if partial_selector in self . _selector_map : return [ partial_selector ] selector_components = partial_selector . split ( '.' ) node = self . _selector_tree for component in reversed ( selector_components ) : if component not in node : return [ ] node = node [ component ] selectors = [ ] dfs_stack = [ node ] while dfs_stack : node = dfs_stack . pop ( ) . copy ( ) selector = node . pop ( _TERMINAL_KEY , None ) dfs_stack . extend ( node . values ( ) ) if selector : selectors . append ( selector ) return selectors

Retrieves all selectors matching partial_selector .


def get_all_matches(self, partial_selector):
    """Returns all values matching `partial_selector` as a list."""
    matching_selectors = self.matching_selectors(partial_selector)
    return [self._selector_map[selector] for selector in matching_selectors]

def get_all_matches ( self , partial_selector ) : matching_selectors = self . matching_selectors ( partial_selector ) return [ self . _selector_map [ selector ] for selector in matching_selectors ]

Returns all values matching partial_selector as a list .


def minimal_selector(self, complete_selector):
    """Returns the minimal selector that uniquely matches `complete_selector`.

    Args:
      complete_selector: A complete selector stored in the map.

    Returns:
      A partial selector that unambiguously matches `complete_selector`.

    Raises:
      KeyError: If `complete_selector` is not in the map.
    """
    if complete_selector not in self._selector_map:
      raise KeyError("No value with selector '{}'.".format(complete_selector))

    selector_components = complete_selector.split('.')
    node = self._selector_tree

    start = None
    for i, component in enumerate(reversed(selector_components)):
      if len(node) == 1:
        if start is None:
          start = -i  # Negative index, since we're iterating in reverse.
      else:
        start = None
      node = node[component]

    if len(node) > 1:  # The selector is a substring of another selector.
      return complete_selector
    return '.'.join(selector_components[start:])

def
minimal_selector
(
self
,
complete_selector
)
:
if
complete_selector
not
in
self
.
_selector_map
:
raise
KeyError
(
"No value with selector '{}'."
.
format
(
complete_selector
)
)
selector_components
=
complete_selector
.
split
(
'.'
)
node
=
self
.
_selector_tree
start
=
for
i
,
component
in
enumerate
(
reversed
(
selector_components
)
)
:
if
len
(
node
)
==
1
:
if
start
is
:
start
=
i
else
:
start
=
node
=
node
[
component
]
if
len
(
node
)
>
1
:
return
complete_selector
return
'.'
.
join
(
selector_components
[
start
:
]
)

Returns the minimal selector that uniquely matches complete_selector .


def sp_search_query(query):
    """Translate a Mopidy search query to a Spotify search query"""

    result = []

    for (field, values) in query.items():
        field = SEARCH_FIELD_MAP.get(field, field)
        if field is None:
            continue

        for value in values:
            if field == 'year':
                value = _transform_year(value)
                if value is not None:
                    result.append('%s:%d' % (field, value))
            elif field == 'any':
                result.append('"%s"' % value)
            else:
                result.append('%s:"%s"' % (field, value))

    return ' '.join(result)

def
sp_search_query
(
query
)
:
result
=
[
]
for
(
field
,
values
)
in
query
.
items
(
)
:
field
=
SEARCH_FIELD_MAP
.
get
(
field
,
field
)
if
field
is
:
continue
for
value
in
values
:
if
field
==
'year'
:
value
=
_transform_year
(
value
)
if
value
is
not
:
result
.
append
(
'%s:%d'
%
(
field
,
value
)
)
elif
field
==
'any'
:
result
.
append
(
'"%s"'
%
value
)
else
:
result
.
append
(
'%s:"%s"'
%
(
field
,
value
)
)
return
' '
.
join
(
result
)


def _parse_retry_after(self, response):
        """Parse Retry-After header from response if it is set."""
        value = response.headers.get('Retry-After')

        if not value:
            seconds = 0
        elif re.match(r'^\s*[0-9]+\s*$', value):
            seconds = int(value)
        else:
            date_tuple = email.utils.parsedate(value)
            if date_tuple is None:
                seconds = 0
            else:
                seconds = time.mktime(date_tuple) - time.time()
        return max(0, seconds)

def _parse_retry_after ( self , response ) : value = response . headers . get ( 'Retry-After' ) if not value : seconds = 0 elif re . match ( r'^\s*[0-9]+\s*$' , value ) : seconds = int ( value ) else : date_tuple = email . utils . parsedate ( value ) if date_tuple is None : seconds = 0 else : seconds = time . mktime ( date_tuple ) - time . time ( ) return max ( 0 , seconds )

Parse Retry - After header from response if it is set .


def validate_value(self, value):
        """
        Validate new property value before setting it.

        value -- New value
        """
        if 'readOnly' in self.metadata and self.metadata['readOnly']:
            raise PropertyError('Read-only property')

        try:
            validate(value, self.metadata)
        except ValidationError:
            raise PropertyError('Invalid property value')

def
validate_value
(
self
,
value
)
:
if
'readOnly'
in
self
.
metadata
and
self
.
metadata
[
'readOnly'
]
:
raise
PropertyError
(
'Read-only property'
)
try
:
validate
(
value
,
self
.
metadata
)
except
ValidationError
:
raise
PropertyError
(
'Invalid property value'
)

Validate new property value before setting it .


def as_property_description(self):
        """
        Get the property description.

        Returns a dictionary describing the property.
        """
        description = deepcopy(self.metadata)

        if 'links' not in description:
            description['links'] = []

        description['links'].append(
            {
                'rel': 'property',
                'href': self.href_prefix + self.href,
            }
        )
        return description

def as_property_description ( self ) : description = deepcopy ( self . metadata ) if 'links' not in description : description [ 'links' ] = [ ] description [ 'links' ] . append ( { 'rel' : 'property' , 'href' : self . href_prefix + self . href , } ) return description


def set_value(self, value):
        """
        Set the current value of the property.

        value -- the value to set
        """
        self.validate_value(value)
        self.value.set(value)

def set_value ( self , value ) : self . validate_value ( value ) self . value . set ( value )


def get_thing(self, idx):
        """
        Get the thing at the given index.

        idx -- the index
        """
        try:
            idx = int(idx)
        except ValueError:
            return None

        if idx < 0 or idx >= len(self.things):
            return None

        return self.things[idx]

def get_thing ( self , idx ) : try : idx = int ( idx ) except ValueError : return None if idx < 0 or idx >= len ( self . things ) : return None return self . things [ idx ]


def initialize(self, things, hosts):
        """
        Initialize the handler.

        things -- list of Things managed by this server
        hosts -- list of allowed hostnames
        """
        self.things = things
        self.hosts = hosts

def initialize ( self , things , hosts ) : self . things = things self . hosts = hosts


def set_default_headers(self, *args, **kwargs):
        """Set the default headers for all requests."""
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Headers',
                        'Origin, X-Requested-With, Content-Type, Accept')
        self.set_header('Access-Control-Allow-Methods',
                        'GET, HEAD, PUT, POST, DELETE')

Set the default headers for all requests .


def prepare(self):
        """Validate Host header."""
        host = self.request.headers.get('Host', None)
        if host is not None and host in self.hosts:
            return

        raise tornado.web.HTTPError(403)

def prepare ( self ) : host = self . request . headers . get ( 'Host' , None ) if host is not None and host in self . hosts : return raise tornado . web . HTTPError ( 403 )


def get(self, thing_id='0'):
        """
        Handle a GET request, including websocket requests.

        thing_id -- ID of the thing this request is for
        """
        self.thing = self.get_thing(thing_id)
        if self.thing is None:
            self.set_status(404)
            self.finish()
            return

        if self.request.headers.get('Upgrade', '').lower() == 'websocket':
            yield tornado.websocket.WebSocketHandler.get(self)
            return

        self.set_header('Content-Type', 'application/json')
        ws_href = '{}://{}'.format(
            'wss' if self.request.protocol == 'https' else 'ws',
            self.request.headers.get('Host', '')
        )

        description = self.thing.as_thing_description()
        description['links'].append({
            'rel': 'alternate',
            'href': '{}{}'.format(ws_href, self.thing.get_href()),
        })

        self.write(json.dumps(description))
        self.finish()

def get ( self , thing_id = '0' ) : self . thing = self . get_thing ( thing_id ) if self . thing is None : self . set_status ( 404 ) self . finish ( ) return if self . request . headers . get ( 'Upgrade' , '' ) . lower ( ) == 'websocket' : yield tornado . websocket . WebSocketHandler . get ( self ) return self . set_header ( 'Content-Type' , 'application/json' ) ws_href = '{}://{}' . format ( 'wss' if self . request . protocol == 'https' else 'ws' , self . request . headers . get ( 'Host' , '' ) ) description = self . thing . as_thing_description ( ) description [ 'links' ] . append ( { 'rel' : 'alternate' , 'href' : '{}{}' . format ( ws_href , self . thing . get_href ( ) ) , } ) self . write ( json . dumps ( description ) ) self . finish ( )

Handle a GET request including websocket requests .


def on_message(self, message):
        """
        Handle an incoming message.

        message -- message to handle
        """
        try:
            message = json.loads(message)
        except ValueError:
            try:
                self.write_message(json.dumps({
                    'messageType': 'error',
                    'data': {
                        'status': '400 Bad Request',
                        'message': 'Parsing request failed',
                    },
                }))
            except tornado.websocket.WebSocketClosedError:
                pass

            return

        if 'messageType' not in message or 'data' not in message:
            try:
                self.write_message(json.dumps({
                    'messageType': 'error',
                    'data': {
                        'status': '400 Bad Request',
                        'message': 'Invalid message',
                    },
                }))
            except tornado.websocket.WebSocketClosedError:
                pass

            return

        msg_type = message['messageType']
        if msg_type == 'setProperty':
            for property_name, property_value in message['data'].items():
                try:
                    self.thing.set_property(property_name, property_value)
                except PropertyError as e:
                    self.write_message(json.dumps({
                        'messageType': 'error',
                        'data': {
                            'status': '400 Bad Request',
                            'message': str(e),
                        },
                    }))
        elif msg_type == 'requestAction':
            for action_name, action_params in message['data'].items():
                input_ = None
                if 'input' in action_params:
                    input_ = action_params['input']

                action = self.thing.perform_action(action_name, input_)
                if action:
                    tornado.ioloop.IOLoop.current().spawn_callback(
                        perform_action,
                        action,
                    )
                else:
                    self.write_message(json.dumps({
                        'messageType': 'error',
                        'data': {
                            'status': '400 Bad Request',
                            'message': 'Invalid action request',
                            'request': message,
                        },
                    }))
        elif msg_type == 'addEventSubscription':
            for event_name in message['data'].keys():
                self.thing.add_event_subscriber(event_name, self)
        else:
            try:
                self.write_message(json.dumps({
                    'messageType': 'error',
                    'data': {
                        'status': '400 Bad Request',
                        'message': 'Unknown messageType: ' + msg_type,
                        'request': message,
                    },
                }))
            except tornado.websocket.WebSocketClosedError:
                pass

def
on_message
(
self
,
message
)
:
try
:
message
=
json
.
loads
(
message
)
except
ValueError
:
try
:
self
.
write_message
(
json
.
dumps
(
{
'messageType'
:
'error'
,
'data'
:
{
'status'
:
'400 Bad Request'
,
'message'
:
'Parsing request failed'
,
}
,
}
)
)
except
tornado
.
websocket
.
WebSocketClosedError
:
pass
return
if
'messageType'
not
in
message
or
'data'
not
in
message
:
try
:
self
.
write_message
(
json
.
dumps
(
{
'messageType'
:
'error'
,
'data'
:
{
'status'
:
'400 Bad Request'
,
'message'
:
'Invalid message'
,
}
,
}
)
)
except
tornado
.
websocket
.
WebSocketClosedError
:
pass
return
msg_type
=
message
[
'messageType'
]
if
msg_type
==
'setProperty'
:
for
property_name
,
property_value
in
message
[
'data'
]
.
items
(
)
:
try
:
self
.
thing
.
set_property
(
property_name
,
property_value
)
except
PropertyError
as
e
:
self
.
write_message
(
json
.
dumps
(
{
'messageType'
:
'error'
,
'data'
:
{
'status'
:
'400 Bad Request'
,
'message'
:
str
(
e
)
,
}
,
}
)
)
elif
msg_type
==
'requestAction'
:
for
action_name
,
action_params
in
message
[
'data'
]
.
items
(
)
:
input_
=
if
'input'
in
action_params
:
input_
=
action_params
[
'input'
]
action
=
self
.
thing
.
perform_action
(
action_name
,
input_
)
if
action
:
tornado
.
ioloop
.
IOLoop
.
current
(
)
.
spawn_callback
(
perform_action
,
action
,
)
else
:
self
.
write_message
(
json
.
dumps
(
{
'messageType'
:
'error'
,
'data'
:
{
'status'
:
'400 Bad Request'
,
'message'
:
'Invalid action request'
,
'request'
:
message
,
}
,
}
)
)
elif
msg_type
==
'addEventSubscription'
:
for
event_name
in
message
[
'data'
]
.
keys
(
)
:
self
.
thing
.
add_event_subscriber
(
event_name
,
self
)
else
:
try
:
self
.
write_message
(
json
.
dumps
(
{
'messageType'
:
'error'
,
'data'
:
{
'status'
:
'400 Bad Request'
,
'message'
:
'Unknown messageType: '
+
msg_type
,
'request'
:
message
,
}
,
}
)
)
except
tornado
.
websocket
.
WebSocketClosedError
:
pass


def post(self, thing_id='0'):
        """
        Handle a POST request.

        thing_id -- ID of the thing this request is for
        """
        thing = self.get_thing(thing_id)
        if thing is None:
            self.set_status(404)
            return

        try:
            message = json.loads(self.request.body.decode())
        except ValueError:
            self.set_status(400)
            return

        response = {}
        for action_name, action_params in message.items():
            input_ = None
            if 'input' in action_params:
                input_ = action_params['input']

            action = thing.perform_action(action_name, input_)
            if action:
                response.update(action.as_action_description())

                # Start the action
                tornado.ioloop.IOLoop.current().spawn_callback(
                    perform_action,
                    action,
                )

        self.set_status(201)
        self.write(json.dumps(response))

def post ( self , thing_id = '0' ) : thing = self . get_thing ( thing_id ) if thing is None : self . set_status ( 404 ) return try : message = json . loads ( self . request . body . decode ( ) ) except ValueError : self . set_status ( 400 ) return response = { } for action_name , action_params in message . items ( ) : input_ = None if 'input' in action_params : input_ = action_params [ 'input' ] action = thing . perform_action ( action_name , input_ ) if action : response . update ( action . as_action_description ( ) ) tornado . ioloop . IOLoop . current ( ) . spawn_callback ( perform_action , action , ) self . set_status ( 201 ) self . write ( json . dumps ( response ) )


def delete(self, thing_id='0', action_name=None, action_id=None):
        """
        Handle a DELETE request.

        thing_id -- ID of the thing this request is for
        action_name -- name of the action from the URL path
        action_id -- the action ID from the URL path
        """
        thing = self.get_thing(thing_id)
        if thing is None:
            self.set_status(404)
            return

        if thing.remove_action(action_name, action_id):
            self.set_status(204)
        else:
            self.set_status(404)

def delete ( self , thing_id = '0' , action_name = None , action_id = None ) : thing = self . get_thing ( thing_id ) if thing is None : self . set_status ( 404 ) return if thing . remove_action ( action_name , action_id ) : self . set_status ( 204 ) else : self . set_status ( 404 )


def start(self):
        """Start listening for incoming connections."""
        self.service_info = ServiceInfo(
            '_webthing._tcp.local.',
            '{}._webthing._tcp.local.'.format(self.name),
            address=socket.inet_aton(get_ip()),
            port=self.port,
            properties={
                'path': '/',
            },
            server='{}.local.'.format(socket.gethostname()))
        self.zeroconf = Zeroconf()
        self.zeroconf.register_service(self.service_info)

        self.server.listen(self.port)
        tornado.ioloop.IOLoop.current().start()

def start ( self ) : self . service_info = ServiceInfo ( '_webthing._tcp.local.' , '{}._webthing._tcp.local.' . format ( self . name ) , address = socket . inet_aton ( get_ip ( ) ) , port = self . port , properties = { 'path' : '/' , } , server = '{}.local.' . format ( socket . gethostname ( ) ) ) self . zeroconf = Zeroconf ( ) self . zeroconf . register_service ( self . service_info ) self . server . listen ( self . port ) tornado . ioloop . IOLoop . current ( ) . start ( )

Start listening for incoming connections .


def as_action_description(self):
        """
        Get the action description.

        Returns a dictionary describing the action.
        """
        description = {
            self.name: {
                'href': self.href_prefix + self.href,
                'timeRequested': self.time_requested,
                'status': self.status,
            },
        }

        if self.input is not None:
            description[self.name]['input'] = self.input

        if self.time_completed is not None:
            description[self.name]['timeCompleted'] = self.time_completed

        return description

def as_action_description ( self ) : description = { self . name : { 'href' : self . href_prefix + self . href , 'timeRequested' : self . time_requested , 'status' : self . status , } , } if self . input is not None : description [ self . name ] [ 'input' ] = self . input if self . time_completed is not None : description [ self . name ] [ 'timeCompleted' ] = self . time_completed return description


def start(self):
        """Start performing the action."""
        self.status = 'pending'
        self.thing.action_notify(self)
        self.perform_action()
        self.finish()

def start ( self ) : self . status = 'pending' self . thing . action_notify ( self ) self . perform_action ( ) self . finish ( )


def finish(self):
        """Finish performing the action."""
        self.status = 'completed'
        self.time_completed = timestamp()
        self.thing.action_notify(self)

def finish ( self ) : self . status = 'completed' self . time_completed = timestamp ( ) self . thing . action_notify ( self )


def as_event_description(self):
        """
        Get the event description.

        Returns a dictionary describing the event.
        """
        description = {
            self.name: {
                'timestamp': self.time,
            },
        }

        if self.data is not None:
            description[self.name]['data'] = self.data

        return description

def as_event_description ( self ) : description = { self . name : { 'timestamp' : self . time , } , } if self . data is not None : description [ self . name ] [ 'data' ] = self . data return description


def get_ip():
    """
    Get the default local IP address.

    From: https://stackoverflow.com/a/28950776
    """
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        s.connect(('10.255.255.255', 1))
        ip = s.getsockname()[0]
    except (socket.error, IndexError):
        ip = '127.0.0.1'
    finally:
        s.close()

    return ip

def get_ip ( ) : s = socket . socket ( socket . AF_INET , socket . SOCK_DGRAM ) try : s . connect ( ( '10.255.255.255' , 1 ) ) ip = s . getsockname ( ) [ 0 ] except ( socket . error , IndexError ) : ip = '127.0.0.1' finally : s . close ( ) return ip


def get_addresses():
    """
    Get all IP addresses.

    Returns list of addresses.
    """
    addresses = set()

    for iface in ifaddr.get_adapters():
        for addr in iface.ips:
            # Filter out link-local addresses.
            if addr.is_IPv4:
                ip = addr.ip

                if not ip.startswith('169.254.'):
                    addresses.add(ip)
            elif addr.is_IPv6:
                # Sometimes, IPv6 addresses will have the interface name
                # appended, e.g. %eth0. Handle that.
                ip = addr.ip[0].split('%')[0].lower()

                if not ip.startswith('fe80:'):
                    addresses.add('[{}]'.format(ip))

    return sorted(list(addresses))

def get_addresses ( ) : addresses = set ( ) for iface in ifaddr . get_adapters ( ) : for addr in iface . ips : if addr . is_IPv4 : ip = addr . ip if not ip . startswith ( '169.254.' ) : addresses . add ( ip ) elif addr . is_IPv6 : ip = addr . ip [ 0 ] . split ( '%' ) [ 0 ] . lower ( ) if not ip . startswith ( 'fe80:' ) : addresses . add ( '[{}]' . format ( ip ) ) return sorted ( list ( addresses ) )


def set(self, value):
        """
        Set a new value for this thing.

        value -- value to set
        """
        if self.value_forwarder is not None:
            self.value_forwarder(value)

        self.notify_of_external_update(value)

def set ( self , value ) : if self . value_forwarder is not None : self . value_forwarder ( value ) self . notify_of_external_update ( value )


def notify_of_external_update(self, value):
        """
        Notify observers of a new value.

        value -- new value
        """
        if value is not None and value != self.last_value:
            self.last_value = value
            self.emit('update', value)

def notify_of_external_update ( self , value ) : if value is not None and value != self . last_value : self . last_value = value self . emit ( 'update' , value )


def as_thing_description(self):
        """
        Return the thing state as a Thing Description.

        Returns the state as a dictionary.
        """
        thing = {
            'name': self.name,
            'href': self.href_prefix if self.href_prefix else '/',
            '@context': self.context,
            '@type': self.type,
            'properties': self.get_property_descriptions(),
            'actions': {},
            'events': {},
            'links': [
                {
                    'rel': 'properties',
                    'href': '{}/properties'.format(self.href_prefix),
                },
                {
                    'rel': 'actions',
                    'href': '{}/actions'.format(self.href_prefix),
                },
                {
                    'rel': 'events',
                    'href': '{}/events'.format(self.href_prefix),
                },
            ],
        }

        for name, action in self.available_actions.items():
            thing['actions'][name] = action['metadata']
            thing['actions'][name]['links'] = [
                {
                    'rel': 'action',
                    'href': '{}/actions/{}'.format(self.href_prefix, name),
                },
            ]

        for name, event in self.available_events.items():
            thing['events'][name] = event['metadata']
            thing['events'][name]['links'] = [
                {
                    'rel': 'event',
                    'href': '{}/events/{}'.format(self.href_prefix, name),
                },
            ]

        if self.ui_href is not None:
            thing['links'].append({
                'rel': 'alternate',
                'mediaType': 'text/html',
                'href': self.ui_href,
            })

        if self.description:
            thing['description'] = self.description

        return thing

def as_thing_description ( self ) : thing = { 'name' : self . name , 'href' : self . href_prefix if self . href_prefix else '/' , '@context' : self . context , '@type' : self . type , 'properties' : self . get_property_descriptions ( ) , 'actions' : { } , 'events' : { } , 'links' : [ { 'rel' : 'properties' , 'href' : '{}/properties' . format ( self . href_prefix ) , } , { 'rel' : 'actions' , 'href' : '{}/actions' . format ( self . href_prefix ) , } , { 'rel' : 'events' , 'href' : '{}/events' . format ( self . href_prefix ) , } , ] , } for name , action in self . available_actions . items ( ) : thing [ 'actions' ] [ name ] = action [ 'metadata' ] thing [ 'actions' ] [ name ] [ 'links' ] = [ { 'rel' : 'action' , 'href' : '{}/actions/{}' . format ( self . href_prefix , name ) , } , ] for name , event in self . available_events . items ( ) : thing [ 'events' ] [ name ] = event [ 'metadata' ] thing [ 'events' ] [ name ] [ 'links' ] = [ { 'rel' : 'event' , 'href' : '{}/events/{}' . format ( self . href_prefix , name ) , } , ] if self . ui_href is not None : thing [ 'links' ] . append ( { 'rel' : 'alternate' , 'mediaType' : 'text/html' , 'href' : self . ui_href , } ) if self . description : thing [ 'description' ] = self . description return thing

Return the thing state as a Thing Description .


def set_href_prefix(self, prefix):
        """
        Set the prefix of any hrefs associated with this thing.

        prefix -- the prefix
        """
        self.href_prefix = prefix

        for property_ in self.properties.values():
            property_.set_href_prefix(prefix)

        for action_name in self.actions.keys():
            for action in self.actions[action_name]:
                action.set_href_prefix(prefix)

def set_href_prefix ( self , prefix ) : self . href_prefix = prefix for property_ in self . properties . values ( ) : property_ . set_href_prefix ( prefix ) for action_name in self . actions . keys ( ) : for action in self . actions [ action_name ] : action . set_href_prefix ( prefix )

Set the prefix of any hrefs associated with this thing .


def get_property_descriptions(self):
        """
        Get the thing's properties as a dictionary.

        Returns the properties as a dictionary, i.e. name -> description.
        """
        return {k: v.as_property_description()
                for k, v in self.properties.items()}

def get_property_descriptions ( self ) : return { k : v . as_property_description ( ) for k , v in self . properties . items ( ) }

Get the thing s properties as a dictionary .


def get_action_descriptions(self, action_name=None):
        """
        Get the thing's actions as an array.

        action_name -- Optional action name to get descriptions for

        Returns the action descriptions.
        """
        descriptions = []

        if action_name is None:
            for name in self.actions:
                for action in self.actions[name]:
                    descriptions.append(action.as_action_description())
        elif action_name in self.actions:
            for action in self.actions[action_name]:
                descriptions.append(action.as_action_description())

        return descriptions

def get_action_descriptions ( self , action_name = None ) : descriptions = [ ] if action_name is None : for name in self . actions : for action in self . actions [ name ] : descriptions . append ( action . as_action_description ( ) ) elif action_name in self . actions : for action in self . actions [ action_name ] : descriptions . append ( action . as_action_description ( ) ) return descriptions


def get_event_descriptions(self, event_name=None):
        """
        Get the thing's events as an array.

        event_name -- Optional event name to get descriptions for

        Returns the event descriptions.
        """
        if event_name is None:
            return [e.as_event_description() for e in self.events]
        else:
            return [e.as_event_description()
                    for e in self.events if e.get_name() == event_name]

def get_event_descriptions ( self , event_name = None ) : if event_name is None : return [ e . as_event_description ( ) for e in self . events ] else : return [ e . as_event_description ( ) for e in self . events if e . get_name ( ) == event_name ]


def add_property(self, property_):
        """
        Add a property to this thing.

        property_ -- property to add
        """
        property_.set_href_prefix(self.href_prefix)
        self.properties[property_.name] = property_

def add_property ( self , property_ ) : property_ . set_href_prefix ( self . href_prefix ) self . properties [ property_ . name ] = property_


def remove_property(self, property_):
        """
        Remove a property from this thing.

        property_ -- property to remove
        """
        if property_.name in self.properties:
            del self.properties[property_.name]

def remove_property ( self , property_ ) : if property_ . name in self . properties : del self . properties [ property_ . name ]


def get_property(self, property_name):
        """
        Get a property's value.

        property_name -- the property to get the value of

        Returns the properties value, if found, else None.
        """
        prop = self.find_property(property_name)
        if prop:
            return prop.get_value()

        return None

def get_property ( self , property_name ) : prop = self . find_property ( property_name ) if prop : return prop . get_value ( ) return None


def get_properties(self):
        """
        Get a mapping of all properties and their values.

        Returns a dictionary of property_name -> value.
        """
        return {prop.get_name(): prop.get_value()
                for prop in self.properties.values()}

def get_properties ( self ) : return { prop . get_name ( ) : prop . get_value ( ) for prop in self . properties . values ( ) }

Get a mapping of all properties and their values .


def set_property(self, property_name, value):
        """
        Set a property value.

        property_name -- name of the property to set
        value -- value to set
        """
        prop = self.find_property(property_name)
        if not prop:
            return

        prop.set_value(value)

def set_property ( self , property_name , value ) : prop = self . find_property ( property_name ) if not prop : return prop . set_value ( value )


def get_action(self, action_name, action_id):
        """
        Get an action.

        action_name -- name of the action
        action_id -- ID of the action

        Returns the requested action if found, else None.
        """
        if action_name not in self.actions:
            return None

        for action in self.actions[action_name]:
            if action.id == action_id:
                return action

        return None

def get_action ( self , action_name , action_id ) : if action_name not in self . actions : return None for action in self . actions [ action_name ] : if action . id == action_id : return action return None


def add_event(self, event):
        """
        Add a new event and notify subscribers.

        event -- the event that occurred
        """
        self.events.append(event)
        self.event_notify(event)

def add_event ( self , event ) : self . events . append ( event ) self . event_notify ( event )

Add a new event and notify subscribers .


def add_available_event(self, name, metadata):
        """
        Add an available event.

        name -- name of the event
        metadata -- event metadata, i.e. type, description, etc., as a dict
        """
        if metadata is None:
            metadata = {}

        self.available_events[name] = {
            'metadata': metadata,
            'subscribers': set(),
        }

def add_available_event ( self , name , metadata ) : if metadata is None : metadata = { } self . available_events [ name ] = { 'metadata' : metadata , 'subscribers' : set ( ) , }


def perform_action(self, action_name, input_=None):
        """
        Perform an action on the thing.

        action_name -- name of the action
        input_ -- any action inputs

        Returns the action that was created.
        """
        if action_name not in self.available_actions:
            return None

        action_type = self.available_actions[action_name]

        if 'input' in action_type['metadata']:
            try:
                validate(input_, action_type['metadata']['input'])
            except ValidationError:
                return None

        action = action_type['class'](self, input_=input_)
        action.set_href_prefix(self.href_prefix)
        self.action_notify(action)
        self.actions[action_name].append(action)
        return action

def perform_action ( self , action_name , input_ = None ) : if action_name not in self . available_actions : return None action_type = self . available_actions [ action_name ] if 'input' in action_type [ 'metadata' ] : try : validate ( input_ , action_type [ 'metadata' ] [ 'input' ] ) except ValidationError : return None action = action_type [ 'class' ] ( self , input_ = input_ ) action . set_href_prefix ( self . href_prefix ) self . action_notify ( action ) self . actions [ action_name ] . append ( action ) return action


def remove_action(self, action_name, action_id):
        """
        Remove an existing action.

        action_name -- name of the action
        action_id -- ID of the action

        Returns a boolean indicating the presence of the action.
        """
        action = self.get_action(action_name, action_id)
        if action is None:
            return False

        action.cancel()
        self.actions[action_name].remove(action)
        return True

def remove_action ( self , action_name , action_id ) : action = self . get_action ( action_name , action_id ) if action is None : return False action . cancel ( ) self . actions [ action_name ] . remove ( action ) return True


def add_available_action(self, name, metadata, cls):
        """
        Add an available action.

        name -- name of the action
        metadata -- action metadata, i.e. type, description, etc., as a dict
        cls -- class to instantiate for this action
        """
        if metadata is None:
            metadata = {}

        self.available_actions[name] = {
            'metadata': metadata,
            'class': cls,
        }
        self.actions[name] = []

def add_available_action ( self , name , metadata , cls ) : if metadata is None : metadata = { } self . available_actions [ name ] = { 'metadata' : metadata , 'class' : cls , } self . actions [ name ] = [ ]


def remove_subscriber(self, ws):
        """
        Remove a websocket subscriber.

        ws -- the websocket
        """
        if ws in self.subscribers:
            self.subscribers.remove(ws)

        for name in self.available_events:
            self.remove_event_subscriber(name, ws)

def remove_subscriber ( self , ws ) : if ws in self . subscribers : self . subscribers . remove ( ws ) for name in self . available_events : self . remove_event_subscriber ( name , ws )


def add_event_subscriber(self, name, ws):
        """
        Add a new websocket subscriber to an event.

        name -- name of the event
        ws -- the websocket
        """
        if name in self.available_events:
            self.available_events[name]['subscribers'].add(ws)

def add_event_subscriber ( self , name , ws ) : if name in self . available_events : self . available_events [ name ] [ 'subscribers' ] . add ( ws )

Add a new websocket subscriber to an event .


def remove_event_subscriber(self, name, ws):
        """
        Remove a websocket subscriber from an event.

        name -- name of the event
        ws -- the websocket
        """
        if name in self.available_events and \
                ws in self.available_events[name]['subscribers']:
            self.available_events[name]['subscribers'].remove(ws)

def remove_event_subscriber ( self , name , ws ) : if name in self . available_events and ws in self . available_events [ name ] [ 'subscribers' ] : self . available_events [ name ] [ 'subscribers' ] . remove ( ws )

Remove a websocket subscriber from an event .


def property_notify(self, property_):
        """
        Notify all subscribers of a property change.

        property_ -- the property that changed
        """
        message = json.dumps({
            'messageType': 'propertyStatus',
            'data': {
                property_.name: property_.get_value(),
            }
        })

        for subscriber in list(self.subscribers):
            try:
                subscriber.write_message(message)
            except tornado.websocket.WebSocketClosedError:
                pass

def property_notify ( self , property_ ) : message = json . dumps ( { 'messageType' : 'propertyStatus' , 'data' : { property_ . name : property_ . get_value ( ) , } } ) for subscriber in list ( self . subscribers ) : try : subscriber . write_message ( message ) except tornado . websocket . WebSocketClosedError : pass

Notify all subscribers of a property change .


def action_notify(self, action):
        """
        Notify all subscribers of an action status change.

        action -- the action whose status changed
        """
        message = json.dumps({
            'messageType': 'actionStatus',
            'data': action.as_action_description(),
        })

        for subscriber in list(self.subscribers):
            try:
                subscriber.write_message(message)
            except tornado.websocket.WebSocketClosedError:
                pass

def action_notify ( self , action ) : message = json . dumps ( { 'messageType' : 'actionStatus' , 'data' : action . as_action_description ( ) , } ) for subscriber in list ( self . subscribers ) : try : subscriber . write_message ( message ) except tornado . websocket . WebSocketClosedError : pass

Notify all subscribers of an action status change .


def event_notify(self, event):
        """
        Notify all subscribers of an event.

        event -- the event that occurred
        """
        if event.name not in self.available_events:
            return

        message = json.dumps({
            'messageType': 'event',
            'data': event.as_event_description(),
        })

        for subscriber in self.available_events[event.name]['subscribers']:
            try:
                subscriber.write_message(message)
            except tornado.websocket.WebSocketClosedError:
                pass

def event_notify ( self , event ) : if event . name not in self . available_events : return message = json . dumps ( { 'messageType' : 'event' , 'data' : event . as_event_description ( ) , } ) for subscriber in self . available_events [ event . name ] [ 'subscribers' ] : try : subscriber . write_message ( message ) except tornado . websocket . WebSocketClosedError : pass


def annotate(self, **annotations):
        """Custom version of the standard annotate function
        that allows using field names as annotated fields.

        Normally, the annotate function doesn't allow you
        to use the name of an existing field on the model
        as the alias name. This version of the function does
        allow that.
        """

        fields = {
            field.name: field
            for field in self.model._meta.get_fields()
        }

        # temporarily rename the fields that have the same
        # name as a field name, we'll rename them back after
        # the function in the base class ran
        new_annotations = {}
        renames = {}
        for name, value in annotations.items():
            if name in fields:
                new_name = '%s_new' % name
                new_annotations[new_name] = value
                renames[new_name] = name
            else:
                new_annotations[name] = value

        # run the base class's annotate function
        result = super().annotate(**new_annotations)

        # rename the annotations back to as specified
        result.rename_annotations(**renames)
        return result

def annotate ( self , ** annotations ) : fields = { field . name : field for field in self . model . _meta . get_fields ( ) } new_annotations = { } renames = { } for name , value in annotations . items ( ) : if name in fields : new_name = '%s_new' % name new_annotations [ new_name ] = value renames [ new_name ] = name else : new_annotations [ name ] = value result = super ( ) . annotate ( ** new_annotations ) result . rename_annotations ( ** renames ) return result

Custom version of the standard annotate function that allows using field names as annotated fields .


def update(self, **fields):
        """Updates all rows that match the filter."""

        # build up the query to execute
        self._for_write = True
        if django.VERSION >= (2, 0):
            query = self.query.chain(UpdateQuery)
        else:
            query = self.query.clone(UpdateQuery)
        query._annotations = None
        query.add_update_values(fields)

        # build the compiler for for the query
        connection = django.db.connections[self.db]
        compiler = PostgresReturningUpdateCompiler(query, connection, self.db)

        # execute the query
        with transaction.atomic(using=self.db, savepoint=False):
            rows = compiler.execute_sql(CURSOR)
        self._result_cache = None

        # send out a signal for each row
        for row in rows:
            signals.update.send(self.model, pk=row[0])

        # the original update(..) returns the amount of rows
        # affected, let's do the same
        return len(rows)

def update ( self , ** fields ) : self . _for_write = True if django . VERSION >= ( 2 , 0 ) : query = self . query . chain ( UpdateQuery ) else : query = self . query . clone ( UpdateQuery ) query . _annotations = None query . add_update_values ( fields ) connection = django . db . connections [ self . db ] compiler = PostgresReturningUpdateCompiler ( query , connection , self . db ) with transaction . atomic ( using = self . db , savepoint = False ) : rows = compiler . execute_sql ( CURSOR ) self . _result_cache = None for row in rows : signals . update . send ( self . model , pk = row [ 0 ] ) return len ( rows )

Updates all rows that match the filter .


def bulk_insert(self, rows, return_model=False):
        """Creates multiple new records in the database.

        This allows specifying custom conflict behavior using .on_conflict().
        If no special behavior was specified, this uses the normal Django create(..)

        Arguments:
            rows:
                An array of dictionaries, where each dictionary
                describes the fields to insert.

            return_model (default: False):
                If model instances should be returned rather than
                just dicts.

        Returns:
            A list of either the dicts of the rows inserted, including the pk or
            the models of the rows inserted with defaults for any fields not specified
        """

        if self.conflict_target or self.conflict_action:
            compiler = self._build_insert_compiler(rows)
            objs = compiler.execute_sql(return_id=True)
            if return_model:
                return [self.model(**dict(r, **k)) for r, k in zip(rows, objs)]
            else:
                return [dict(r, **k) for r, k in zip(rows, objs)]

        # no special action required, use the standard Django bulk_create(..)
        return super().bulk_create([self.model(**fields) for fields in rows])

def bulk_insert ( self , rows , return_model = False ) : if self . conflict_target or self . conflict_action : compiler = self . _build_insert_compiler ( rows ) objs = compiler . execute_sql ( return_id = True ) if return_model : return [ self . model ( ** dict ( r , ** k ) ) for r , k in zip ( rows , objs ) ] else : return [ dict ( r , ** k ) for r , k in zip ( rows , objs ) ] return super ( ) . bulk_create ( [ self . model ( ** fields ) for fields in rows ] )

Creates multiple new records in the database .


def insert(self, **fields):
        """Creates a new record in the database.

        This allows specifying custom conflict behavior using .on_conflict().
        If no special behavior was specified, this uses the normal Django create(..)

        Arguments:
            fields:
                The fields of the row to create.

        Returns:
            The primary key of the record that was created.
        """

        if self.conflict_target or self.conflict_action:
            compiler = self._build_insert_compiler([fields])
            rows = compiler.execute_sql(return_id=True)

            pk_field_name = self.model._meta.pk.name
            return rows[0][pk_field_name]

        # no special action required, use the standard Django create(..)
        return super().create(**fields).pk

def insert ( self , ** fields ) : if self . conflict_target or self . conflict_action : compiler = self . _build_insert_compiler ( [ fields ] ) rows = compiler . execute_sql ( return_id = True ) pk_field_name = self . model . _meta . pk . name return rows [ 0 ] [ pk_field_name ] return super ( ) . create ( ** fields ) . pk


def insert_and_get(self, **fields):
        """Creates a new record in the database and then gets
        the entire row.

        This allows specifying custom conflict behavior using .on_conflict().
        If no special behavior was specified, this uses the normal Django create(..)

        Arguments:
            fields:
                The fields of the row to create.

        Returns:
            The model instance representing the row that was created.
        """

        if not self.conflict_target and not self.conflict_action:
            # no special action required, use the standard Django create(..)
            return super().create(**fields)

        compiler = self._build_insert_compiler([fields])
        rows = compiler.execute_sql(return_id=False)

        columns = rows[0]

        # get a list of columns that are officially part of the model and preserve the fact that the attribute name
        # might be different than the database column name
        model_columns = {}
        for field in self.model._meta.local_concrete_fields:
            model_columns[field.column] = field.attname

        # strip out any columns/fields returned by the db that
        # are not present in the model
        model_init_fields = {}
        for column_name, column_value in columns.items():
            try:
                model_init_fields[model_columns[column_name]] = column_value
            except KeyError:
                pass

        return self.model(**model_init_fields)

def insert_and_get ( self , ** fields ) : if not self . conflict_target and not self . conflict_action : return super ( ) . create ( ** fields ) compiler = self . _build_insert_compiler ( [ fields ] ) rows = compiler . execute_sql ( return_id = False ) columns = rows [ 0 ] model_columns = { } for field in self . model . _meta . local_concrete_fields : model_columns [ field . column ] = field . attname model_init_fields = { } for column_name , column_value in columns . items ( ) : try : model_init_fields [ model_columns [ column_name ] ] = column_value except KeyError : pass return self . model ( ** model_init_fields )

Creates a new record in the database and then gets the entire row .


def _build_insert_compiler(self, rows: List[Dict]):
        """Builds the SQL compiler for a insert query.

        Arguments:
            rows:
                A list of dictionaries, where each entry
                describes a record to insert.

        Returns:
            The SQL compiler for the insert.
        """

        # create model objects, we also have to detect cases
        # such as:
        #   [dict(first_name='swen'), dict(fist_name='swen', last_name='kooij')]
        # we need to be certain that each row specifies the exact same
        # amount of fields/columns
        objs = []
        field_count = len(rows[0])
        for index, row in enumerate(rows):
            if field_count != len(row):
                raise SuspiciousOperation((
                    'In bulk upserts, you cannot have rows with different field '
                    'configurations. Row {0} has a different field config than '
                    'the first row.'
                ).format(index))

            objs.append(self.model(**row))

        # indicate this query is going to perform write
        self._for_write = True

        # get the fields to be used during update/insert
        insert_fields, update_fields = self._get_upsert_fields(rows[0])

        # build a normal insert query
        query = PostgresInsertQuery(self.model)
        query.conflict_action = self.conflict_action
        query.conflict_target = self.conflict_target
        query.index_predicate = self.index_predicate
        query.values(objs, insert_fields, update_fields)

        # use the postgresql insert query compiler to transform the insert
        # into an special postgresql insert
        connection = django.db.connections[self.db]
        compiler = PostgresInsertCompiler(query, connection, self.db)

        return compiler

Builds the SQL compiler for a insert query .


def _is_magical_field(self, model_instance, field, is_insert: bool):
        """Verifies whether this field is gonna modify something
        on its own.

        "Magical" means that a field modifies the field value
        during the pre_save.

        Arguments:
            model_instance:
                The model instance the field is defined on.

            field:
                The field to get of whether the field is
                magical.

            is_insert:
                Pretend whether this is an insert?

        Returns:
            True when this field modifies something.
        """

        # does this field modify someting upon insert?
        old_value = getattr(model_instance, field.name, None)
        field.pre_save(model_instance, is_insert)
        new_value = getattr(model_instance, field.name, None)

        return old_value != new_value

def _is_magical_field ( self , model_instance , field , is_insert : bool ) : old_value = getattr ( model_instance , field . name , None ) field . pre_save ( model_instance , is_insert ) new_value = getattr ( model_instance , field . name , None ) return old_value != new_value

Verifies whether this field is gonna modify something on its own .


def _get_upsert_fields(self, kwargs):
        """Gets the fields to use in an upsert.

        This some nice magic. We'll split the fields into
        a group of "insert fields" and "update fields":

        INSERT INTO bla ("val1", "val2") ON CONFLICT DO UPDATE SET val1 = EXCLUDED.val1

                         ^^^^^^^^^^^^^^                            ^^^^^^^^^^^^^^^^^^^^
                         insert_fields                                 update_fields

        Often, fields appear in both lists. But, for example,
        a :see:DateTime field with `auto_now_add=True` set, will
        only appear in "insert_fields", since it won't be set
        on existing rows.

        Other than that, the user specificies a list of fields
        in the upsert() call. That migt not be all fields. The
        user could decide to leave out optional fields. If we
        end up doing an update, we don't want to overwrite
        those non-specified fields.

        We cannot just take the list of fields the user
        specifies, because as mentioned, some fields
        make modifications to the model on their own.

        We'll have to detect which fields make modifications
        and include them in the list of insert/update fields.
        """

        model_instance = self.model(**kwargs)
        insert_fields = []
        update_fields = []

        for field in model_instance._meta.local_concrete_fields:
            has_default = field.default != NOT_PROVIDED
            if (field.name in kwargs or field.column in kwargs):
                insert_fields.append(field)
                update_fields.append(field)
                continue
            elif has_default:
                insert_fields.append(field)
                continue

            # special handling for 'pk' which always refers to
            # the primary key, so if we the user specifies `pk`
            # instead of a concrete field, we have to handle that
            if field.primary_key is True and 'pk' in kwargs:
                insert_fields.append(field)
                update_fields.append(field)
                continue

            if self._is_magical_field(model_instance, field, is_insert=True):
                insert_fields.append(field)

            if self._is_magical_field(model_instance, field, is_insert=False):
                update_fields.append(field)

        return insert_fields, update_fields

def _get_upsert_fields ( self , kwargs ) : model_instance = self . model ( ** kwargs ) insert_fields = [ ] update_fields = [ ] for field in model_instance . _meta . local_concrete_fields : has_default = field . default != NOT_PROVIDED if ( field . name in kwargs or field . column in kwargs ) : insert_fields . append ( field ) update_fields . append ( field ) continue elif has_default : insert_fields . append ( field ) continue if field . primary_key is True and 'pk' in kwargs : insert_fields . append ( field ) update_fields . append ( field ) continue if self . _is_magical_field ( model_instance , field , is_insert = True ) : insert_fields . append ( field ) if self . _is_magical_field ( model_instance , field , is_insert = False ) : update_fields . append ( field ) return insert_fields , update_fields


def _on_model_save(sender, **kwargs):
        """When a model gets created or updated."""

        created, instance = kwargs['created'], kwargs['instance']

        if created:
            signals.create.send(sender, pk=instance.pk)
        else:
            signals.update.send(sender, pk=instance.pk)

def _on_model_save ( sender , ** kwargs ) : created , instance = kwargs [ 'created' ] , kwargs [ 'instance' ] if created : signals . create . send ( sender , pk = instance . pk ) else : signals . update . send ( sender , pk = instance . pk )


def _on_model_delete(sender, **kwargs):
        """When a model gets deleted."""

        instance = kwargs['instance']
        signals.delete.send(sender, pk=instance.pk)

def _on_model_delete ( sender , ** kwargs ) : instance = kwargs [ 'instance' ] signals . delete . send ( sender , pk = instance . pk )