def is_connected(self):
        """
            Return bool indicating connection state
        """
        # need to wrap in try/except b/c of wc3270's socket connection dynamics
        try:
            # this is basically a no-op, but it results in the the current status
            # getting updated
            self.exec_command(b"Query(ConnectionState)")

            # connected status is like 'C(192.168.1.1)', disconnected is 'N'
            return self.status.connection_state.startswith(b"C(")
        except NotConnectedException:
            return False

def is_connected ( self ) : try : self . exec_command ( b"Query(ConnectionState)" ) return self . status . connection_state . startswith ( b"C(" ) except NotConnectedException : return False


def connect(self, host):
        """
            Connect to a host
        """
        if not self.app.connect(host):
            command = "Connect({0})".format(host).encode("ascii")
            self.exec_command(command)
        self.last_host = host

def connect ( self , host ) : if not self . app . connect ( host ) : command = "Connect({0})" . format ( host ) . encode ( "ascii" ) self . exec_command ( command ) self . last_host = host


def wait_for_field(self):
        """
            Wait until the screen is ready, the cursor has been positioned
            on a modifiable field, and the keyboard is unlocked.

            Sometimes the server will "unlock" the keyboard but the screen will
            not yet be ready.  In that case, an attempt to read or write to the
            screen will result in a 'E' keyboard status because we tried to
            read from a screen that is not yet ready.

            Using this method tells the client to wait until a field is
            detected and the cursor has been positioned on it.
        """
        self.exec_command("Wait({0}, InputField)".format(self.timeout).encode("ascii"))
        if self.status.keyboard != b"U":
            raise KeyboardStateError(
                "keyboard not unlocked, state was: {0}".format(
                    self.status.keyboard.decode("ascii")
                )
            )

def
wait_for_field
(
self
)
:
self
.
exec_command
(
"Wait({0}, InputField)"
.
format
(
self
.
timeout
)
.
encode
(
"ascii"
)
)
if
self
.
status
.
keyboard
!=
b"U"
:
raise
KeyboardStateError
(
"keyboard not unlocked, state was: {0}"
.
format
(
self
.
status
.
keyboard
.
decode
(
"ascii"
)
)
)

Wait until the screen is ready the cursor has been positioned on a modifiable field and the keyboard is unlocked .


def move_to(self, ypos, xpos):
        """
            move the cursor to the given co-ordinates.  Co-ordinates are 1
            based, as listed in the status area of the terminal.
        """
        # the screen's co-ordinates are 1 based, but the command is 0 based
        xpos -= 1
        ypos -= 1
        self.exec_command("MoveCursor({0}, {1})".format(ypos, xpos).encode("ascii"))

def
move_to
(
self
,
ypos
,
xpos
)
:
xpos
-=
1
ypos
-=
1
self
.
exec_command
(
"MoveCursor({0}, {1})"
.
format
(
ypos
,
xpos
)
.
encode
(
"ascii"
)
)

move the cursor to the given co - ordinates . Co - ordinates are 1 based as listed in the status area of the terminal .


def fill_field(self, ypos, xpos, tosend, length):
        """
            clears the field at the position given and inserts the string
            `tosend`

            tosend: the string to insert
            length: the length of the field

            Co-ordinates are 1 based, as listed in the status area of the
            terminal.

            raises: FieldTruncateError if `tosend` is longer than
                `length`.
        """
        if length < len(tosend):
            raise FieldTruncateError('length limit %d, but got "%s"' % (length, tosend))
        if xpos is not None and ypos is not None:
            self.move_to(ypos, xpos)
        self.delete_field()
        self.send_string(tosend)

def
fill_field
(
self
,
ypos
,
xpos
,
tosend
,
length
)
:
if
length
<
len
(
tosend
)
:
raise
FieldTruncateError
(
'length limit %d, but got "%s"'
%
(
length
,
tosend
)
)
if
xpos
is
not
and
ypos
is
not
:
self
.
move_to
(
ypos
,
xpos
)
self
.
delete_field
(
)
self
.
send_string
(
tosend
)

clears the field at the position given and inserts the string tosend


def from_func(cls, func, variables, vartype, name=None):
        """Construct a constraint from a validation function.

        Args:
            func (function):
                Function that evaluates True when the variables satisfy the constraint.

            variables (iterable):
                Iterable of variable labels.

            vartype (:class:`~dimod.Vartype`/str/set):
                Variable type for the constraint. Accepted input values:

                * :attr:`~dimod.Vartype.SPIN`, ``'SPIN'``, ``{-1, 1}``
                * :attr:`~dimod.Vartype.BINARY`, ``'BINARY'``, ``{0, 1}``

            name (string, optional, default='Constraint'):
                Name for the constraint.

        Examples:
            This example creates a constraint that binary variables `a` and `b`
            are not equal.

            >>> import dwavebinarycsp
            >>> import operator
            >>> const = dwavebinarycsp.Constraint.from_func(operator.ne, ['a', 'b'], 'BINARY')
            >>> print(const.name)
            Constraint
            >>> (0, 1) in const.configurations
            True

            This example creates a constraint that :math:`out = NOT(x)`
            for spin variables.

            >>> import dwavebinarycsp
            >>> def not_(y, x):  # y=NOT(x) for spin variables
            ...     return (y == -x)
            ...
            >>> const = dwavebinarycsp.Constraint.from_func(
            ...               not_,
            ...               ['out', 'in'],
            ...               {1, -1},
            ...               name='not_spin')
            >>> print(const.name)
            not_spin
            >>> (1, -1) in const.configurations
            True

        """
        variables = tuple(variables)

        configurations = frozenset(config
                                   for config in itertools.product(vartype.value, repeat=len(variables))
                                   if func(*config))

        return cls(func, configurations, variables, vartype, name)

def from_func ( cls , func , variables , vartype , name = None ) : variables = tuple ( variables ) configurations = frozenset ( config for config in itertools . product ( vartype . value , repeat = len ( variables ) ) if func ( * config ) ) return cls ( func , configurations , variables , vartype , name )

Construct a constraint from a validation function .


def from_configurations(cls, configurations, variables, vartype, name=None):
        """Construct a constraint from valid configurations.

        Args:
            configurations (iterable[tuple]):
                Valid configurations of the variables. Each configuration is a tuple of variable
                assignments ordered by :attr:`~Constraint.variables`.

            variables (iterable):
                Iterable of variable labels.

            vartype (:class:`~dimod.Vartype`/str/set):
                Variable type for the constraint. Accepted input values:

                * :attr:`~dimod.Vartype.SPIN`, ``'SPIN'``, ``{-1, 1}``
                * :attr:`~dimod.Vartype.BINARY`, ``'BINARY'``, ``{0, 1}``

            name (string, optional, default='Constraint'):
                Name for the constraint.

        Examples:

            This example creates a constraint that variables `a` and `b` are not equal.

            >>> import dwavebinarycsp
            >>> const = dwavebinarycsp.Constraint.from_configurations([(0, 1), (1, 0)],
            ...                   ['a', 'b'], dwavebinarycsp.BINARY)
            >>> print(const.name)
            Constraint
            >>> (0, 0) in const.configurations   # Order matches variables: a,b
            False

            This example creates a constraint based on specified valid configurations
            that represents an OR gate for spin variables.

            >>> import dwavebinarycsp
            >>> const = dwavebinarycsp.Constraint.from_configurations(
            ...           [(-1, -1, -1), (1, -1, 1), (1, 1, -1), (1, 1, 1)],
            ...           ['y', 'x1', 'x2'],
            ...           dwavebinarycsp.SPIN, name='or_spin')
            >>> print(const.name)
            or_spin
            >>> (1, 1, -1) in const.configurations   # Order matches variables: y,x1,x2
            True

        """
        def func(*args): return args in configurations

        return cls(func, configurations, variables, vartype, name)

def from_configurations ( cls , configurations , variables , vartype , name = None ) : def func ( * args ) : return args in configurations return cls ( func , configurations , variables , vartype , name )

Construct a constraint from valid configurations .


def check(self, solution):
        """Check that a solution satisfies the constraint.

        Args:
            solution (container):
                An assignment for the variables in the constraint.

        Returns:
            bool: True if the solution satisfies the constraint; otherwise False.

        Examples:
            This example creates a constraint that :math:`a \\ne b` on binary variables
            and tests it for two candidate solutions, with additional unconstrained
            variable c.

            >>> import dwavebinarycsp
            >>> const = dwavebinarycsp.Constraint.from_configurations([(0, 1), (1, 0)],
            ...             ['a', 'b'], dwavebinarycsp.BINARY)
            >>> solution = {'a': 1, 'b': 1, 'c': 0}
            >>> const.check(solution)
            False
            >>> solution = {'a': 1, 'b': 0, 'c': 0}
            >>> const.check(solution)
            True

        """
        return self.func(*(solution[v] for v in self.variables))

def check ( self , solution ) : return self . func ( * ( solution [ v ] for v in self . variables ) )

Check that a solution satisfies the constraint .


def fix_variable(self, v, value):
        """Fix the value of a variable and remove it from the constraint.

        Args:
            v (variable):
                Variable in the constraint to be set to a constant value.

            val (int):
                Value assigned to the variable. Values must match the :class:`.Vartype` of the
                constraint.

        Examples:
            This example creates a constraint that :math:`a \\ne b` on binary variables,
            fixes variable a to 0, and tests two candidate solutions.

            >>> import dwavebinarycsp
            >>> const = dwavebinarycsp.Constraint.from_func(operator.ne,
            ...             ['a', 'b'], dwavebinarycsp.BINARY)
            >>> const.fix_variable('a', 0)
            >>> const.check({'b': 1})
            True
            >>> const.check({'b': 0})
            False

        """
        variables = self.variables
        try:
            idx = variables.index(v)
        except ValueError:
            raise ValueError("given variable {} is not part of the constraint".format(v))

        if value not in self.vartype.value:
            raise ValueError("expected value to be in {}, received {} instead".format(self.vartype.value, value))

        configurations = frozenset(config[:idx] + config[idx + 1:]  # exclude the fixed var
                                   for config in self.configurations
                                   if config[idx] == value)

        if not configurations:
            raise UnsatError("fixing {} to {} makes this constraint unsatisfiable".format(v, value))

        variables = variables[:idx] + variables[idx + 1:]

        self.configurations = configurations
        self.variables = variables

        def func(*args): return args in configurations
        self.func = func

        self.name = '{} ({} fixed to {})'.format(self.name, v, value)

Fix the value of a variable and remove it from the constraint .


def flip_variable(self, v):
        """Flip a variable in the constraint.

        Args:
            v (variable):
                Variable in the constraint to take the complementary value of its
                construction value.

        Examples:
            This example creates a constraint that :math:`a = b` on binary variables
            and flips variable a.

            >>> import dwavebinarycsp
            >>> const = dwavebinarycsp.Constraint.from_func(operator.eq,
            ...             ['a', 'b'], dwavebinarycsp.BINARY)
            >>> const.check({'a': 0, 'b': 0})
            True
            >>> const.flip_variable('a')
            >>> const.check({'a': 1, 'b': 0})
            True
            >>> const.check({'a': 0, 'b': 0})
            False

        """
        try:
            idx = self.variables.index(v)
        except ValueError:
            raise ValueError("variable {} is not a variable in constraint {}".format(v, self.name))

        if self.vartype is dimod.BINARY:

            original_func = self.func

            def func(*args):
                new_args = list(args)
                new_args[idx] = 1 - new_args[idx]  # negate v
                return original_func(*new_args)

            self.func = func

            self.configurations = frozenset(config[:idx] + (1 - config[idx],) + config[idx + 1:]
                                            for config in self.configurations)

        else:  # SPIN

            original_func = self.func

            def func(*args):
                new_args = list(args)
                new_args[idx] = -new_args[idx]  # negate v
                return original_func(*new_args)

            self.func = func

            self.configurations = frozenset(config[:idx] + (-config[idx],) + config[idx + 1:]
                                            for config in self.configurations)

        self.name = '{} ({} flipped)'.format(self.name, v)


def copy(self):
        """Create a copy.

        Examples:
            This example copies constraint :math:`a \\ne b` and tests a solution
            on the copied constraint.

            >>> import dwavebinarycsp
            >>> import operator
            >>> const = dwavebinarycsp.Constraint.from_func(operator.ne,
            ...             ['a', 'b'], 'BINARY')
            >>> const2 = const.copy()
            >>> const2 is const
            False
            >>> const2.check({'a': 1, 'b': 1})
            False

        """
        # each object is itself immutable (except the function)
        return self.__class__(self.func, self.configurations, self.variables, self.vartype, name=self.name)

def copy ( self ) : return self . __class__ ( self . func , self . configurations , self . variables , self . vartype , name = self . name )


def projection(self, variables):
        """Create a new constraint that is the projection onto a subset of the variables.

        Args:
            variables (iterable):
                Subset of the constraint's variables.

        Returns:
            :obj:`.Constraint`: A new constraint over a subset of the variables.

        Examples:

            >>> import dwavebinarycsp
            ...
            >>> const = dwavebinarycsp.Constraint.from_configurations([(0, 0), (0, 1)],
            ...                                                       ['a', 'b'],
            ...                                                       dwavebinarycsp.BINARY)
            >>> proj = const.projection(['a'])
            >>> proj.variables
            ['a']
            >>> proj.configurations
            {(0,)}

        """
        # resolve iterables or mutability problems by casting the variables to a set
        variables = set(variables)

        if not variables.issubset(self.variables):
            raise ValueError("Cannot project to variables not in the constraint.")

        idxs = [i for i, v in enumerate(self.variables) if v in variables]

        configurations = frozenset(tuple(config[i] for i in idxs) for config in self.configurations)
        variables = tuple(self.variables[i] for i in idxs)

        return self.from_configurations(configurations, variables, self.vartype)

Create a new constraint that is the projection onto a subset of the variables .


def assert_penaltymodel_factory_available():
    """For `dwavebinarycsp` to be functional, at least one penalty model factory
    has to be installed. See discussion in setup.py for details.
    """

    from pkg_resources import iter_entry_points
    from penaltymodel.core import FACTORY_ENTRYPOINT
    from itertools import chain

    supported = ('maxgap', 'mip')
    factories = chain(*(iter_entry_points(FACTORY_ENTRYPOINT, name) for name in supported))

    try:
        next(factories)
    except StopIteration:
        raise AssertionError(
            "To use 'dwavebinarycsp', at least one penaltymodel factory must be installed. "
            "Try {}.".format(
                " or ".join("'pip install dwavebinarycsp[{}]'".format(name) for name in supported)
            ))

For dwavebinarycsp to be functional at least one penalty model factory has to be installed . See discussion in setup . py for details .


def add_constraint(self, constraint, variables=tuple()):
        """Add a constraint.

        Args:
            constraint (function/iterable/:obj:`.Constraint`):
                Constraint definition in one of the supported formats:

                1. Function, with input arguments matching the order and
                   :attr:`~.ConstraintSatisfactionProblem.vartype` type of the `variables`
                   argument, that evaluates True when the constraint is satisfied.
                2. List explicitly specifying each allowed configuration as a tuple.
                3. :obj:`.Constraint` object built either explicitly or by :mod:`dwavebinarycsp.factories`.

            variables(iterable):
                Variables associated with the constraint. Not required when `constraint` is
                a :obj:`.Constraint` object.

        Examples:
            This example defines a function that evaluates True when the constraint is satisfied.
            The function's input arguments match the order and type of the `variables` argument.

            >>> import dwavebinarycsp
            >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
            >>> def all_equal(a, b, c):  # works for both dwavebinarycsp.BINARY and dwavebinarycsp.SPIN
            ...     return (a == b) and (b == c)
            >>> csp.add_constraint(all_equal, ['a', 'b', 'c'])
            >>> csp.check({'a': 0, 'b': 0, 'c': 0})
            True
            >>> csp.check({'a': 0, 'b': 0, 'c': 1})
            False

            This example explicitly lists allowed configurations.

            >>> import dwavebinarycsp
            >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.SPIN)
            >>> eq_configurations = {(-1, -1), (1, 1)}
            >>> csp.add_constraint(eq_configurations, ['v0', 'v1'])
            >>> csp.check({'v0': -1, 'v1': +1})
            False
            >>> csp.check({'v0': -1, 'v1': -1})
            True

            This example uses a :obj:`.Constraint` object built by :mod:`dwavebinarycsp.factories`.

            >>> import dwavebinarycsp
            >>> import dwavebinarycsp.factories.constraint.gates as gates
            >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
            >>> csp.add_constraint(gates.and_gate(['a', 'b', 'c']))  # add an AND gate
            >>> csp.add_constraint(gates.xor_gate(['a', 'c', 'd']))  # add an XOR gate
            >>> csp.check({'a': 1, 'b': 0, 'c': 0, 'd': 1})
            True

        """
        if isinstance(constraint, Constraint):
            if variables and (tuple(variables) != constraint.variables):
                raise ValueError("mismatched variables and Constraint")
        elif isinstance(constraint, Callable):
            constraint = Constraint.from_func(constraint, variables, self.vartype)
        elif isinstance(constraint, Iterable):
            constraint = Constraint.from_configurations(constraint, variables, self.vartype)
        else:
            raise TypeError("Unknown constraint type given")

        self.constraints.append(constraint)
        for v in constraint.variables:
            self.variables[v].append(constraint)

def
add_constraint
(
self
,
constraint
,
variables
=
tuple
(
)
)
:
if
isinstance
(
constraint
,
Constraint
)
:
if
variables
and
(
tuple
(
variables
)
!=
constraint
.
variables
)
:
raise
ValueError
(
"mismatched variables and Constraint"
)
elif
isinstance
(
constraint
,
Callable
)
:
constraint
=
Constraint
.
from_func
(
constraint
,
variables
,
self
.
vartype
)
elif
isinstance
(
constraint
,
Iterable
)
:
constraint
=
Constraint
.
from_configurations
(
constraint
,
variables
,
self
.
vartype
)
else
:
raise
TypeError
(
"Unknown constraint type given"
)
self
.
constraints
.
append
(
constraint
)
for
v
in
constraint
.
variables
:
self
.
variables
[
v
]
.
append
(
constraint
)


def stitch(csp, min_classical_gap=2.0, max_graph_size=8):
    """Build a binary quadratic model with minimal energy levels at solutions to the specified constraint satisfaction
    problem.

    Args:
        csp (:obj:`.ConstraintSatisfactionProblem`):
            Constraint satisfaction problem.

        min_classical_gap (float, optional, default=2.0):
            Minimum energy gap from ground. Each constraint violated by the solution increases
            the energy level of the binary quadratic model by at least this much relative
            to ground energy.

        max_graph_size (int, optional, default=8):
            Maximum number of variables in the binary quadratic model that can be used to
            represent a single constraint.

    Returns:
        :class:`~dimod.BinaryQuadraticModel`

    Notes:
        For a `min_classical_gap` > 2 or constraints with more than two variables, requires
        access to factories from the penaltymodel_ ecosystem to construct the binary quadratic
        model.

    .. _penaltymodel: https://github.com/dwavesystems/penaltymodel

    Examples:
        This example creates a binary-valued constraint satisfaction problem
        with two constraints, :math:`a = b` and :math:`b \\ne c`, and builds
        a binary quadratic model with a minimum energy level of -2 such that
        each constraint violation by a solution adds the default minimum energy gap.

        >>> import dwavebinarycsp
        >>> import operator
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(operator.eq, ['a', 'b'])  # a == b
        >>> csp.add_constraint(operator.ne, ['b', 'c'])  # b != c
        >>> bqm = dwavebinarycsp.stitch(csp)
        >>> bqm.energy({'a': 0, 'b': 0, 'c': 1})  # satisfies csp
        -2.0
        >>> bqm.energy({'a': 0, 'b': 0, 'c': 0})  # violates one constraint
        0.0
        >>> bqm.energy({'a': 1, 'b': 0, 'c': 0}) # violates two constraints
        2.0

        This example creates a binary-valued constraint satisfaction problem
        with two constraints, :math:`a = b` and :math:`b \\ne c`, and builds
        a binary quadratic model with a minimum energy gap of 4.
        Note that in this case the conversion to binary quadratic model adds two
        ancillary variables that must be minimized over when solving.

        >>> import dwavebinarycsp
        >>> import operator
        >>> import itertools
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(operator.eq, ['a', 'b'])  # a == b
        >>> csp.add_constraint(operator.ne, ['b', 'c'])  # b != c
        >>> bqm = dwavebinarycsp.stitch(csp, min_classical_gap=4.0)
        >>> list(bqm)   # # doctest: +SKIP
        ['a', 'aux1', 'aux0', 'b', 'c']
        >>> min([bqm.energy({'a': 0, 'b': 0, 'c': 1, 'aux0': aux0, 'aux1': aux1}) for
        ... aux0, aux1 in list(itertools.product([0, 1], repeat=2))])  # satisfies csp
        -6.0
        >>> min([bqm.energy({'a': 0, 'b': 0, 'c': 0, 'aux0': aux0, 'aux1': aux1}) for
        ... aux0, aux1 in list(itertools.product([0, 1], repeat=2))])  # violates one constraint
        -2.0
        >>> min([bqm.energy({'a': 1, 'b': 0, 'c': 0, 'aux0': aux0, 'aux1': aux1}) for
        ... aux0, aux1 in list(itertools.product([0, 1], repeat=2))])  # violates two constraints
        2.0

        This example finds for the previous example the minimum graph size.

        >>> import dwavebinarycsp
        >>> import operator
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(operator.eq, ['a', 'b'])  # a == b
        >>> csp.add_constraint(operator.ne, ['b', 'c'])  # b != c
        >>> for n in range(8, 1, -1):
        ...     try:
        ...         bqm = dwavebinarycsp.stitch(csp, min_classical_gap=4.0, max_graph_size=n)
        ...     except dwavebinarycsp.exceptions.ImpossibleBQM:
        ...         print(n+1)
        ...
        3

    """

    # ensure we have penaltymodel factory available
    try:
        dwavebinarycsp.assert_penaltymodel_factory_available()
    except AssertionError as e:
        raise RuntimeError(e)

    def aux_factory():
        for i in count():
            yield 'aux{}'.format(i)

    aux = aux_factory()

    bqm = dimod.BinaryQuadraticModel.empty(csp.vartype)

    # developer note: we could cache them and relabel, for now though let's do the simple thing
    # penalty_models = {}
    for const in csp.constraints:
        configurations = const.configurations

        if len(const.variables) > max_graph_size:
            msg = ("The given csp contains a constraint {const} with {num_var} variables. "
                   "This cannot be mapped to a graph with {max_graph_size} nodes. "
                   "Consider checking whether your constraint is irreducible."
                   "").format(const=const, num_var=len(const.variables), max_graph_size=max_graph_size)
            raise ImpossibleBQM(msg)

        pmodel = None

        if len(const) == 0:
            # empty constraint
            continue

        if min_classical_gap <= 2.0:
            if len(const) == 1 and max_graph_size >= 1:
                bqm.update(_bqm_from_1sat(const))
                continue
            elif len(const) == 2 and max_graph_size >= 2:
                bqm.update(_bqm_from_2sat(const))
                continue

        # developer note: we could cache them and relabel, for now though let's do the simple thing
        # if configurations in penalty_models:
        #     raise NotImplementedError

        for G in iter_complete_graphs(const.variables, max_graph_size + 1, aux):

            # construct a specification
            spec = pm.Specification(
                graph=G,
                decision_variables=const.variables,
                feasible_configurations=configurations,
                min_classical_gap=min_classical_gap,
                vartype=csp.vartype
            )

            # try to use the penaltymodel ecosystem
            try:
                pmodel = pm.get_penalty_model(spec)
            except pm.ImpossiblePenaltyModel:
                # hopefully adding more variables will make it possible
                continue

            if pmodel.classical_gap >= min_classical_gap:
                break

        # developer note: we could cache them and relabel, for now though let's do the simple thing
        # penalty_models[configurations] = pmodel

        else:
            msg = ("No penalty model can be build for constraint {}".format(const))
            raise ImpossibleBQM(msg)

        bqm.update(pmodel.model)

    return bqm

Build a binary quadratic model with minimal energy levels at solutions to the specified constraint satisfaction problem .


def _bqm_from_1sat(constraint):
    """create a bqm for a constraint with only one variable

    bqm will have exactly classical gap 2.
    """
    configurations = constraint.configurations
    num_configurations = len(configurations)

    bqm = dimod.BinaryQuadraticModel.empty(constraint.vartype)

    if num_configurations == 1:
        val, = next(iter(configurations))
        v, = constraint.variables
        bqm.add_variable(v, -1 if val > 0 else +1, vartype=dimod.SPIN)
    else:
        bqm.add_variables_from((v, 0.0) for v in constraint.variables)

    return bqm

def _bqm_from_1sat ( constraint ) : configurations = constraint . configurations num_configurations = len ( configurations ) bqm = dimod . BinaryQuadraticModel . empty ( constraint . vartype ) if num_configurations == 1 : val , = next ( iter ( configurations ) ) v , = constraint . variables bqm . add_variable ( v , - 1 if val > 0 else + 1 , vartype = dimod . SPIN ) else : bqm . add_variables_from ( ( v , 0.0 ) for v in constraint . variables ) return bqm


def _bqm_from_2sat(constraint):
    """create a bqm for a constraint with two variables.

    bqm will have exactly classical gap 2.
    """
    configurations = constraint.configurations
    variables = constraint.variables
    vartype = constraint.vartype
    u, v = constraint.variables

    # if all configurations are present, then nothing is infeasible and the bqm is just all
    # 0.0s
    if len(configurations) == 4:
        return dimod.BinaryQuadraticModel.empty(constraint.vartype)

    # check if the constraint is irreducible, and if so, build the bqm for its two
    # components
    components = irreducible_components(constraint)
    if len(components) > 1:
        const0 = Constraint.from_configurations(((config[0],) for config in configurations),
                                                (u,), vartype)
        const1 = Constraint.from_configurations(((config[1],) for config in configurations),
                                                (v,), vartype)
        bqm = _bqm_from_1sat(const0)
        bqm.update(_bqm_from_1sat(const1))
        return bqm

    assert len(configurations) > 1, "single configurations should be irreducible"

    # if it is not irreducible, and there are infeasible configurations, then it is time to
    # start building a bqm
    bqm = dimod.BinaryQuadraticModel.empty(vartype)

    # if the constraint is not irreducible and has two configurations, then it is either eq or ne
    if all(operator.eq(*config) for config in configurations):
        bqm.add_interaction(u, v, -1, vartype=dimod.SPIN)  # equality
    elif all(operator.ne(*config) for config in configurations):
        bqm.add_interaction(u, v, +1, vartype=dimod.SPIN)  # inequality
    elif (1, 1) not in configurations:
        bqm.add_interaction(u, v, 2, vartype=dimod.BINARY)  # penalize (1, 1)
    elif (-1, +1) not in configurations and (0, 1) not in configurations:
        bqm.add_interaction(u, v, -2, vartype=dimod.BINARY)
        bqm.add_variable(v, 2, vartype=dimod.BINARY)
    elif (+1, -1) not in configurations and (1, 0) not in configurations:
        bqm.add_interaction(u, v, -2, vartype=dimod.BINARY)
        bqm.add_variable(u, 2, vartype=dimod.BINARY)
    else:
        # (0, 0) not in configurations
        bqm.add_interaction(u, v, 2, vartype=dimod.BINARY)
        bqm.add_variable(u, -2, vartype=dimod.BINARY)
        bqm.add_variable(v, -2, vartype=dimod.BINARY)

    return bqm

create a bqm for a constraint with two variables .


def iter_complete_graphs(start, stop, factory=None):
    """Iterate over complete graphs.

    Args:
        start (int/iterable):
            Define the size of the starting graph.
            If an int, the nodes will be index-labeled, otherwise should be an iterable of node
            labels.

        stop (int):
            Stops yielding graphs when the size equals stop.

        factory (iterator, optional):
            If provided, nodes added will be labeled according to the values returned by factory.
            Otherwise the extra nodes will be index-labeled.

    Yields:
        :class:`nx.Graph`

    """
    _, nodes = start
    nodes = list(nodes)  # we'll be appending

    if factory is None:
        factory = count()

    while len(nodes) < stop:
        # we need to construct a new graph each time, this is actually faster than copy and add
        # the new edges in any case
        G = nx.complete_graph(nodes)
        yield G

        v = next(factory)
        while v in G:
            v = next(factory)

        nodes.append(v)

def iter_complete_graphs ( start , stop , factory = None ) : _ , nodes = start nodes = list ( nodes ) if factory is None : factory = count ( ) while len ( nodes ) < stop : G = nx . complete_graph ( nodes ) yield G v = next ( factory ) while v in G : v = next ( factory ) nodes . append ( v )


def load_cnf(fp):
    """Load a constraint satisfaction problem from a .cnf file.

    Args:
        fp (file, optional):
            `.write()`-supporting `file object`_ DIMACS CNF formatted_ file.

    Returns:
        :obj:`.ConstraintSatisfactionProblem` a binary-valued SAT problem.

    Examples:

        >>> import dwavebinarycsp as dbcsp
        ...
        >>> with open('test.cnf', 'r') as fp: # doctest: +SKIP
        ...     csp = dbcsp.cnf.load_cnf(fp)

    .. _file object: https://docs.python.org/3/glossary.html#term-file-object

    .. _formatted: http://www.satcompetition.org/2009/format-benchmarks2009.html

    """

    fp = iter(fp)  # handle lists/tuples/etc

    csp = ConstraintSatisfactionProblem(dimod.BINARY)

    # first look for the problem
    num_clauses = num_variables = 0
    problem_pattern = re.compile(_PROBLEM_REGEX)
    for line in fp:
        matches = problem_pattern.findall(line)
        if matches:
            if len(matches) > 1:
                raise ValueError
            nv, nc = matches[0]
            num_variables, num_clauses = int(nv), int(nc)
            break

    # now parse the clauses, picking up where we left off looking for the header
    clause_pattern = re.compile(_CLAUSE_REGEX)
    for line in fp:
        if clause_pattern.match(line) is not None:
            clause = [int(v) for v in line.split(' ')[:-1]]  # line ends with a trailing 0

            # -1 is the notation for NOT(1)
            variables = [abs(v) for v in clause]

            f = _cnf_or(clause)

            csp.add_constraint(f, variables)

    for v in range(1, num_variables+1):
        csp.add_variable(v)
    for v in csp.variables:
        if v > num_variables:
            msg = ("given .cnf file's header defines variables [1, {}] and {} clauses "
                   "but constraints a reference to variable {}").format(num_variables, num_clauses, v)
            raise ValueError(msg)

    if len(csp) != num_clauses:
        msg = ("given .cnf file's header defines {} "
               "clauses but the file contains {}").format(num_clauses, len(csp))
        raise ValueError(msg)

    return csp

Load a constraint satisfaction problem from a . cnf file .


def and_gate(variables, vartype=dimod.BINARY, name='AND'):
    """AND gate.

    Args:
        variables (list): Variable labels for the and gate as `[in1, in2, out]`,
            where `in1, in2` are inputs and `out` the gate's output.
        vartype (Vartype, optional, default='BINARY'): Variable type. Accepted
            input values:

            * Vartype.SPIN, 'SPIN', {-1, 1}
            * Vartype.BINARY, 'BINARY', {0, 1}
        name (str, optional, default='AND'): Name for the constraint.

    Returns:
        Constraint(:obj:`.Constraint`): Constraint that is satisfied when its variables are
        assigned values that match the valid states of an AND gate.

    Examples:
        >>> import dwavebinarycsp
        >>> import dwavebinarycsp.factories.constraint.gates as gates
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(gates.and_gate(['a', 'b', 'c'], name='AND1'))
        >>> csp.check({'a': 1, 'b': 0, 'c': 0})
        True
    """

    variables = tuple(variables)

    if vartype is dimod.BINARY:
        configurations = frozenset([(0, 0, 0),
                                    (0, 1, 0),
                                    (1, 0, 0),
                                    (1, 1, 1)])

        def func(in1, in2, out): return (in1 and in2) == out

    else:
        # SPIN, vartype is checked by the decorator
        configurations = frozenset([(-1, -1, -1),
                                    (-1, +1, -1),
                                    (+1, -1, -1),
                                    (+1, +1, +1)])

        def func(in1, in2, out): return ((in1 > 0) and (in2 > 0)) == (out > 0)

    return Constraint(func, configurations, variables, vartype=vartype, name=name)

def and_gate ( variables , vartype = dimod . BINARY , name = 'AND' ) : variables = tuple ( variables ) if vartype is dimod . BINARY : configurations = frozenset ( [ ( 0 , 0 , 0 ) , ( 0 , 1 , 0 ) , ( 1 , 0 , 0 ) , ( 1 , 1 , 1 ) ] ) def func ( in1 , in2 , out ) : return ( in1 and in2 ) == out else : configurations = frozenset ( [ ( - 1 , - 1 , - 1 ) , ( - 1 , + 1 , - 1 ) , ( + 1 , - 1 , - 1 ) , ( + 1 , + 1 , + 1 ) ] ) def func ( in1 , in2 , out ) : return ( ( in1 > 0 ) and ( in2 > 0 ) ) == ( out > 0 ) return Constraint ( func , configurations , variables , vartype = vartype , name = name )


def xor_gate(variables, vartype=dimod.BINARY, name='XOR'):
    """XOR gate.

    Args:
        variables (list): Variable labels for the and gate as `[in1, in2, out]`,
            where `in1, in2` are inputs and `out` the gate's output.
        vartype (Vartype, optional, default='BINARY'): Variable type. Accepted
            input values:

            * Vartype.SPIN, 'SPIN', {-1, 1}
            * Vartype.BINARY, 'BINARY', {0, 1}
        name (str, optional, default='XOR'): Name for the constraint.

    Returns:
        Constraint(:obj:`.Constraint`): Constraint that is satisfied when its variables are
        assigned values that match the valid states of an XOR gate.

    Examples:
        >>> import dwavebinarycsp
        >>> import dwavebinarycsp.factories.constraint.gates as gates
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(gates.xor_gate(['x', 'y', 'z'], name='XOR1'))
        >>> csp.check({'x': 1, 'y': 1, 'z': 1})
        False
    """

    variables = tuple(variables)
    if vartype is dimod.BINARY:
        configs = frozenset([(0, 0, 0),
                             (0, 1, 1),
                             (1, 0, 1),
                             (1, 1, 0)])

        def func(in1, in2, out): return (in1 != in2) == out

    else:
        # SPIN, vartype is checked by the decorator
        configs = frozenset([(-1, -1, -1),
                             (-1, +1, +1),
                             (+1, -1, +1),
                             (+1, +1, -1)])

        def func(in1, in2, out): return ((in1 > 0) != (in2 > 0)) == (out > 0)

    return Constraint(func, configs, variables, vartype=vartype, name=name)

def xor_gate ( variables , vartype = dimod . BINARY , name = 'XOR' ) : variables = tuple ( variables ) if vartype is dimod . BINARY : configs = frozenset ( [ ( 0 , 0 , 0 ) , ( 0 , 1 , 1 ) , ( 1 , 0 , 1 ) , ( 1 , 1 , 0 ) ] ) def func ( in1 , in2 , out ) : return ( in1 != in2 ) == out else : configs = frozenset ( [ ( - 1 , - 1 , - 1 ) , ( - 1 , + 1 , + 1 ) , ( + 1 , - 1 , + 1 ) , ( + 1 , + 1 , - 1 ) ] ) def func ( in1 , in2 , out ) : return ( ( in1 > 0 ) != ( in2 > 0 ) ) == ( out > 0 ) return Constraint ( func , configs , variables , vartype = vartype , name = name )


def halfadder_gate(variables, vartype=dimod.BINARY, name='HALF_ADDER'):
    """Half adder.

    Args:
        variables (list): Variable labels for the and gate as `[in1, in2, sum, carry]`,
            where `in1, in2` are inputs to be added and `sum` and 'carry' the resultant
            outputs.
        vartype (Vartype, optional, default='BINARY'): Variable type. Accepted
            input values:

            * Vartype.SPIN, 'SPIN', {-1, 1}
            * Vartype.BINARY, 'BINARY', {0, 1}
        name (str, optional, default='HALF_ADDER'): Name for the constraint.

    Returns:
        Constraint(:obj:`.Constraint`): Constraint that is satisfied when its variables are
        assigned values that match the valid states of a Boolean half adder.

    Examples:
        >>> import dwavebinarycsp
        >>> import dwavebinarycsp.factories.constraint.gates as gates
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(gates.halfadder_gate(['a', 'b', 'total', 'carry'], name='HA1'))
        >>> csp.check({'a': 1, 'b': 1, 'total': 0, 'carry': 1})
        True

    """

    variables = tuple(variables)

    if vartype is dimod.BINARY:
        configs = frozenset([(0, 0, 0, 0),
                             (0, 1, 1, 0),
                             (1, 0, 1, 0),
                             (1, 1, 0, 1)])

    else:
        # SPIN, vartype is checked by the decorator
        configs = frozenset([(-1, -1, -1, -1),
                             (-1, +1, +1, -1),
                             (+1, -1, +1, -1),
                             (+1, +1, -1, +1)])

    def func(augend, addend, sum_, carry):
        total = (augend > 0) + (addend > 0)
        if total == 0:
            return (sum_ <= 0) and (carry <= 0)
        elif total == 1:
            return (sum_ > 0) and (carry <= 0)
        elif total == 2:
            return (sum_ <= 0) and (carry > 0)
        else:
            raise ValueError("func recieved unexpected values")

    return Constraint(func, configs, variables, vartype=vartype, name=name)

def
halfadder_gate
(
variables
,
vartype
=
dimod
.
BINARY
,
name
=
'HALF_ADDER'
)
:
variables
=
tuple
(
variables
)
if
vartype
is
dimod
.
BINARY
:
configs
=
frozenset
(
[
(
0
,
0
,
0
,
0
)
,
(
0
,
1
,
1
,
0
)
,
(
1
,
0
,
1
,
0
)
,
(
1
,
1
,
0
,
1
)
]
)
else
:
configs
=
frozenset
(
[
(
1
,
1
,
1
,
1
)
,
(
1
,
+
1
,
+
1
,
1
)
,
(
+
1
,
1
,
+
1
,
1
)
,
(
+
1
,
+
1
,
1
,
+
1
)
]
)
def
func
(
augend
,
addend
,
sum_
,
carry
)
:
total
=
(
augend
>
0
)
+
(
addend
>
0
)
if
total
==
0
:
return
(
sum_
<=
0
)
and
(
carry
<=
0
)
elif
total
==
1
:
return
(
sum_
>
0
)
and
(
carry
<=
0
)
elif
total
==
2
:
return
(
sum_
<=
0
)
and
(
carry
>
0
)
else
:
raise
ValueError
(
"func recieved unexpected values"
)
return
Constraint
(
func
,
configs
,
variables
,
vartype
=
vartype
,
name
=
name
)


def fulladder_gate(variables, vartype=dimod.BINARY, name='FULL_ADDER'):
    """Full adder.

    Args:
        variables (list): Variable labels for the and gate as `[in1, in2, in3, sum, carry]`,
            where `in1, in2, in3` are inputs to be added and `sum` and 'carry' the resultant
            outputs.
        vartype (Vartype, optional, default='BINARY'): Variable type. Accepted
            input values:

            * Vartype.SPIN, 'SPIN', {-1, 1}
            * Vartype.BINARY, 'BINARY', {0, 1}
        name (str, optional, default='FULL_ADDER'): Name for the constraint.

    Returns:
        Constraint(:obj:`.Constraint`): Constraint that is satisfied when its variables are
        assigned values that match the valid states of a Boolean full adder.

    Examples:
        >>> import dwavebinarycsp
        >>> import dwavebinarycsp.factories.constraint.gates as gates
        >>> csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)
        >>> csp.add_constraint(gates.fulladder_gate(['a', 'b', 'c_in', 'total', 'c_out'], name='FA1'))
        >>> csp.check({'a': 1, 'b': 0, 'c_in': 1, 'total': 0, 'c_out': 1})
        True

    """

    variables = tuple(variables)

    if vartype is dimod.BINARY:
        configs = frozenset([(0, 0, 0, 0, 0),
                             (0, 0, 1, 1, 0),
                             (0, 1, 0, 1, 0),
                             (0, 1, 1, 0, 1),
                             (1, 0, 0, 1, 0),
                             (1, 0, 1, 0, 1),
                             (1, 1, 0, 0, 1),
                             (1, 1, 1, 1, 1)])

    else:
        # SPIN, vartype is checked by the decorator
        configs = frozenset([(-1, -1, -1, -1, -1),
                             (-1, -1, +1, +1, -1),
                             (-1, +1, -1, +1, -1),
                             (-1, +1, +1, -1, +1),
                             (+1, -1, -1, +1, -1),
                             (+1, -1, +1, -1, +1),
                             (+1, +1, -1, -1, +1),
                             (+1, +1, +1, +1, +1)])

    def func(in1, in2, in3, sum_, carry):
        total = (in1 > 0) + (in2 > 0) + (in3 > 0)
        if total == 0:
            return (sum_ <= 0) and (carry <= 0)
        elif total == 1:
            return (sum_ > 0) and (carry <= 0)
        elif total == 2:
            return (sum_ <= 0) and (carry > 0)
        elif total == 3:
            return (sum_ > 0) and (carry > 0)
        else:
            raise ValueError("func recieved unexpected values")

    return Constraint(func, configs, variables, vartype=vartype, name=name)

def
fulladder_gate
(
variables
,
vartype
=
dimod
.
BINARY
,
name
=
'FULL_ADDER'
)
:
variables
=
tuple
(
variables
)
if
vartype
is
dimod
.
BINARY
:
configs
=
frozenset
(
[
(
0
,
0
,
0
,
0
,
0
)
,
(
0
,
0
,
1
,
1
,
0
)
,
(
0
,
1
,
0
,
1
,
0
)
,
(
0
,
1
,
1
,
0
,
1
)
,
(
1
,
0
,
0
,
1
,
0
)
,
(
1
,
0
,
1
,
0
,
1
)
,
(
1
,
1
,
0
,
0
,
1
)
,
(
1
,
1
,
1
,
1
,
1
)
]
)
else
:
configs
=
frozenset
(
[
(
1
,
1
,
1
,
1
,
1
)
,
(
1
,
1
,
+
1
,
+
1
,
1
)
,
(
1
,
+
1
,
1
,
+
1
,
1
)
,
(
1
,
+
1
,
+
1
,
1
,
+
1
)
,
(
+
1
,
1
,
1
,
+
1
,
1
)
,
(
+
1
,
1
,
+
1
,
1
,
+
1
)
,
(
+
1
,
+
1
,
1
,
1
,
+
1
)
,
(
+
1
,
+
1
,
+
1
,
+
1
,
+
1
)
]
)
def
func
(
in1
,
in2
,
in3
,
sum_
,
carry
)
:
total
=
(
in1
>
0
)
+
(
in2
>
0
)
+
(
in3
>
0
)
if
total
==
0
:
return
(
sum_
<=
0
)
and
(
carry
<=
0
)
elif
total
==
1
:
return
(
sum_
>
0
)
and
(
carry
<=
0
)
elif
total
==
2
:
return
(
sum_
<=
0
)
and
(
carry
>
0
)
elif
total
==
3
:
return
(
sum_
>
0
)
and
(
carry
>
0
)
else
:
raise
ValueError
(
"func recieved unexpected values"
)
return
Constraint
(
func
,
configs
,
variables
,
vartype
=
vartype
,
name
=
name
)


def random_xorsat(num_variables, num_clauses, vartype=dimod.BINARY, satisfiable=True):
    """Random XOR constraint satisfaction problem.

    Args:
        num_variables (integer): Number of variables (at least three).
        num_clauses (integer): Number of constraints that together constitute the
            constraint satisfaction problem.
        vartype (Vartype, optional, default='BINARY'): Variable type. Accepted
            input values:

            * Vartype.SPIN, 'SPIN', {-1, 1}
            * Vartype.BINARY, 'BINARY', {0, 1}
        satisfiable (bool, optional, default=True): True if the CSP can be satisfied.

    Returns:
        CSP (:obj:`.ConstraintSatisfactionProblem`): CSP that is satisfied when its variables
        are assigned values that satisfy a XOR satisfiability problem.

    Examples:
        This example creates a CSP with 5 variables and two random constraints and checks
        whether a particular assignment of variables satisifies it.

        >>> import dwavebinarycsp
        >>> import dwavebinarycsp.factories as sat
        >>> csp = sat.random_xorsat(5, 2)
        >>> csp.constraints    # doctest: +SKIP
        [Constraint.from_configurations(frozenset({(1, 0, 0), (1, 1, 1), (0, 1, 0), (0, 0, 1)}), (4, 3, 0),
         Vartype.BINARY, name='XOR (0 flipped)'),
         Constraint.from_configurations(frozenset({(1, 1, 0), (0, 1, 1), (0, 0, 0), (1, 0, 1)}), (2, 0, 4),
         Vartype.BINARY, name='XOR (2 flipped) (0 flipped)')]
        >>> csp.check({0: 1, 1: 0, 2: 0, 3: 1, 4: 1})       # doctest: +SKIP
        True

    """
    if num_variables < 3:
        raise ValueError("a xor problem needs at least 3 variables")
    if num_clauses > 8 * _nchoosek(num_variables, 3):  # 8 different negation patterns
        raise ValueError("too many clauses")

    # also checks the vartype argument
    csp = ConstraintSatisfactionProblem(vartype)

    variables = list(range(num_variables))

    constraints = set()

    if satisfiable:
        values = tuple(vartype.value)
        planted_solution = {v: choice(values) for v in variables}

        configurations = [(0, 0, 0), (0, 1, 1), (1, 0, 1), (1, 1, 0)]

        while len(constraints) < num_clauses:
            # because constraints are hashed on configurations/variables, and because the inputs
            # to xor can be swapped without loss of generality, we can order them
            x, y, z = sample(variables, 3)
            if y > x:
                x, y = y, x

            # get the constraint
            const = xor_gate([x, y, z], vartype=vartype)

            # pick (uniformly) a configuration and determine which variables we need to negate to
            # match the chosen configuration
            config = choice(configurations)

            for idx, v in enumerate(const.variables):
                if config[idx] != (planted_solution[v] > 0):
                    const.flip_variable(v)

            assert const.check(planted_solution)

            constraints.add(const)
    else:
        while len(constraints) < num_clauses:
            # because constraints are hashed on configurations/variables, and because the inputs
            # to xor can be swapped without loss of generality, we can order them
            x, y, z = sample(variables, 3)
            if y > x:
                x, y = y, x

            # get the constraint
            const = xor_gate([x, y, z], vartype=vartype)

            # randomly flip each variable in the constraint
            for idx, v in enumerate(const.variables):
                if random() > .5:
                    const.flip_variable(v)

            assert const.check(planted_solution)

            constraints.add(const)

    for const in constraints:
        csp.add_constraint(const)

    # in case any variables didn't make it in
    for v in variables:
        csp.add_variable(v)

    return csp

Random XOR constraint satisfaction problem .


def signature_matches(func, args=(), kwargs={}):
    """
    Work out if a function is callable with some args or not.
    """
    try:
        sig = inspect.signature(func)
        sig.bind(*args, **kwargs)
    except TypeError:
        return False
    else:
        return True

def signature_matches ( func , args = ( ) , kwargs = { } ) : try : sig = inspect . signature ( func ) sig . bind ( * args , ** kwargs ) except TypeError : return False else : return True

Work out if a function is callable with some args or not .


def last_arg_decorator(func):
    """
    Allows a function to be used as either a decorator with args, or called as
    a normal function.

    @last_arg_decorator
    def register_a_thing(foo, func, bar=True):
        ..

    # Called as a decorator
    @register_a_thing("abc", bar=False)
    def my_func():
        ...

    # Called as a normal function call
    def my_other_func():
        ...

    register_a_thing("def", my_other_func, bar=True)
    """
    @wraps(func)
    def decorator(*args, **kwargs):
        if signature_matches(func, args, kwargs):
            return func(*args, **kwargs)
        else:
            return lambda last: func(*(args + (last,)), **kwargs)
    return decorator

def last_arg_decorator ( func ) : @ wraps ( func ) def decorator ( * args , ** kwargs ) : if signature_matches ( func , args , kwargs ) : return func ( * args , ** kwargs ) else : return lambda last : func ( * ( args + ( last , ) ) , ** kwargs ) return decorator

Allows a function to be used as either a decorator with args or called as a normal function .


def register_chooser(self, chooser, **kwargs):
        """Adds a model chooser definition to the registry."""
        if not issubclass(chooser, Chooser):
            return self.register_simple_chooser(chooser, **kwargs)

        self.choosers[chooser.model] = chooser(**kwargs)
        return chooser

def register_chooser ( self , chooser , ** kwargs ) : if not issubclass ( chooser , Chooser ) : return self . register_simple_chooser ( chooser , ** kwargs ) self . choosers [ chooser . model ] = chooser ( ** kwargs ) return chooser

Adds a model chooser definition to the registry .


def register_simple_chooser(self, model, **kwargs):
        """
        Generates a model chooser definition from a model, and adds it to the
        registry.
        """
        name = '{}Chooser'.format(model._meta.object_name)
        attrs = {'model': model}
        attrs.update(kwargs)

        chooser = type(name, (Chooser,), attrs)
        self.register_chooser(chooser)

        return model

def register_simple_chooser ( self , model , ** kwargs ) : name = '{}Chooser' . format ( model . _meta . object_name ) attrs = { 'model' : model } attrs . update ( kwargs ) chooser = type ( name , ( Chooser , ) , attrs ) self . register_chooser ( chooser ) return model

Generates a model chooser definition from a model and adds it to the registry .


def formatter(self, api_client, data, newval):
        """Get audio-related fields

        Try to find fields for the audio url for specified preferred quality
        level, or next-lowest available quality url otherwise.
        """
        url_map = data.get("audioUrlMap")
        audio_url = data.get("audioUrl")

        # Only an audio URL, not a quality map. This happens for most of the
        # mobile client tokens and some of the others now. In this case
        # substitute the empirically determined default values in the format
        # used by the rest of the function so downstream consumers continue to
        # work.
        if audio_url and not url_map:
            url_map = {
                BaseAPIClient.HIGH_AUDIO_QUALITY: {
                    "audioUrl": audio_url,
                    "bitrate": 64,
                    "encoding": "aacplus",
                }
            }
        elif not url_map:  # No audio url available (e.g. ad tokens)
            return None

        valid_audio_formats = [BaseAPIClient.HIGH_AUDIO_QUALITY,
                               BaseAPIClient.MED_AUDIO_QUALITY,
                               BaseAPIClient.LOW_AUDIO_QUALITY]

        # Only iterate over sublist, starting at preferred audio quality, or
        # from the beginning of the list if nothing is found. Ensures that the
        # bitrate used will always be the same or lower quality than was
        # specified to prevent audio from skipping for slow connections.
        preferred_quality = api_client.default_audio_quality
        if preferred_quality in valid_audio_formats:
            i = valid_audio_formats.index(preferred_quality)
            valid_audio_formats = valid_audio_formats[i:]

        for quality in valid_audio_formats:
            audio_url = url_map.get(quality)

            if audio_url:
                return audio_url[self.field]

        return audio_url[self.field] if audio_url else None

def formatter ( self , api_client , data , newval ) : url_map = data . get ( "audioUrlMap" ) audio_url = data . get ( "audioUrl" ) if audio_url and not url_map : url_map = { BaseAPIClient . HIGH_AUDIO_QUALITY : { "audioUrl" : audio_url , "bitrate" : 64 , "encoding" : "aacplus" , } } elif not url_map : return None valid_audio_formats = [ BaseAPIClient . HIGH_AUDIO_QUALITY , BaseAPIClient . MED_AUDIO_QUALITY , BaseAPIClient . LOW_AUDIO_QUALITY ] preferred_quality = api_client . default_audio_quality if preferred_quality in valid_audio_formats : i = valid_audio_formats . index ( preferred_quality ) valid_audio_formats = valid_audio_formats [ i : ] for quality in valid_audio_formats : audio_url = url_map . get ( quality ) if audio_url : return audio_url [ self . field ] return audio_url [ self . field ] if audio_url else None


def formatter(self, api_client, data, newval):
        """Parse additional url fields and map them to inputs

        Attempt to create a dictionary with keys being user input, and
        response being the returned URL
        """
        if newval is None:
            return None

        user_param = data['_paramAdditionalUrls']
        urls = {}
        if isinstance(newval, str):
            urls[user_param[0]] = newval
        else:
            for key, url in zip(user_param, newval):
                urls[key] = url
        return urls

def formatter ( self , api_client , data , newval ) : if newval is None : return None user_param = data [ '_paramAdditionalUrls' ] urls = { } if isinstance ( newval , str ) : urls [ user_param [ 0 ] ] = newval else : for key , url in zip ( user_param , newval ) : urls [ key ] = url return urls


def from_json_list(cls, api_client, data):
        """Convert a list of JSON values to a list of models
        """
        return [cls.from_json(api_client, item) for item in data]

def from_json_list ( cls , api_client , data ) : return [ cls . from_json ( api_client , item ) for item in data ]


def populate_fields(api_client, instance, data):
        """Populate all fields of a model with data

        Given a model with a PandoraModel superclass will enumerate all
        declared fields on that model and populate the values of their Field
        and SyntheticField classes. All declared fields will have a value after
        this function runs even if they are missing from the incoming JSON.
        """
        for key, value in instance.__class__._fields.items():
            default = getattr(value, "default", None)
            newval = data.get(value.field, default)

            if isinstance(value, SyntheticField):
                newval = value.formatter(api_client, data, newval)
                setattr(instance, key, newval)
                continue

            model_class = getattr(value, "model", None)
            if newval and model_class:
                if isinstance(newval, list):
                    newval = model_class.from_json_list(api_client, newval)
                else:
                    newval = model_class.from_json(api_client, newval)

            if newval and value.formatter:
                newval = value.formatter(api_client, newval)

            setattr(instance, key, newval)

def populate_fields ( api_client , instance , data ) : for key , value in instance . __class__ . _fields . items ( ) : default = getattr ( value , "default" , None ) newval = data . get ( value . field , default ) if isinstance ( value , SyntheticField ) : newval = value . formatter ( api_client , data , newval ) setattr ( instance , key , newval ) continue model_class = getattr ( value , "model" , None ) if newval and model_class : if isinstance ( newval , list ) : newval = model_class . from_json_list ( api_client , newval ) else : newval = model_class . from_json ( api_client , newval ) if newval and value . formatter : newval = value . formatter ( api_client , newval ) setattr ( instance , key , newval )


def from_json(cls, api_client, data):
        """Convert one JSON value to a model object
        """
        self = cls(api_client)
        PandoraModel.populate_fields(api_client, self, data)
        return self

def from_json ( cls , api_client , data ) : self = cls ( api_client ) PandoraModel . populate_fields ( api_client , self , data ) return self


def _base_repr(self, and_also=None):
        """Common repr logic for subclasses to hook
        """
        items = [
            "=".join((key, repr(getattr(self, key))))
            for key in sorted(self._fields.keys())]

        if items:
            output = ", ".join(items)
        else:
            output = None

        if and_also:
            return "{}({}, {})".format(self.__class__.__name__,
                                       output, and_also)
        else:
            return "{}({})".format(self.__class__.__name__, output)

def
_base_repr
(
self
,
and_also
=
)
:
items
=
[
"="
.
join
(
(
key
,
repr
(
getattr
(
self
,
key
)
)
)
)
for
key
in
sorted
(
self
.
_fields
.
keys
(
)
)
]
if
items
:
output
=
", "
.
join
(
items
)
else
:
output
=
if
and_also
:
return
"{}({}, {})"
.
format
(
self
.
__class__
.
__name__
,
output
,
and_also
)
else
:
return
"{}({})"
.
format
(
self
.
__class__
.
__name__
,
output
)


def _send_cmd(self, cmd):
        """Write command to remote process
        """
        self._process.stdin.write("{}\n".format(cmd).encode("utf-8"))
        self._process.stdin.flush()

def _send_cmd ( self , cmd ) : self . _process . stdin . write ( "{}\n" . format ( cmd ) . encode ( "utf-8" ) ) self . _process . stdin . flush ( )


def _ensure_started(self):
        """Ensure player backing process is started
        """
        if self._process and self._process.poll() is None:
            return

        if not getattr(self, "_cmd"):
            raise RuntimeError("Player command is not configured")

        log.debug("Starting playback command: %r", self._cmd)
        self._process = SilentPopen(self._cmd)
        self._post_start()

def
_ensure_started
(
self
)
:
if
self
.
_process
and
self
.
_process
.
poll
(
)
is
:
return
if
not
getattr
(
self
,
"_cmd"
)
:
raise
RuntimeError
(
"Player command is not configured"
)
log
.
debug
(
"Starting playback command: %r"
,
self
.
_cmd
)
self
.
_process
=
SilentPopen
(
self
.
_cmd
)
self
.
_post_start
(
)


def play(self, song):
        """Play a new song from a Pandora model

        Returns once the stream starts but does not shut down the remote audio
        output backend process. Calls the input callback when the user has
        input.
        """
        self._callbacks.play(song)
        self._load_track(song)
        time.sleep(2)  # Give the backend time to load the track

        while True:
            try:
                self._callbacks.pre_poll()
                self._ensure_started()
                self._loop_hook()

                readers, _, _ = select.select(
                    self._get_select_readers(), [], [], 1)

                for handle in readers:
                    if handle.fileno() == self._control_fd:
                        self._callbacks.input(handle.readline().strip(), song)
                    else:
                        value = self._read_from_process(handle)
                        if self._player_stopped(value):
                            return
            finally:
                self._callbacks.post_poll()

def play ( self , song ) : self . _callbacks . play ( song ) self . _load_track ( song ) time . sleep ( 2 ) while True : try : self . _callbacks . pre_poll ( ) self . _ensure_started ( ) self . _loop_hook ( ) readers , _ , _ = select . select ( self . _get_select_readers ( ) , [ ] , [ ] , 1 ) for handle in readers : if handle . fileno ( ) == self . _control_fd : self . _callbacks . input ( handle . readline ( ) . strip ( ) , song ) else : value = self . _read_from_process ( handle ) if self . _player_stopped ( value ) : return finally : self . _callbacks . post_poll ( )


def play_station(self, station):
        """Play the station until something ends it

        This function will run forever until termintated by calling
        end_station.
        """
        for song in iterate_forever(station.get_playlist):
            try:
                self.play(song)
            except StopIteration:
                self.stop()
                return

def play_station ( self , station ) : for song in iterate_forever ( station . get_playlist ) : try : self . play ( song ) except StopIteration : self . stop ( ) return


def _post_start(self):
        """Set stdout to non-blocking

        VLC does not always return a newline when reading status so in order to
        be lazy and still use the read API without caring about how much output
        there is we switch stdout to nonblocking mode and just read a large
        chunk of datin order to be lazy and still use the read API without
        caring about how much output there is we switch stdout to nonblocking
        mode and just read a large chunk of data.
        """
        flags = fcntl.fcntl(self._process.stdout, fcntl.F_GETFL)
        fcntl.fcntl(self._process.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)

def _post_start ( self ) : flags = fcntl . fcntl ( self . _process . stdout , fcntl . F_GETFL ) fcntl . fcntl ( self . _process . stdout , fcntl . F_SETFL , flags | os . O_NONBLOCK )


def station_selection_menu(self, error=None):
        """Format a station menu and make the user select a station
        """
        self.screen.clear()

        if error:
            self.screen.print_error("{}\n".format(error))

        for i, station in enumerate(self.stations):
            i = "{:>3}".format(i)
            print("{}: {}".format(Colors.yellow(i), station.name))

        return self.stations[self.screen.get_integer("Station: ")]

def
station_selection_menu
(
self
,
error
=
)
:
self
.
screen
.
clear
(
)
if
error
:
self
.
screen
.
print_error
(
"{}\n"
.
format
(
error
)
)
for
i
,
station
in
enumerate
(
self
.
stations
)
:
i
=
"{:>3}"
.
format
(
i
)
print
(
"{}: {}"
.
format
(
Colors
.
yellow
(
i
)
,
station
.
name
)
)
return
self
.
stations
[
self
.
screen
.
get_integer
(
"Station: "
)
]


def input(self, input, song):
        """Input callback, handles key presses
        """
        try:
            cmd = getattr(self, self.CMD_MAP[input][1])
        except (IndexError, KeyError):
            return self.screen.print_error(
                "Invalid command {!r}!".format(input))

        cmd(song)

def
input
(
self
,
input
,
song
)
:
try
:
cmd
=
getattr
(
self
,
self
.
CMD_MAP
[
input
]
[
1
]
)
except
(
IndexError
,
KeyError
)
:
return
self
.
screen
.
print_error
(
"Invalid command {!r}!"
.
format
(
input
)
)
cmd
(
song
)


def retries(max_tries, exceptions=(Exception,)):
    """Function decorator implementing retrying logic.

    exceptions: A tuple of exception classes; default (Exception,)

    The decorator will call the function up to max_tries times if it raises
    an exception.

    By default it catches instances of the Exception class and subclasses.
    This will recover after all but the most fatal errors. You may specify a
    custom tuple of exception classes with the 'exceptions' argument; the
    function will only be retried if it raises one of the specified
    exceptions.
    """
    def decorator(func):
        def function(*args, **kwargs):

            retries_left = max_tries
            while retries_left > 0:
                try:
                    retries_left -= 1
                    return func(*args, **kwargs)

                except exceptions as exc:
                    # Don't retry for PandoraExceptions - unlikely that result
                    # will change for same set of input parameters.
                    if isinstance(exc, PandoraException):
                        raise
                    if retries_left > 0:
                        time.sleep(delay_exponential(
                            0.5, 2, max_tries - retries_left))
                    else:
                        raise

        return function

    return decorator

def retries ( max_tries , exceptions = ( Exception , ) ) : def decorator ( func ) : def function ( * args , ** kwargs ) : retries_left = max_tries while retries_left > 0 : try : retries_left -= 1 return func ( * args , ** kwargs ) except exceptions as exc : if isinstance ( exc , PandoraException ) : raise if retries_left > 0 : time . sleep ( delay_exponential ( 0.5 , 2 , max_tries - retries_left ) ) else : raise return function return decorator

Function decorator implementing retrying logic .


def iterate_forever(func, *args, **kwargs):
    """Iterate over a finite iterator forever

    When the iterator is exhausted will call the function again to generate a
    new iterator and keep iterating.
    """
    output = func(*args, **kwargs)

    while True:
        try:
            playlist_item = next(output)
            playlist_item.prepare_playback()
            yield playlist_item
        except StopIteration:
            output = func(*args, **kwargs)

def iterate_forever ( func , * args , ** kwargs ) : output = func ( * args , ** kwargs ) while True : try : playlist_item = next ( output ) playlist_item . prepare_playback ( ) yield playlist_item except StopIteration : output = func ( * args , ** kwargs )


def get_integer(prompt):
        """Gather user input and convert it to an integer

        Will keep trying till the user enters an interger or until they ^C the
        program.
        """
        while True:
            try:
                return int(input(prompt).strip())
            except ValueError:
                print(Colors.red("Invalid Input!"))

def
get_integer
(
prompt
)
:
while
True
:
try
:
return
int
(
input
(
prompt
)
.
strip
(
)
)
except
ValueError
:
print
(
Colors
.
red
(
"Invalid Input!"
)
)


def open(self):
        """open the drop box

        You need to call this method before starting putting packages.

        Returns
        -------
        None

        """

        self.workingArea.open()
        self.runid_pkgidx_map = { }
        self.runid_to_return = deque()

def open ( self ) : self . workingArea . open ( ) self . runid_pkgidx_map = { } self . runid_to_return = deque ( )


def put(self, package):
        """put a task

        This method places a task in the working area and have the
        dispatcher execute it.

        If you need to put multiple tasks, it can be much faster to
        use `put_multiple()` than to use this method multiple times
        depending of the dispatcher.

        Parameters
        ----------
        package : callable
            A task

        Returns
        -------
        int
            A package index assigned by the working area

        """

        pkgidx = self.workingArea.put_package(package)

        logger = logging.getLogger(__name__)
        logger.info('submitting {}'.format(self.workingArea.package_relpath(pkgidx)))

        runid = self.dispatcher.run(self.workingArea, pkgidx)
        self.runid_pkgidx_map[runid] = pkgidx
        return pkgidx

def
put
(
self
,
package
)
:
pkgidx
=
self
.
workingArea
.
put_package
(
package
)
logger
=
logging
.
getLogger
(
__name__
)
logger
.
info
(
'submitting {}'
.
format
(
self
.
workingArea
.
package_relpath
(
pkgidx
)
)
)
runid
=
self
.
dispatcher
.
run
(
self
.
workingArea
,
pkgidx
)
self
.
runid_pkgidx_map
[
runid
]
=
pkgidx
return
pkgidx


def receive(self):
        """return pairs of package indices and results of all tasks

        This method waits until all tasks finish.

        Returns
        -------
        list
            A list of pairs of package indices and results

        """

        ret = [ ] # a list of (pkgid, result)
        while True:

            if self.runid_pkgidx_map:
                self.runid_to_return.extend(self.dispatcher.poll())
                ret.extend(self._collect_all_finished_pkgidx_result_pairs())

            if not self.runid_pkgidx_map:
                break
            time.sleep(self.sleep)

        ret = sorted(ret, key=itemgetter(0))

        return ret


def poll(self):
        """return pairs of package indices and results of finished tasks

        This method does not wait for tasks to finish.

        Returns
        -------
        list
            A list of pairs of package indices and results

        """

        self.runid_to_return.extend(self.dispatcher.poll())
        ret = self._collect_all_finished_pkgidx_result_pairs()
        return ret


def receive_one(self):
        """return a pair of a package index and result of a task

        This method waits until a tasks finishes. It returns `None` if
        no task is running.

        Returns
        -------
        tuple or None
            A pair of a package index and result. `None` if no tasks
            is running.

        """

        if not self.runid_pkgidx_map:
            return None

        while True:

            if not self.runid_to_return:
                self.runid_to_return.extend(self.dispatcher.poll())

            ret = self._collect_next_finished_pkgidx_result_pair()

            if ret is not None:
                break

            if self.runid_pkgidx_map:
                time.sleep(self.sleep)

        return ret


def run_multiple(self, eventLoops):
        """run the event loops in the background.

        Args:
            eventLoops (list): a list of event loops to run

        """

        self.nruns += len(eventLoops)
        return self.communicationChannel.put_multiple(eventLoops)

def run_multiple ( self , eventLoops ) : self . nruns += len ( eventLoops ) return self . communicationChannel . put_multiple ( eventLoops )


def poll(self):
        """Return pairs of run ids and results of finish event loops.
        """
        ret = self.communicationChannel.receive_finished()
        self.nruns -= len(ret)
        return ret

def poll ( self ) : ret = self . communicationChannel . receive_finished ( ) self . nruns -= len ( ret ) return ret

Return pairs of run ids and results of finish event loops .


def receive_one(self):
        """Return a pair of a run id and a result.

        This method waits until an event loop finishes.
        This method returns None if no loop is running.
        """
        if self.nruns == 0:
            return None
        ret = self.communicationChannel.receive_one()
        if ret is not None:
            self.nruns -= 1
        return ret

def receive_one ( self ) : if self . nruns == 0 : return None ret = self . communicationChannel . receive_one ( ) if ret is not None : self . nruns -= 1 return ret

Return a pair of a run id and a result .


def receive(self):
        """Return pairs of run ids and results.

        This method waits until all event loops finish
        """
        ret = self.communicationChannel.receive_all()
        self.nruns -= len(ret)
        if self.nruns > 0:
            import logging
            logger = logging.getLogger(__name__)
            logger.warning(
                'too few results received: {} results received, {} more expected'.format(
                    len(ret), self.nruns))
        elif self.nruns < 0:
            import logging
            logger = logging.getLogger(__name__)
            logger.warning(
                'too many results received: {} results received, {} too many'.format(
                    len(ret), -self.nruns))
        return ret


def end(self):
        """wait until all event loops end and returns the results.

        """

        results = self.communicationChannel.receive()

        if self.nruns != len(results):
            import logging
            logger = logging.getLogger(__name__)
            # logger.setLevel(logging.DEBUG)
            logger.warning(
                'too few results received: {} results received, {} expected'.format(
                    len(results),
                    self.nruns
                ))

        return results

wait until all event loops end and returns the results .


def key_vals_dict_to_tuple_list(key_vals_dict, fill=float('nan')):
    """Convert ``key_vals_dict`` to `tuple_list``.

    Args:
        key_vals_dict (dict): The first parameter.
        fill: a value to fill missing data

    Returns:
        A list of tuples

    """

    tuple_list = [ ]

    if not key_vals_dict: return tuple_list

    vlen = max([len(vs) for vs in itertools.chain(*key_vals_dict.values())])

    for k, vs in key_vals_dict.items():
        try:
            tuple_list.extend([k + tuple(v) + (fill, )*(vlen - len(v)) for v in vs])
        except TypeError:
            # assume k is not a tuple
            tuple_list.extend([(k, ) + tuple(v) + (fill, )*(vlen - len(v)) for v in vs])

    return tuple_list

def key_vals_dict_to_tuple_list ( key_vals_dict , fill = float ( 'nan' ) ) : tuple_list = [ ] if not key_vals_dict : return tuple_list vlen = max ( [ len ( vs ) for vs in itertools . chain ( * key_vals_dict . values ( ) ) ] ) for k , vs in key_vals_dict . items ( ) : try : tuple_list . extend ( [ k + tuple ( v ) + ( fill , ) * ( vlen - len ( v ) ) for v in vs ] ) except TypeError : tuple_list . extend ( [ ( k , ) + tuple ( v ) + ( fill , ) * ( vlen - len ( v ) ) for v in vs ] ) return tuple_list


def open(self):
        """Open the working area

        Returns
        -------
        None
        """

        self.path = self._prepare_dir(self.topdir)
        self._copy_executable(area_path=self.path)
        self._save_logging_levels(area_path=self.path)
        self._put_python_modules(modules=self.python_modules, area_path=self.path)

def open ( self ) : self . path = self . _prepare_dir ( self . topdir ) self . _copy_executable ( area_path = self . path ) self . _save_logging_levels ( area_path = self . path ) self . _put_python_modules ( modules = self . python_modules , area_path = self . path )


def put_package(self, package):
        """Put a package

        Parameters
        ----------
        package :
            a task package

        Returns
        -------
        int
            A package index

        """

        self.last_package_index += 1
        package_index = self.last_package_index

        package_fullpath = self.package_fullpath(package_index)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/task_00009.p.gz'

        with gzip.open(package_fullpath, 'wb') as f:
            pickle.dump(package, f, protocol=pickle.HIGHEST_PROTOCOL)
            f.close()

        result_fullpath = self.result_fullpath(package_index)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009/result.p.gz'

        result_dir = os.path.dirname(result_fullpath)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009'

        alphatwirl.mkdir_p(result_dir)

        return package_index

def put_package ( self , package ) : self . last_package_index += 1 package_index = self . last_package_index package_fullpath = self . package_fullpath ( package_index ) with gzip . open ( package_fullpath , 'wb' ) as f : pickle . dump ( package , f , protocol = pickle . HIGHEST_PROTOCOL ) f . close ( ) result_fullpath = self . result_fullpath ( package_index ) result_dir = os . path . dirname ( result_fullpath ) alphatwirl . mkdir_p ( result_dir ) return package_index


def collect_result(self, package_index):
        """Collect the result of a task

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        obj
            The result of the task

        """

        result_fullpath = self.result_fullpath(package_index)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009/result.p.gz'

        try:
            with gzip.open(result_fullpath, 'rb') as f:
                result = pickle.load(f)
        except Exception as e:
            logger = logging.getLogger(__name__)
            logger.warning(e)
            return None

        return result

def collect_result ( self , package_index ) : result_fullpath = self . result_fullpath ( package_index ) try : with gzip . open ( result_fullpath , 'rb' ) as f : result = pickle . load ( f ) except Exception as e : logger = logging . getLogger ( __name__ ) logger . warning ( e ) return None return result


def package_fullpath(self, package_index):
        """Returns the full path of the package

        This method returns the full path to the package. This method
        simply constructs the path based on the convention and doesn't
        check if the package actually exists.

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        str
            the full path to the package

        """

        ret = os.path.join(self.path, self.package_relpath(package_index))
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/task_00009.p.gz'

        return ret

def package_fullpath ( self , package_index ) : ret = os . path . join ( self . path , self . package_relpath ( package_index ) ) return ret


def result_relpath(self, package_index):
        """Returns the relative path of the result

        This method returns the path to the result relative to the
        top dir of the working area. This method simply constructs the
        path based on the convention and doesn't check if the result
        actually exists.

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        str
            the relative path to the result

        """

        dirname = 'task_{:05d}'.format(package_index)
        # e.g., 'task_00009'

        ret = os.path.join('results', dirname, 'result.p.gz')
        # e.g., 'results/task_00009/result.p.gz'

        return ret

def result_relpath ( self , package_index ) : dirname = 'task_{:05d}' . format ( package_index ) ret = os . path . join ( 'results' , dirname , 'result.p.gz' ) return ret


def result_fullpath(self, package_index):
        """Returns the full path of the result

        This method returns the full path to the result. This method
        simply constructs the path based on the convention and doesn't
        check if the result actually exists.

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        str
            the full path to the result

        """

        ret = os.path.join(self.path, self.result_relpath(package_index))
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009/result.p.gz'

        return ret

def result_fullpath ( self , package_index ) : ret = os . path . join ( self . path , self . result_relpath ( package_index ) ) return ret


def run_multiple(self, workingArea, package_indices):
        """Submit multiple jobs

        Parameters
        ----------
        workingArea :
            A workingArea
        package_indices : list(int)
            A list of package indices

        Returns
        -------
        list(str)
            The list of the run IDs of the jobs

        """

        if not package_indices:
            return [ ]

        job_desc = self._compose_job_desc(workingArea, package_indices)

        clusterprocids = submit_jobs(job_desc, cwd=workingArea.path)

        # TODO: make configurable
        clusterids = clusterprocids2clusterids(clusterprocids)
        for clusterid in clusterids:
            change_job_priority([clusterid], 10)

        self.clusterprocids_outstanding.extend(clusterprocids)

        return clusterprocids

def run_multiple ( self , workingArea , package_indices ) : if not package_indices : return [ ] job_desc = self . _compose_job_desc ( workingArea , package_indices ) clusterprocids = submit_jobs ( job_desc , cwd = workingArea . path ) clusterids = clusterprocids2clusterids ( clusterprocids ) for clusterid in clusterids : change_job_priority ( [ clusterid ] , 10 ) self . clusterprocids_outstanding . extend ( clusterprocids ) return clusterprocids


def poll(self):
        """Return the run IDs of the finished jobs

        Returns
        -------
        list(str)
            The list of the run IDs of the finished jobs

        """

        clusterids = clusterprocids2clusterids(self.clusterprocids_outstanding)
        clusterprocid_status_list = query_status_for(clusterids)
        # e.g., [['1730126.0', 2], ['1730127.0', 2], ['1730129.1', 1], ['1730130.0', 1]]

        if clusterprocid_status_list:
            clusterprocids, statuses = zip(*clusterprocid_status_list)
        else:
            clusterprocids, statuses = (), ()

        clusterprocids_finished = [i for i in self.clusterprocids_outstanding if i not in clusterprocids]
        self.clusterprocids_finished.extend(clusterprocids_finished)
        self.clusterprocids_outstanding[:] = clusterprocids

        # logging
        counter = collections.Counter(statuses)
        messages = [ ]
        if counter:
            messages.append(', '.join(['{}: {}'.format(HTCONDOR_JOBSTATUS[k], counter[k]) for k in counter.keys()]))
        if self.clusterprocids_finished:
            messages.append('Finished {}'.format(len(self.clusterprocids_finished)))
        logger = logging.getLogger(__name__)
        logger.info(', '.join(messages))

        return clusterprocids_finished

def
poll
(
self
)
:
clusterids
=
clusterprocids2clusterids
(
self
.
clusterprocids_outstanding
)
clusterprocid_status_list
=
query_status_for
(
clusterids
)
if
clusterprocid_status_list
:
clusterprocids
,
statuses
=
zip
(
*
clusterprocid_status_list
)
else
:
clusterprocids
,
statuses
=
(
)
,
(
)
clusterprocids_finished
=
[
i
for
i
in
self
.
clusterprocids_outstanding
if
i
not
in
clusterprocids
]
self
.
clusterprocids_finished
.
extend
(
clusterprocids_finished
)
self
.
clusterprocids_outstanding
[
:
]
=
clusterprocids
counter
=
collections
.
Counter
(
statuses
)
messages
=
[
]
if
counter
:
messages
.
append
(
', '
.
join
(
[
'{}: {}'
.
format
(
HTCONDOR_JOBSTATUS
[
k
]
,
counter
[
k
]
)
for
k
in
counter
.
keys
(
)
]
)
)
if
self
.
clusterprocids_finished
:
messages
.
append
(
'Finished {}'
.
format
(
len
(
self
.
clusterprocids_finished
)
)
)
logger
=
logging
.
getLogger
(
__name__
)
logger
.
info
(
', '
.
join
(
messages
)
)
return
clusterprocids_finished


def wait(self):
        """Wait until all jobs finish and return the run IDs of the finished jobs

        Returns
        -------
        list(str)
            The list of the run IDs of the finished jobs

        """

        sleep = 5
        while True:
            if self.clusterprocids_outstanding:
                self.poll()
            if not self.clusterprocids_outstanding:
                break
            time.sleep(sleep)
        return self.clusterprocids_finished

def wait ( self ) : sleep = 5 while True : if self . clusterprocids_outstanding : self . poll ( ) if not self . clusterprocids_outstanding : break time . sleep ( sleep ) return self . clusterprocids_finished


def failed_runids(self, runids):
        """Provide the run IDs of failed jobs

        Returns
        -------
        None

        """

        # remove failed clusterprocids from self.clusterprocids_finished
        # so that len(self.clusterprocids_finished)) becomes the number
        # of the successfully finished jobs
        for i in runids:
            try:
                self.clusterprocids_finished.remove(i)
            except ValueError:
                pass

def failed_runids ( self , runids ) : for i in runids : try : self . clusterprocids_finished . remove ( i ) except ValueError : pass


def getArrays(self, tree, branchName):
        """return the array.array objects for the branch and its counter branch

        This method returns a pair of the array.array objects. The first one is
        for the given tree and branch name. The second one is for its counter
        branch. The second one will be None when the branch does not have a
        counter. A pair of None will be returned when the tree does not have
        the branch.

        """
        itsArray = self._getArray(tree, branchName)
        if itsArray is None: return None, None
        itsCountArray = self._getCounterArray(tree, branchName)
        return itsArray, itsCountArray

def getArrays ( self , tree , branchName ) : itsArray = self . _getArray ( tree , branchName ) if itsArray is None : return None , None itsCountArray = self . _getCounterArray ( tree , branchName ) return itsArray , itsCountArray

return the array . array objects for the branch and its counter branch


def put(self, task, *args, **kwargs):
        """put a task and its arguments

        If you need to put multiple tasks, it can be faster to put
        multiple tasks with `put_multiple()` than to use this method
        multiple times.

        Parameters
        ----------
        task : a function
            A function to be executed
        args : list
            A list of positional arguments to the `task`
        kwargs : dict
            A dict with keyword arguments to the `task`

        Returns
        -------
        int, str, or any hashable and sortable
            A task ID. IDs are sortable in the order in which the
            corresponding tasks are put.

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        package = TaskPackage(task=task, args=args, kwargs=kwargs)
        return self.dropbox.put(package)

def
put
(
self
,
task
,
*
args
,
**
kwargs
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
package
=
TaskPackage
(
task
=
task
,
args
=
args
,
kwargs
=
kwargs
)
return
self
.
dropbox
.
put
(
package
)


def put_multiple(self, task_args_kwargs_list):
        """put a list of tasks and their arguments

        This method can be used to put multiple tasks at once. Calling
        this method once with multiple tasks can be much faster than
        calling `put()` multiple times.

        Parameters
        ----------
        task_args_kwargs_list : list

            A list of lists with three items that can be parameters of
            `put()`, i.e., `task`, `args`, `kwargs`.

        Returns
        -------
        list
            A list of task IDs.

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return

        packages = [ ]
        for t in task_args_kwargs_list:
            try:
                task = t['task']
                args = t.get('args', ())
                kwargs = t.get('kwargs', {})
                package = TaskPackage(task=task, args=args, kwargs=kwargs)
            except TypeError:
                package = TaskPackage(task=t, args=(), kwargs={})
            packages.append(package)
        return self.dropbox.put_multiple(packages)

def
put_multiple
(
self
,
task_args_kwargs_list
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
packages
=
[
]
for
t
in
task_args_kwargs_list
:
try
:
task
=
t
[
'task'
]
args
=
t
.
get
(
'args'
,
(
)
)
kwargs
=
t
.
get
(
'kwargs'
,
{
}
)
package
=
TaskPackage
(
task
=
task
,
args
=
args
,
kwargs
=
kwargs
)
except
TypeError
:
package
=
TaskPackage
(
task
=
t
,
args
=
(
)
,
kwargs
=
{
}
)
packages
.
append
(
package
)
return
self
.
dropbox
.
put_multiple
(
packages
)


def receive_finished(self):
        """return a list of pairs of IDs and results of finished tasks.

        This method doesn't wait for tasks to finish. It returns IDs
        and results which have already finished.

        Returns
        -------
        list
            A list of pairs of IDs and results

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        return self.dropbox.poll()

def
receive_finished
(
self
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
return
self
.
dropbox
.
poll
(
)

return a list of pairs of IDs and results of finished tasks .


def receive_one(self):
        """return a pair of an ID and a result of a task.

        This method waits for a task to finish.

        Returns
        -------
        An ID and a result of a task. `None` if no task is running.

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        return self.dropbox.receive_one()

def
receive_one
(
self
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
return
self
.
dropbox
.
receive_one
(
)

return a pair of an ID and a result of a task .


def receive_all(self):
        """return a list of pairs of IDs and results of all tasks.

        This method waits for all tasks to finish.

        Returns
        -------
        list
            A list of pairs of IDs and results

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        return self.dropbox.receive()

def
receive_all
(
self
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
return
self
.
dropbox
.
receive
(
)

return a list of pairs of IDs and results of all tasks .


def receive(self):
        """return a list results of all tasks.

        This method waits for all tasks to finish.

        Returns
        -------
        list
            A list of results of the tasks. The results are sorted in
            the order in which the tasks are put.

        """
        pkgidx_result_pairs = self.receive_all()
        if pkgidx_result_pairs is None:
            return
        results = [r for _, r in pkgidx_result_pairs]
        return results

def receive ( self ) : pkgidx_result_pairs = self . receive_all ( ) if pkgidx_result_pairs is None : return results = [ r for _ , r in pkgidx_result_pairs ] return results


def expand_path_cfg(path_cfg, alias_dict={ }, overriding_kargs={ }):
    """expand a path config

    Args:
        path_cfg (str, tuple, dict): a config for path
        alias_dict (dict): a dict for aliases
        overriding_kargs (dict): to be used for recursive call
    """

    if isinstance(path_cfg, str):
        return _expand_str(path_cfg, alias_dict, overriding_kargs)

    if isinstance(path_cfg, dict):
        return _expand_dict(path_cfg, alias_dict)

    # assume tuple or list
    return _expand_tuple(path_cfg, alias_dict, overriding_kargs)

def expand_path_cfg ( path_cfg , alias_dict = { } , overriding_kargs = { } ) : if isinstance ( path_cfg , str ) : return _expand_str ( path_cfg , alias_dict , overriding_kargs ) if isinstance ( path_cfg , dict ) : return _expand_dict ( path_cfg , alias_dict ) return _expand_tuple ( path_cfg , alias_dict , overriding_kargs )


def _expand_tuple(path_cfg, alias_dict, overriding_kargs):
    """expand a path config given as a tuple

    """

    # e.g.,
    # path_cfg = ('ev : {low} <= ev.var[0] < {high}', {'low': 10, 'high': 200})
    # overriding_kargs = {'alias': 'var_cut', 'name': 'var_cut25', 'low': 25}

    new_path_cfg = path_cfg[0]
    # e.g., 'ev : {low} <= ev.var[0] < {high}'

    new_overriding_kargs = path_cfg[1].copy()
    # e.g., {'low': 10, 'high': 200}

    new_overriding_kargs.update(overriding_kargs)
    # e.g., {'low': 25, 'high': 200, 'alias': 'var_cut', 'name': 'var_cut25'}

    return expand_path_cfg(
        new_path_cfg,
        overriding_kargs=new_overriding_kargs,
        alias_dict=alias_dict
    )


def poll(self):
        """check if the jobs are running and return a list of pids for
        finished jobs

        """
        finished_procs = [p for p in self.running_procs if p.poll() is not None]
        self.running_procs = collections.deque([p for p in self.running_procs if p not in finished_procs])

        for proc in finished_procs:
            stdout, stderr = proc.communicate()
            ## proc.communicate() returns (stdout, stderr) when
            ## self.pipe = True. Otherwise they are (None, None)

        finished_pids = [p.pid for p in finished_procs]
        self.finished_pids.extend(finished_pids)

        logger = logging.getLogger(__name__)
        messages = 'Running: {}, Finished: {}'.format(len(self.running_procs), len(self.finished_pids))
        logger.info(messages)

        return finished_pids

def
poll
(
self
)
:
finished_procs
=
[
p
for
p
in
self
.
running_procs
if
p
.
poll
(
)
is
not
]
self
.
running_procs
=
collections
.
deque
(
[
p
for
p
in
self
.
running_procs
if
p
not
in
finished_procs
]
)
for
proc
in
finished_procs
:
stdout
,
stderr
=
proc
.
communicate
(
)
finished_pids
=
[
p
.
pid
for
p
in
finished_procs
]
self
.
finished_pids
.
extend
(
finished_pids
)
logger
=
logging
.
getLogger
(
__name__
)
messages
=
'Running: {}, Finished: {}'
.
format
(
len
(
self
.
running_procs
)
,
len
(
self
.
finished_pids
)
)
logger
.
info
(
messages
)
return
finished_pids


def wait(self):
        """wait until all jobs finish and return a list of pids
        """
        finished_pids = [ ]
        while self.running_procs:
            finished_pids.extend(self.poll())
        return finished_pids

def wait ( self ) : finished_pids = [ ] while self . running_procs : finished_pids . extend ( self . poll ( ) ) return finished_pids


def getVector(self, tree, branchName):
        """return the ROOT.vector object for the branch.

        """

        if (tree, branchName) in self.__class__.addressDict:
            return self.__class__.addressDict[(tree, branchName)]

        itsVector = self._getVector(tree, branchName)
        self.__class__.addressDict[(tree, branchName)] = itsVector

        return itsVector

def getVector ( self , tree , branchName ) : if ( tree , branchName ) in self . __class__ . addressDict : return self . __class__ . addressDict [ ( tree , branchName ) ] itsVector = self . _getVector ( tree , branchName ) self . __class__ . addressDict [ ( tree , branchName ) ] = itsVector return itsVector

return the ROOT . vector object for the branch .