def itertrain(self, train, valid=None, algo='rmsprop', subalgo='rmsprop',
                  save_every=0, save_progress=None, **kwargs):
        '''Train our network, one batch at a time.

        This method yields a series of ``(train, valid)`` monitor pairs. The
        ``train`` value is a dictionary mapping names to monitor values
        evaluated on the training dataset. The ``valid`` value is also a
        dictionary mapping names to values, but these values are evaluated on
        the validation dataset.

        Because validation might not occur every training iteration, the
        validation monitors might be repeated for multiple training iterations.
        It is probably most helpful to think of the validation monitors as being
        the "most recent" values that have been computed.

        After training completes, the network attribute of this class will
        contain the trained network parameters.

        Parameters
        ----------
        train : :class:`Dataset <downhill.dataset.Dataset>` or list
            A dataset to use when training the network. If this is a
            ``downhill.Dataset`` instance, it will be used directly as the
            training datset. If it is a list of numpy arrays or a list of
            callables, it will be converted to a ``downhill.Dataset`` and then
            used as the training set.
        valid : :class:`Dataset <downhill.dataset.Dataset>` or list, optional
            If this is provided, it will be used as a validation dataset. If not
            provided, the training set will be used for validation. (This is not
            recommended!)
        algo : str, optional
            An optimization algorithm to use for training our network. If not
            provided, :class:`RMSProp <downhill.adaptive.RMSProp>` will be used.
        subalgo : str, optional
            An optimization algorithm to use for a trainer that requires a
            "sub-algorithm," sugh as an unsupervised pretrainer. Defaults to
            :class:`RMSProp <downhill.adaptive.RMSProp>`.
        save_every : int or float, optional
            If this is nonzero and ``save_progress`` is not None, then the model
            being trained will be saved periodically. If this is a float, it is
            treated as a number of minutes to wait between savings. If it is an
            int, it is treated as the number of training epochs to wait between
            savings. Defaults to 0.
        save_progress : str or file handle, optional
            If this is not None, and ``save_progress`` is nonzero, then save the
            model periodically during training. This parameter gives either (a)
            the full path of a file to save the model, or (b) a file-like object
            where the model should be saved. If it is a string and the given
            name contains a "{}" format specifier, it will be filled with the
            integer Unix timestamp at the time the model is saved. Defaults to
            None, which does not save models.

        Yields
        ------
        training : dict
            A dictionary of monitor values computed using the training dataset,
            at the conclusion of training. This dictionary will at least contain
            a 'loss' key that indicates the value of the loss function. Other
            keys may be available depending on the trainer being used.
        validation : dict
            A dictionary of monitor values computed using the validation
            dataset, at the conclusion of training.
        '''
        if 'rng' not in kwargs:
            kwargs['rng'] = self._rng

        def create_dataset(data, **kwargs):
            name = kwargs.get('name', 'dataset')
            s = '{}_batches'.format(name)
            return downhill.Dataset(
                data,
                name=name,
                batch_size=kwargs.get('batch_size', 32),
                iteration_size=kwargs.get('iteration_size', kwargs.get(s)),
                axis=kwargs.get('axis', 0),
                rng=kwargs['rng'])

        # set up datasets ...
        if valid is None:
            valid = train
        if not isinstance(valid, downhill.Dataset):
            valid = create_dataset(valid, name='valid', **kwargs)
        if not isinstance(train, downhill.Dataset):
            train = create_dataset(train, name='train', **kwargs)

        if 'algorithm' in kwargs:
            warnings.warn(
                'please use the "algo" keyword arg instead of "algorithm"',
                DeprecationWarning)
            algo = kwargs.pop('algorithm')
            if isinstance(algo, (list, tuple)):
                algo = algo[0]

        # set up trainer ...
        if isinstance(algo, util.basestring):
            algo = algo.lower()
            if algo == 'sample':
                algo = trainer.SampleTrainer(self)
            elif algo.startswith('layer') or algo.startswith('sup'):
                algo = trainer.SupervisedPretrainer(subalgo, self)
            elif algo.startswith('pre') or algo.startswith('unsup'):
                algo = trainer.UnsupervisedPretrainer(subalgo, self)
            else:
                algo = trainer.DownhillTrainer(algo, self)

        # set up check to save model ...
        def needs_saving(elapsed, iteration):
            if save_progress is None:
                return False
            if isinstance(save_every, float):
                return elapsed > 60 * save_every
            if isinstance(save_every, int):
                return iteration % save_every == 0
            return False

        # train it!
        start = time.time()
        for i, monitors in enumerate(algo.itertrain(train, valid, **kwargs)):
            yield monitors
            now = time.time()
            if i and needs_saving(now - start, i):
                filename_or_handle = save_progress
                if isinstance(filename_or_handle, util.basestring):
                    filename_or_handle = save_progress.format(int(now))
                self.save(filename_or_handle)
                start = now


def train(self, *args, **kwargs):
        '''Train the network until the trainer converges.

        All arguments are passed to :func:`itertrain`.

        Returns
        -------
        training : dict
            A dictionary of monitor values computed using the training dataset,
            at the conclusion of training. This dictionary will at least contain
            a 'loss' key that indicates the value of the loss function. Other
            keys may be available depending on the trainer being used.
        validation : dict
            A dictionary of monitor values computed using the validation
            dataset, at the conclusion of training.
        '''
        monitors = None
        for monitors in self.itertrain(*args, **kwargs):
            pass
        return monitors

def train ( self , * args , ** kwargs ) : monitors = None for monitors in self . itertrain ( * args , ** kwargs ) : pass return monitors

Train the network until the trainer converges .


def _hash(self, regularizers=()):
        '''Construct a string key for representing a computation graph.

        This key will be unique for a given (a) network topology, (b) set of
        losses, and (c) set of regularizers.

        Returns
        -------
        key : str
            A hash representing the computation graph for the current network.
        '''
        def add(s):
            h.update(str(s).encode('utf-8'))
        h = hashlib.md5()
        for l in self.layers:
            add('{}{}{}'.format(l.__class__.__name__, l.name, l.output_shape))
        for l in self.losses:
            add('{}{}'.format(l.__class__.__name__, l.weight))
        for r in regularizers:
            add('{}{}{}'.format(r.__class__.__name__, r.weight, r.pattern))
        return h.hexdigest()

def _hash ( self , regularizers = ( ) ) : def add ( s ) : h . update ( str ( s ) . encode ( 'utf-8' ) ) h = hashlib . md5 ( ) for l in self . layers : add ( '{}{}{}' . format ( l . __class__ . __name__ , l . name , l . output_shape ) ) for l in self . losses : add ( '{}{}' . format ( l . __class__ . __name__ , l . weight ) ) for r in regularizers : add ( '{}{}{}' . format ( r . __class__ . __name__ , r . weight , r . pattern ) ) return h . hexdigest ( )

Construct a string key for representing a computation graph .


def build_graph(self, regularizers=()):
        '''Connect the layers in this network to form a computation graph.

        Parameters
        ----------
        regularizers : list of :class:`theanets.regularizers.Regularizer`
            A list of the regularizers to apply while building the computation
            graph.

        Returns
        -------
        outputs : list of Theano variables
            A list of expressions giving the output of each layer in the graph.
        updates : list of update tuples
            A list of updates that should be performed by a Theano function that
            computes something using this graph.
        '''
        key = self._hash(regularizers)
        if key not in self._graphs:
            util.log('building computation graph')
            for loss in self.losses:
                loss.log()
            for reg in regularizers:
                reg.log()
            outputs = {}
            updates = []
            for layer in self.layers:
                out, upd = layer.connect(outputs)
                for reg in regularizers:
                    reg.modify_graph(out)
                outputs.update(out)
                updates.extend(upd)
            self._graphs[key] = outputs, updates
        return self._graphs[key]

def
build_graph
(
self
,
regularizers
=
(
)
)
:
key
=
self
.
_hash
(
regularizers
)
if
key
not
in
self
.
_graphs
:
util
.
log
(
'building computation graph'
)
for
loss
in
self
.
losses
:
loss
.
log
(
)
for
reg
in
regularizers
:
reg
.
log
(
)
outputs
=
{
}
updates
=
[
]
for
layer
in
self
.
layers
:
out
,
upd
=
layer
.
connect
(
outputs
)
for
reg
in
regularizers
:
reg
.
modify_graph
(
out
)
outputs
.
update
(
out
)
updates
.
extend
(
upd
)
self
.
_graphs
[
key
]
=
outputs
,
updates
return
self
.
_graphs
[
key
]

Connect the layers in this network to form a computation graph .


def inputs(self):
        '''A list of Theano variables for feedforward computations.'''
        return [l.input for l in self.layers if isinstance(l, layers.Input)]

def inputs ( self ) : return [ l . input for l in self . layers if isinstance ( l , layers . Input ) ]

A list of Theano variables for feedforward computations .


def variables(self):
        '''A list of Theano variables for loss computations.'''
        result = self.inputs
        seen = set(i.name for i in result)
        for loss in self.losses:
            for v in loss.variables:
                if v.name not in seen:
                    result.append(v)
                    seen.add(v.name)
        return result

def variables ( self ) : result = self . inputs seen = set ( i . name for i in result ) for loss in self . losses : for v in loss . variables : if v . name not in seen : result . append ( v ) seen . add ( v . name ) return result

A list of Theano variables for loss computations .


def find(self, which, param):
        '''Get a parameter from a layer in the network.

        Parameters
        ----------
        which : int or str
            The layer that owns the parameter to return.

            If this is an integer, then 0 refers to the input layer, 1 refers
            to the first hidden layer, 2 to the second, and so on.

            If this is a string, the layer with the corresponding name, if any,
            will be used.

        param : int or str
            Name of the parameter to retrieve from the specified layer, or its
            index in the parameter list of the layer.

        Raises
        ------
        KeyError
            If there is no such layer, or if there is no such parameter in the
            specified layer.

        Returns
        -------
        param : Theano shared variable
            A shared parameter variable from the indicated layer.
        '''
        for i, layer in enumerate(self.layers):
            if which == i or which == layer.name:
                return layer.find(param)
        raise KeyError(which)

def find ( self , which , param ) : for i , layer in enumerate ( self . layers ) : if which == i or which == layer . name : return layer . find ( param ) raise KeyError ( which )

Get a parameter from a layer in the network .


def feed_forward(self, x, **kwargs):
        '''Compute a forward pass of all layers from the given input.

        All keyword arguments are passed directly to :func:`build_graph`.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        layers : list of ndarray (num-examples, num-units)
            The activation values of each layer in the the network when given
            input `x`. For each of the hidden layers, an array is returned
            containing one row per input example; the columns of each array
            correspond to units in the respective layer. The "output" of the
            network is the last element of this list.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        key = self._hash(regs)
        if key not in self._functions:
            outputs, updates = self.build_graph(regs)
            labels, exprs = list(outputs.keys()), list(outputs.values())
            util.log('compiling feed_forward function')
            self._functions[key] = (labels, theano.function(
                self.inputs, exprs, updates=updates))
        labels, f = self._functions[key]
        return dict(zip(labels, f(x)))

def
feed_forward
(
self
,
x
,
**
kwargs
)
:
regs
=
regularizers
.
from_kwargs
(
self
,
**
kwargs
)
key
=
self
.
_hash
(
regs
)
if
key
not
in
self
.
_functions
:
outputs
,
updates
=
self
.
build_graph
(
regs
)
labels
,
exprs
=
list
(
outputs
.
keys
(
)
)
,
list
(
outputs
.
values
(
)
)
util
.
log
(
'compiling feed_forward function'
)
self
.
_functions
[
key
]
=
(
labels
,
theano
.
function
(
self
.
inputs
,
exprs
,
updates
=
updates
)
)
labels
,
f
=
self
.
_functions
[
key
]
return
dict
(
zip
(
labels
,
f
(
x
)
)
)

Compute a forward pass of all layers from the given input .


def predict(self, x, **kwargs):
        '''Compute a forward pass of the inputs, returning the network output.

        All keyword arguments end up being passed to :func:`build_graph`.

        Parameters
        ----------
        x : ndarray (num-examples, num-variables)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        y : ndarray (num-examples, num-variables)
            Returns the values of the network output units when given input `x`.
            Rows in this array correspond to examples, and columns to output
            variables.
        '''
        return self.feed_forward(x, **kwargs)[self.layers[-1].output_name]

def predict ( self , x , ** kwargs ) : return self . feed_forward ( x , ** kwargs ) [ self . layers [ - 1 ] . output_name ]

Compute a forward pass of the inputs returning the network output .


def score(self, x, y, w=None, **kwargs):
        '''Compute R^2 coefficient of determination for a given labeled input.

        Parameters
        ----------
        x : ndarray (num-examples, num-inputs)
            An array containing data to be fed into the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.
        y : ndarray (num-examples, num-outputs)
            An array containing expected target data for the network. Multiple
            examples are arranged as rows in this array, with columns containing
            the variables for each example.

        Returns
        -------
        r2 : float
            The R^2 correlation between the prediction of this netork and its
            target output.
        '''
        u = y - self.predict(x, **kwargs)
        v = y - y.mean()
        if w is None:
            w = np.ones_like(u)
        return 1 - (w * u * u).sum() / (w * v * v).sum()

def score ( self , x , y , w = None , ** kwargs ) : u = y - self . predict ( x , ** kwargs ) v = y - y . mean ( ) if w is None : w = np . ones_like ( u ) return 1 - ( w * u * u ) . sum ( ) / ( w * v * v ) . sum ( )

Compute R^2 coefficient of determination for a given labeled input .


def save(self, filename_or_handle):
        '''Save the state of this network to a pickle file on disk.

        Parameters
        ----------
        filename_or_handle : str or file handle
            Save the state of this network to a pickle file. If this parameter
            is a string, it names the file where the pickle will be saved. If it
            is a file-like object, this object will be used for writing the
            pickle. If the filename ends in ".gz" then the output will
            automatically be gzipped.
        '''
        if isinstance(filename_or_handle, util.basestring):
            opener = gzip.open if filename_or_handle.lower().endswith('.gz') else open
            handle = opener(filename_or_handle, 'wb')
        else:
            handle = filename_or_handle
        pickle.dump(self, handle, -1)
        if isinstance(filename_or_handle, util.basestring):
            handle.close()
        util.log('saved model to {}', filename_or_handle)

def
save
(
self
,
filename_or_handle
)
:
if
isinstance
(
filename_or_handle
,
util
.
basestring
)
:
opener
=
gzip
.
open
if
filename_or_handle
.
lower
(
)
.
endswith
(
'.gz'
)
else
open
handle
=
opener
(
filename_or_handle
,
'wb'
)
else
:
handle
=
filename_or_handle
pickle
.
dump
(
self
,
handle
,
1
)
if
isinstance
(
filename_or_handle
,
util
.
basestring
)
:
handle
.
close
(
)
util
.
log
(
'saved model to {}'
,
filename_or_handle
)

Save the state of this network to a pickle file on disk .


def load(cls, filename_or_handle):
        '''Load a saved network from disk.

        Parameters
        ----------
        filename_or_handle : str or file handle
            Load the state of this network from a pickle file. If this parameter
            is a string, it names the file where the pickle will be saved. If it
            is a file-like object, this object will be used for reading the
            pickle. If the filename ends in ".gz" then the output will
            automatically be gunzipped.
        '''
        assert not isinstance(cls, Network), \
            'cannot load an instance! say instead: net = Network.load(source)'
        if isinstance(filename_or_handle, util.basestring):
            opener = gzip.open if filename_or_handle.lower().endswith('.gz') else open
            handle = opener(filename_or_handle, 'rb')
        else:
            handle = filename_or_handle
        model = pickle.load(handle)
        if isinstance(filename_or_handle, util.basestring):
            handle.close()
        util.log('loaded model from {}', filename_or_handle)
        return model


def loss(self, **kwargs):
        '''Return a variable representing the regularized loss for this network.

        The regularized loss includes both the :ref:`loss computation <losses>`
        for the network as well as any :ref:`regularizers <regularizers>` that
        are in place.

        Keyword arguments are passed directly to
        :func:`theanets.regularizers.from_kwargs`.

        Returns
        -------
        loss : Theano expression
            A Theano expression representing the loss of this network.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        outputs, _ = self.build_graph(regs)
        return sum(l.weight * l(outputs) for l in self.losses) + \
            sum(r.weight * r.loss(self.layers, outputs) for r in regs)

def loss ( self , ** kwargs ) : regs = regularizers . from_kwargs ( self , ** kwargs ) outputs , _ = self . build_graph ( regs ) return sum ( l . weight * l ( outputs ) for l in self . losses ) + sum ( r . weight * r . loss ( self . layers , outputs ) for r in regs )

Return a variable representing the regularized loss for this network .


def updates(self, **kwargs):
        '''Return expressions to run as updates during network training.

        Returns
        -------
        updates : list of (parameter, expression) pairs
            A list of named parameter update expressions for this network.
        '''
        regs = regularizers.from_kwargs(self, **kwargs)
        _, updates = self.build_graph(regs)
        return updates

def updates ( self , ** kwargs ) : regs = regularizers . from_kwargs ( self , ** kwargs ) _ , updates = self . build_graph ( regs ) return updates

Return expressions to run as updates during network training .


def output_size(self):
        '''Number of "neurons" in this layer's default output.'''
        shape = self.output_shape
        if shape is None:
            raise util.ConfigurationError(
                'undefined output size for layer "{}"'.format(self.name))
        return shape[-1]

def
output_size
(
self
)
:
shape
=
self
.
output_shape
if
shape
is
:
raise
util
.
ConfigurationError
(
'undefined output size for layer "{}"'
.
format
(
self
.
name
)
)
return
shape
[
1
]

Number of neurons in this layer s default output .


def connect(self, inputs):
        '''Create Theano variables representing the outputs of this layer.

        Parameters
        ----------
        inputs : dict of Theano expressions
            Symbolic inputs to this layer, given as a dictionary mapping string
            names to Theano expressions. Each string key should be of the form
            "{layer_name}:{output_name}" and refers to a specific output from
            a specific layer in the graph.

        Returns
        -------
        outputs : dict
            A dictionary mapping names to Theano expressions for the outputs
            from this layer.
        updates : sequence of (parameter, expression) tuples
            Updates that should be performed by a Theano function that computes
            something using this layer.
        '''
        outputs, updates = self.transform(inputs)
        # transform the outputs to be a list of ordered pairs if needed.
        if isinstance(outputs, dict):
            outputs = sorted(outputs.items())
        if isinstance(outputs, (TT.TensorVariable, SS.SparseVariable)):
            outputs = [('out', outputs)]
        outs = {self.full_name(name): expr for name, expr in outputs}
        return outs, updates

def connect ( self , inputs ) : outputs , updates = self . transform ( inputs ) if isinstance ( outputs , dict ) : outputs = sorted ( outputs . items ( ) ) if isinstance ( outputs , ( TT . TensorVariable , SS . SparseVariable ) ) : outputs = [ ( 'out' , outputs ) ] outs = { self . full_name ( name ) : expr for name , expr in outputs } return outs , updates

Create Theano variables representing the outputs of this layer .


def bind(self, graph, reset=True, initialize=True):
        '''Bind this layer into a computation graph.

        This method is a wrapper for performing common initialization tasks. It
        calls :func:`resolve`, :func:`setup`, and :func:`log`.

        Parameters
        ----------
        graph : :class:`Network <theanets.graph.Network>`
            A computation network in which this layer is to be bound.
        reset : bool, optional
            If ``True`` (the default), reset the resolved layers for this layer.
        initialize : bool, optional
            If ``True`` (the default), initialize the parameters for this layer
            by calling :func:`setup`.

        Raises
        ------
        theanets.util.ConfigurationError :
            If an input cannot be resolved.
        '''
        if reset:
            for k in self._input_shapes:
                self._input_shapes[k] = None
            for k in self._output_shapes:
                self._output_shapes[k] = None
        self.resolve_inputs(graph.layers)
        self.resolve_outputs()
        self.activate = activations.build(
            self.kwargs.get('activation', 'relu'), self)
        if initialize:
            self.setup()
        self.log()

def bind ( self , graph , reset = True , initialize = True ) : if reset : for k in self . _input_shapes : self . _input_shapes [ k ] = None for k in self . _output_shapes : self . _output_shapes [ k ] = None self . resolve_inputs ( graph . layers ) self . resolve_outputs ( ) self . activate = activations . build ( self . kwargs . get ( 'activation' , 'relu' ) , self ) if initialize : self . setup ( ) self . log ( )

Bind this layer into a computation graph .


def resolve_inputs(self, layers):
        '''Resolve the names of inputs for this layer into shape tuples.

        Parameters
        ----------
        layers : list of :class:`Layer`
            A list of the layers that are available for resolving inputs.

        Raises
        ------
        theanets.util.ConfigurationError :
            If an input cannot be resolved.
        '''
        resolved = {}
        for name, shape in self._input_shapes.items():
            if shape is None:
                name, shape = self._resolve_shape(name, layers)
            resolved[name] = shape
        self._input_shapes = resolved

def resolve_inputs ( self , layers ) : resolved = { } for name , shape in self . _input_shapes . items ( ) : if shape is None : name , shape = self . _resolve_shape ( name , layers ) resolved [ name ] = shape self . _input_shapes = resolved

Resolve the names of inputs for this layer into shape tuples .


def resolve_outputs(self):
        '''Resolve the names of outputs for this layer into shape tuples.'''
        input_shape = None
        for i, shape in enumerate(self._input_shapes.values()):
            if i == 0:
                input_shape = shape
            if len(input_shape) != len(shape) or any(
                    a is not None and b is not None and a != b
                    for a, b in zip(input_shape[:-1], shape[:-1])):
                raise util.ConfigurationError(
                    'layer "{}" incompatible input shapes {}'
                    .format(self.name, self._input_shapes))
        size = self.kwargs.get('size')
        shape = self.kwargs.get('shape')
        if shape is not None:
            pass
        elif size is not None:
            shape = tuple(input_shape[:-1]) + (size, )
        else:
            raise util.ConfigurationError(
                'layer "{}" does not specify a size'.format(self.name))
        self._output_shapes['out'] = shape

Resolve the names of outputs for this layer into shape tuples .


def log(self):
        '''Log some information about this layer.'''
        inputs = ', '.join('"{0}" {1}'.format(*ns) for ns in self._input_shapes.items())
        util.log('layer {0.__class__.__name__} "{0.name}" {0.output_shape} {1} from {2}',
                 self, getattr(self.activate, 'name', self.activate), inputs)
        util.log('learnable parameters: {}', self.log_params())


def log_params(self):
        '''Log information about this layer's parameters.'''
        total = 0
        for p in self.params:
            shape = p.get_value().shape
            util.log('parameter "{}" {}', p.name, shape)
            total += np.prod(shape)
        return total

def
log_params
(
self
)
:
total
=
0
for
p
in
self
.
params
:
shape
=
p
.
get_value
(
)
.
shape
util
.
log
(
'parameter "{}" {}'
,
p
.
name
,
shape
)
total
+=
np
.
prod
(
shape
)
return
total

Log information about this layer s parameters .


def _fmt(self, string):
        '''Helper method to format our name into a string.'''
        if '{' not in string:
            string = '{}.' + string
        return string.format(self.name)

def _fmt ( self , string ) : if '{' not in string : string = '{}.' + string return string . format ( self . name )

Helper method to format our name into a string .


def _resolve_shape(self, name, layers):
        '''Given a list of layers, find the layer output with the given name.

        Parameters
        ----------
        name : str
            Name of a layer to resolve.
        layers : list of :class:`theanets.layers.base.Layer`
            A list of layers to search in.

        Raises
        ------
        util.ConfigurationError :
            If there is no such layer, or if there are more than one.

        Returns
        -------
        name : str
            The fully-scoped name of the desired output.
        shape : tuple of None and/or int
            The shape of the named output.
        '''
        matches = [l for l in layers if name.split(':')[0] == l.name]
        if len(matches) != 1:
            raise util.ConfigurationError(
                'layer "{}" cannot resolve "{}" using {}'
                .format(self.name, name, [l.name for l in layers]))
        name = name if ':' in name else matches[0].output_name
        return name, matches[0]._output_shapes[name.split(':')[1]]

Given a list of layers find the layer output with the given name .


def find(self, key):
        '''Get a shared variable for a parameter by name.

        Parameters
        ----------
        key : str or int
            The name of the parameter to look up, or the index of the parameter
            in our parameter list. These are both dependent on the
            implementation of the layer.

        Returns
        -------
        param : shared variable
            A shared variable containing values for the given parameter.

        Raises
        ------
        KeyError
            If a param with the given name does not exist.
        '''
        name = self._fmt(str(key))
        for i, p in enumerate(self._params):
            if key == i or name == p.name:
                return p
        raise KeyError(key)

def find ( self , key ) : name = self . _fmt ( str ( key ) ) for i , p in enumerate ( self . _params ) : if key == i or name == p . name : return p raise KeyError ( key )

Get a shared variable for a parameter by name .


def add_bias(self, name, size, mean=0, std=1):
        '''Helper method to create a new bias vector.

        Parameters
        ----------
        name : str
            Name of the parameter to add.
        size : int
            Size of the bias vector.
        mean : float, optional
            Mean value for randomly-initialized biases. Defaults to 0.
        std : float, optional
            Standard deviation for randomly-initialized biases. Defaults to 1.
        '''
        mean = self.kwargs.get('mean_{}'.format(name), mean)
        std = self.kwargs.get('std_{}'.format(name), std)
        self._params.append(theano.shared(
            util.random_vector(size, mean, std, rng=self.rng),
            name=self._fmt(name)))

def add_bias ( self , name , size , mean = 0 , std = 1 ) : mean = self . kwargs . get ( 'mean_{}' . format ( name ) , mean ) std = self . kwargs . get ( 'std_{}' . format ( name ) , std ) self . _params . append ( theano . shared ( util . random_vector ( size , mean , std , rng = self . rng ) , name = self . _fmt ( name ) ) )

Helper method to create a new bias vector .


def to_spec(self):
        '''Create a specification dictionary for this layer.

        Returns
        -------
        spec : dict
            A dictionary specifying the configuration of this layer.
        '''
        spec = dict(**self.kwargs)
        spec.update(
            form=self.__class__.__name__.lower(),
            name=self.name,
            activation=self.kwargs.get('activation', 'relu'),
        )
        return spec

def to_spec ( self ) : spec = dict ( ** self . kwargs ) spec . update ( form = self . __class__ . __name__ . lower ( ) , name = self . name , activation = self . kwargs . get ( 'activation' , 'relu' ) , ) return spec

Create a specification dictionary for this layer .


def loggabor(self, x_pos, y_pos, sf_0, B_sf, theta, B_theta, preprocess=True):
        """
        Returns the envelope of a LogGabor

        Note that the convention for coordinates follows that of matrices:
        the origin is at the top left of the image, and coordinates are first
        the rows (vertical axis, going down) then the columns (horizontal axis,
        going right).

        """

        env = np.multiply(self.band(sf_0, B_sf), self.orientation(theta, B_theta))
        if not(x_pos==0.) and not(y_pos==0.): # bypass translation whenever none is needed
              env = env.astype(np.complex128) * self.trans(x_pos*1., y_pos*1.)
        if preprocess : env *= self.f_mask # retina processing
        # normalizing energy:
        env /= np.sqrt((np.abs(env)**2).mean())
        # in the case a a single bump (see ``orientation``), we should compensate the fact that the distribution gets complex:
        env *= np.sqrt(2.)
        return env

def loggabor ( self , x_pos , y_pos , sf_0 , B_sf , theta , B_theta , preprocess = True ) : env = np . multiply ( self . band ( sf_0 , B_sf ) , self . orientation ( theta , B_theta ) ) if not ( x_pos == 0. ) and not ( y_pos == 0. ) : env = env . astype ( np . complex128 ) * self . trans ( x_pos * 1. , y_pos * 1. ) if preprocess : env *= self . f_mask env /= np . sqrt ( ( np . abs ( env ) ** 2 ) . mean ( ) ) env *= np . sqrt ( 2. ) return env


def loggabor_image(self, x_pos, y_pos, theta, sf_0, phase, B_sf, B_theta):
        """
        Returns the image of a LogGabor

        Note that the convention for coordinates follows that of matrices:
        the origin is at the top left of the image, and coordinates are first
        the rows (vertical axis, going down) then the columns (horizontal axis,
        going right).

        """

        FT_lg = self.loggabor(x_pos, y_pos, sf_0=sf_0, B_sf=B_sf, theta=theta, B_theta=B_theta)
        FT_lg = FT_lg * np.exp(1j * phase)
        return self.invert(FT_lg, full=False)

def loggabor_image ( self , x_pos , y_pos , theta , sf_0 , phase , B_sf , B_theta ) : FT_lg = self . loggabor ( x_pos , y_pos , sf_0 = sf_0 , B_sf = B_sf , theta = theta , B_theta = B_theta ) FT_lg = FT_lg * np . exp ( 1j * phase ) return self . invert ( FT_lg , full = False )


def add_tier(self, name, tier_type='IntervalTier', number=None):
        """Add an IntervalTier or a TextTier on the specified location.

        :param str name: Name of the tier, duplicate names is allowed.
        :param str tier_type: Type of the tier.
        :param int number: Place to insert the tier, when ``None`` the number
            is generated and the tier will be placed on the bottom.
        :returns: The created tier.
        :raises ValueError: If the number is out of bounds.
        """
        if number is None:
            number = 1 if not self.tiers else len(self.tiers)+1
        elif number < 1 or number > len(self.tiers):
            raise ValueError('Number not in [1..{}]'.format(len(self.tiers)))
        elif tier_type not in Tier.P_TIERS:
            raise ValueError('tier_type has to be in {}'.format(self.P_TIERS))
        self.tiers.insert(number-1,
                          Tier(self.xmin, self.xmax, name, tier_type))
        return self.tiers[number-1]

def
add_tier
(
self
,
name
,
tier_type
=
'IntervalTier'
,
number
=
)
:
if
number
is
:
number
=
1
if
not
self
.
tiers
else
len
(
self
.
tiers
)
+
1
elif
number
<
1
or
number
>
len
(
self
.
tiers
)
:
raise
ValueError
(
'Number not in [1..{}]'
.
format
(
len
(
self
.
tiers
)
)
)
elif
tier_type
not
in
Tier
.
P_TIERS
:
raise
ValueError
(
'tier_type has to be in {}'
.
format
(
self
.
P_TIERS
)
)
self
.
tiers
.
insert
(
number
1
,
Tier
(
self
.
xmin
,
self
.
xmax
,
name
,
tier_type
)
)
return
self
.
tiers
[
number
1
]

Add an IntervalTier or a TextTier on the specified location .


def remove_tier(self, name_num):
        """Remove a tier, when multiple tiers exist with that name only the
        first is removed.

        :param name_num: Name or number of the tier to remove.
        :type name_num: int or str
        :raises IndexError: If there is no tier with that number.
        """
        if isinstance(name_num, int):
            del(self.tiers[name_num-1])
        else:
            self.tiers = [i for i in self.tiers if i.name != name_num]

def remove_tier ( self , name_num ) : if isinstance ( name_num , int ) : del ( self . tiers [ name_num - 1 ] ) else : self . tiers = [ i for i in self . tiers if i . name != name_num ]

Remove a tier when multiple tiers exist with that name only the first is removed .


def get_tier(self, name_num):
        """Gives a tier, when multiple tiers exist with that name only the
        first is returned.

        :param name_num: Name or number of the tier to return.
        :type name_num: int or str
        :returns: The tier.
        :raises IndexError: If the tier doesn't exist.
        """
        return self.tiers[name_num - 1] if isinstance(name_num, int) else\
            [i for i in self.tiers if i.name == name_num][0]

def get_tier ( self , name_num ) : return self . tiers [ name_num - 1 ] if isinstance ( name_num , int ) else [ i for i in self . tiers if i . name == name_num ] [ 0 ]

Gives a tier when multiple tiers exist with that name only the first is returned .


def to_eaf(self, skipempty=True, pointlength=0.1):
        """Convert the object to an pympi.Elan.Eaf object

        :param int pointlength: Length of respective interval from points in
                                seconds
        :param bool skipempty: Skip the empty annotations
        :returns: :class:`pympi.Elan.Eaf` object
        :raises ImportError: If the Eaf module can't be loaded.
        :raises ValueError: If the pointlength is not strictly positive.
        """
        from pympi.Elan import Eaf
        eaf_out = Eaf()
        if pointlength <= 0:
            raise ValueError('Pointlength should be strictly positive')
        for tier in self.get_tiers():
            eaf_out.add_tier(tier.name)
            for ann in tier.get_intervals(True):
                if tier.tier_type == 'TextTier':
                    ann = (ann[0], ann[0]+pointlength, ann[1])
                if ann[2].strip() or not skipempty:
                    eaf_out.add_annotation(tier.name, int(round(ann[0]*1000)),
                                           int(round(ann[1]*1000)), ann[2])
        return eaf_out

Convert the object to an pympi . Elan . Eaf object


def add_point(self, point, value, check=True):
        """Add a point to the TextTier

        :param int point: Time of the point.
        :param str value: Text of the point.
        :param bool check: Flag to check for overlap.
        :raises Exception: If overlap or wrong tiertype.
        """
        if self.tier_type != 'TextTier':
            raise Exception('Tiertype must be TextTier.')
        if check and any(i for i in self.intervals if i[0] == point):
                raise Exception('No overlap is allowed')
        self.intervals.append((point, value))

def
add_point
(
self
,
point
,
value
,
check
=
True
)
:
if
self
.
tier_type
!=
'TextTier'
:
raise
Exception
(
'Tiertype must be TextTier.'
)
if
check
and
any
(
i
for
i
in
self
.
intervals
if
i
[
0
]
==
point
)
:
raise
Exception
(
'No overlap is allowed'
)
self
.
intervals
.
append
(
(
point
,
value
)
)


def add_interval(self, begin, end, value, check=True):
        """Add an interval to the IntervalTier.

        :param float begin: Start time of the interval.
        :param float end: End time of the interval.
        :param str value: Text of the interval.
        :param bool check: Flag to check for overlap.
        :raises Exception: If overlap, begin > end or wrong tiertype.
        """
        if self.tier_type != 'IntervalTier':
            raise Exception('Tiertype must be IntervalTier')
        if check:
            if any(i for i in self.intervals if begin < i[1] and end > i[0]):
                raise Exception('No overlap is allowed')
            if begin > end:
                raise Exception('Begin must be smaller then end')
        self.intervals.append((begin, end, value))

def
add_interval
(
self
,
begin
,
end
,
value
,
check
=
True
)
:
if
self
.
tier_type
!=
'IntervalTier'
:
raise
Exception
(
'Tiertype must be IntervalTier'
)
if
check
:
if
any
(
i
for
i
in
self
.
intervals
if
begin
<
i
[
1
]
and
end
>
i
[
0
]
)
:
raise
Exception
(
'No overlap is allowed'
)
if
begin
>
end
:
raise
Exception
(
'Begin must be smaller then end'
)
self
.
intervals
.
append
(
(
begin
,
end
,
value
)
)


def remove_interval(self, time):
        """Remove an interval, if no interval is found nothing happens.

        :param int time: Time of the interval.
        :raises TierTypeException: If the tier is not a IntervalTier.
        """
        if self.tier_type != 'IntervalTier':
            raise Exception('Tiertype must be IntervalTier.')
        self.intervals = [i for i in self.intervals
                          if not(i[0] <= time and i[1] >= time)]

def
remove_interval
(
self
,
time
)
:
if
self
.
tier_type
!=
'IntervalTier'
:
raise
Exception
(
'Tiertype must be IntervalTier.'
)
self
.
intervals
=
[
i
for
i
in
self
.
intervals
if
not
(
i
[
0
]
<=
time
and
i
[
1
]
>=
time
)
]

Remove an interval if no interval is found nothing happens .


def remove_point(self, time):
        """Remove a point, if no point is found nothing happens.

        :param int time: Time of the point.
        :raises TierTypeException: If the tier is not a TextTier.
        """
        if self.tier_type != 'TextTier':
            raise Exception('Tiertype must be TextTier.')
        self.intervals = [i for i in self.intervals if i[0] != time]

def
remove_point
(
self
,
time
)
:
if
self
.
tier_type
!=
'TextTier'
:
raise
Exception
(
'Tiertype must be TextTier.'
)
self
.
intervals
=
[
i
for
i
in
self
.
intervals
if
i
[
0
]
!=
time
]

Remove a point if no point is found nothing happens .


def get_intervals(self, sort=False):
        """Give all the intervals or points.

        :param bool sort: Flag for yielding the intervals or points sorted.
        :yields: All the intervals
        """
        for i in sorted(self.intervals) if sort else self.intervals:
            yield i

def get_intervals ( self , sort = False ) : for i in sorted ( self . intervals ) if sort else self . intervals : yield i


def get_all_intervals(self):
        """Returns the true list of intervals including the empty intervals."""
        ints = sorted(self.get_intervals(True))
        if self.tier_type == 'IntervalTier':
            if not ints:
                ints.append((self.xmin, self.xmax, ''))
            else:
                if ints[0][0] > self.xmin:
                    ints.insert(0, (self.xmin, ints[0][0], ''))
                if ints[-1][1] < self.xmax:
                    ints.append((ints[-1][1], self.xmax, ''))
                p = ints[-1]
                for index, i in reversed(list(enumerate(ints[:-1], 1))):
                    if p[0] - i[1] != 0:
                        ints.insert(index, (i[1], p[0], ''))
                    p = i
        return ints

def get_all_intervals ( self ) : ints = sorted ( self . get_intervals ( True ) ) if self . tier_type == 'IntervalTier' : if not ints : ints . append ( ( self . xmin , self . xmax , '' ) ) else : if ints [ 0 ] [ 0 ] > self . xmin : ints . insert ( 0 , ( self . xmin , ints [ 0 ] [ 0 ] , '' ) ) if ints [ - 1 ] [ 1 ] < self . xmax : ints . append ( ( ints [ - 1 ] [ 1 ] , self . xmax , '' ) ) p = ints [ - 1 ] for index , i in reversed ( list ( enumerate ( ints [ : - 1 ] , 1 ) ) ) : if p [ 0 ] - i [ 1 ] != 0 : ints . insert ( index , ( i [ 1 ] , p [ 0 ] , '' ) ) p = i return ints

Returns the true list of intervals including the empty intervals .


def indent(el, level=0):
    """Function to pretty print the xml, meaning adding tabs and newlines.

    :param ElementTree.Element el: Current element.
    :param int level: Current level.
    """
    i = '\n' + level * '\t'
    if len(el):
        if not el.text or not el.text.strip():
            el.text = i+'\t'
        if not el.tail or not el.tail.strip():
            el.tail = i
        for elem in el:
            indent(elem, level+1)
        if not el.tail or not el.tail.strip():
            el.tail = i
    else:
        if level and (not el.tail or not el.tail.strip()):
            el.tail = i

def indent ( el , level = 0 ) : i = '\n' + level * '\t' if len ( el ) : if not el . text or not el . text . strip ( ) : el . text = i + '\t' if not el . tail or not el . tail . strip ( ) : el . tail = i for elem in el : indent ( elem , level + 1 ) if not el . tail or not el . tail . strip ( ) : el . tail = i else : if level and ( not el . tail or not el . tail . strip ( ) ) : el . tail = i

Function to pretty print the xml meaning adding tabs and newlines .


def add_annotation(self, id_tier, start, end, value='', svg_ref=None):
        """Add an annotation.

        :param str id_tier: Name of the tier.
        :param int start: Start time of the annotation.
        :param int end: End time of the annotation.
        :param str value: Value of the annotation.
        :param str svg_ref: Svg reference.
        :raises KeyError: If the tier is non existent.
        :raises ValueError: If one of the values is negative or start is bigger
                            then end or if the tiers already contains ref
                            annotations.
        """
        if self.tiers[id_tier][1]:
            raise ValueError('Tier already contains ref annotations...')
        if start == end:
            raise ValueError('Annotation length is zero...')
        if start > end:
            raise ValueError('Annotation length is negative...')
        if start < 0:
            raise ValueError('Start is negative...')
        start_ts = self.generate_ts_id(start)
        end_ts = self.generate_ts_id(end)
        aid = self.generate_annotation_id()
        self.annotations[aid] = id_tier
        self.tiers[id_tier][0][aid] = (start_ts, end_ts, value, svg_ref)


def add_cv_entry(self, cv_id, cve_id, values, ext_ref=None):
        """Add an entry to a controlled vocabulary.

        :param str cv_id: Name of the controlled vocabulary to add an entry.
        :param str cve_id: Name of the entry.
        :param list values: List of values of the form:
            ``(value, lang_ref, description)`` where description can be
            ``None``.
        :param str ext_ref: External reference.
        :throws KeyError: If there is no controlled vocabulary with that id.
        :throws ValueError: If a language in one of the entries doesn't exist.
        """
        for value, lang_ref, description in values:
            if lang_ref not in self.languages:
                raise ValueError('Language not present: {}'.format(lang_ref))
        self.controlled_vocabularies[cv_id][1][cve_id] = (values, ext_ref)

def
add_cv_entry
(
self
,
cv_id
,
cve_id
,
values
,
ext_ref
=
)
:
for
value
,
lang_ref
,
description
in
values
:
if
lang_ref
not
in
self
.
languages
:
raise
ValueError
(
'Language not present: {}'
.
format
(
lang_ref
)
)
self
.
controlled_vocabularies
[
cv_id
]
[
1
]
[
cve_id
]
=
(
values
,
ext_ref
)

Add an entry to a controlled vocabulary .


def add_cv_description(self, cv_id, lang_ref, description=None):
        """Add a description to a controlled vocabulary.

        :param str cv_id: Name of the controlled vocabulary to add the
            description.
        :param str lang_ref: Language reference.
        :param str description: Description, this can be none.
        :throws KeyError: If there is no controlled vocabulary with that id.
        :throws ValueError: If the language provided doesn't exist.
        """
        if lang_ref not in self.languages:
            raise ValueError('Language not present: {}'.format(lang_ref))
        self.controlled_vocabularies[cv_id][0].append((lang_ref, description))

def
add_cv_description
(
self
,
cv_id
,
lang_ref
,
description
=
)
:
if
lang_ref
not
in
self
.
languages
:
raise
ValueError
(
'Language not present: {}'
.
format
(
lang_ref
)
)
self
.
controlled_vocabularies
[
cv_id
]
[
0
]
.
append
(
(
lang_ref
,
description
)
)

Add a description to a controlled vocabulary .


def add_external_ref(self, eid, etype, value):
        """Add an external reference.

        :param str eid: Name of the external reference.
        :param str etype: Type of the external reference, has to be in
            ``['iso12620', 'ecv', 'cve_id', 'lexen_id', 'resource_url']``.
        :param str value: Value of the external reference.
        :throws KeyError: if etype is not in the list of possible types.
        """
        if etype not in self.ETYPES:
            raise KeyError('etype not in {}'.format(self.ETYPES))
        self.external_refs[eid] = (etype, value)

def
add_external_ref
(
self
,
eid
,
etype
,
value
)
:
if
etype
not
in
self
.
ETYPES
:
raise
KeyError
(
'etype not in {}'
.
format
(
self
.
ETYPES
)
)
self
.
external_refs
[
eid
]
=
(
etype
,
value
)


def add_language(self, lang_id, lang_def=None, lang_label=None):
        """Add a language.

        :param str lang_id: ID of the language.
        :param str lang_def: Definition of the language(preferably ISO-639-3).
        :param str lang_label: Label of the language.
        """
        self.languages[lang_id] = (lang_def, lang_label)

def add_language ( self , lang_id , lang_def = None , lang_label = None ) : self . languages [ lang_id ] = ( lang_def , lang_label )


def add_lexicon_ref(self, lrid, name, lrtype, url, lexicon_id,
                        lexicon_name, datcat_id=None, datcat_name=None):
        """Add lexicon reference.

        :param str lrid: Lexicon reference internal ID.
        :param str name: Lexicon reference display name.
        :param str lrtype: Lexicon reference service type.
        :param str url: Lexicon reference service location
        :param str lexicon_id: Lexicon reference service id.
        :param str lexicon_name: Lexicon reference service name.
        :param str datacat_id: Lexicon reference identifier of data category.
        :param str datacat_name: Lexicon reference name of data category.
        """
        self.lexicon_refs[lrid] = {
            'LEX_REF_ID': lrid,
            'NAME': name,
            'TYPE': lrtype,
            'URL': url,
            'LEXICON_ID': lexicon_id,
            'LEXICON_NAME': lexicon_name,
            'DATCAT_ID': datcat_id,
            'DATCAT_NAME': datcat_name
            }

def add_lexicon_ref ( self , lrid , name , lrtype , url , lexicon_id , lexicon_name , datcat_id = None , datcat_name = None ) : self . lexicon_refs [ lrid ] = { 'LEX_REF_ID' : lrid , 'NAME' : name , 'TYPE' : lrtype , 'URL' : url , 'LEXICON_ID' : lexicon_id , 'LEXICON_NAME' : lexicon_name , 'DATCAT_ID' : datcat_id , 'DATCAT_NAME' : datcat_name }


def add_linguistic_type(self, lingtype, constraints=None,
                            timealignable=True, graphicreferences=False,
                            extref=None, param_dict=None):
        """Add a linguistic type.

        :param str lingtype: Name of the linguistic type.
        :param str constraints: Constraint name.
        :param bool timealignable: Flag for time alignable.
        :param bool graphicreferences: Flag for graphic references.
        :param str extref: External reference.
        :param dict param_dict: TAG attributes, when this is not ``None`` it
                                 will ignore all other options. Please only use
                                 dictionaries coming from the
                                 :func:`get_parameters_for_linguistic_type`
        :raises KeyError: If a constraint is not defined
        """
        if param_dict:
            self.linguistic_types[lingtype] = param_dict
        else:
            if constraints:
                self.constraints[constraints]
            self.linguistic_types[lingtype] = {
                'LINGUISTIC_TYPE_ID': lingtype,
                'TIME_ALIGNABLE': str(timealignable).lower(),
                'GRAPHIC_REFERENCES': str(graphicreferences).lower(),
                'CONSTRAINTS': constraints}
            if extref is not None:
                self.linguistic_types[lingtype]['EXT_REF'] = extref

def add_linguistic_type ( self , lingtype , constraints = None , timealignable = True , graphicreferences = False , extref = None , param_dict = None ) : if param_dict : self . linguistic_types [ lingtype ] = param_dict else : if constraints : self . constraints [ constraints ] self . linguistic_types [ lingtype ] = { 'LINGUISTIC_TYPE_ID' : lingtype , 'TIME_ALIGNABLE' : str ( timealignable ) . lower ( ) , 'GRAPHIC_REFERENCES' : str ( graphicreferences ) . lower ( ) , 'CONSTRAINTS' : constraints } if extref is not None : self . linguistic_types [ lingtype ] [ 'EXT_REF' ] = extref


def add_linked_file(self, file_path, relpath=None, mimetype=None,
                        time_origin=None, ex_from=None):
        """Add a linked file.

        :param str file_path: Path of the file.
        :param str relpath: Relative path of the file.
        :param str mimetype: Mimetype of the file, if ``None`` it tries to
            guess it according to the file extension which currently only works
            for wav, mpg, mpeg and xml.
        :param int time_origin: Time origin for the media file.
        :param str ex_from: Extracted from field.
        :raises KeyError: If mimetype had to be guessed and a non standard
                          extension or an unknown mimetype.
        """
        if mimetype is None:
            mimetype = self.MIMES[file_path.split('.')[-1]]
        self.media_descriptors.append({
            'MEDIA_URL': file_path, 'RELATIVE_MEDIA_URL': relpath,
            'MIME_TYPE': mimetype, 'TIME_ORIGIN': time_origin,
            'EXTRACTED_FROM': ex_from})

def add_linked_file ( self , file_path , relpath = None , mimetype = None , time_origin = None , ex_from = None ) : if mimetype is None : mimetype = self . MIMES [ file_path . split ( '.' ) [ - 1 ] ] self . media_descriptors . append ( { 'MEDIA_URL' : file_path , 'RELATIVE_MEDIA_URL' : relpath , 'MIME_TYPE' : mimetype , 'TIME_ORIGIN' : time_origin , 'EXTRACTED_FROM' : ex_from } )


def add_locale(self, language_code, country_code=None, variant=None):
        """Add a locale.

        :param str language_code: The language code of the locale.
        :param str country_code: The country code of the locale.
        :param str variant: The variant of the locale.
        """
        self.locales[language_code] = (country_code, variant)

def add_locale ( self , language_code , country_code = None , variant = None ) : self . locales [ language_code ] = ( country_code , variant )


def add_secondary_linked_file(self, file_path, relpath=None, mimetype=None,
                                  time_origin=None, assoc_with=None):
        """Add a secondary linked file.

        :param str file_path: Path of the file.
        :param str relpath: Relative path of the file.
        :param str mimetype: Mimetype of the file, if ``None`` it tries to
            guess it according to the file extension which currently only works
            for wav, mpg, mpeg and xml.
        :param int time_origin: Time origin for the media file.
        :param str assoc_with: Associated with field.
        :raises KeyError: If mimetype had to be guessed and a non standard
                          extension or an unknown mimetype.
        """
        if mimetype is None:
            mimetype = self.MIMES[file_path.split('.')[-1]]
        self.linked_file_descriptors.append({
            'LINK_URL': file_path, 'RELATIVE_LINK_URL': relpath,
            'MIME_TYPE': mimetype, 'TIME_ORIGIN': time_origin,
            'ASSOCIATED_WITH': assoc_with})

def add_secondary_linked_file ( self , file_path , relpath = None , mimetype = None , time_origin = None , assoc_with = None ) : if mimetype is None : mimetype = self . MIMES [ file_path . split ( '.' ) [ - 1 ] ] self . linked_file_descriptors . append ( { 'LINK_URL' : file_path , 'RELATIVE_LINK_URL' : relpath , 'MIME_TYPE' : mimetype , 'TIME_ORIGIN' : time_origin , 'ASSOCIATED_WITH' : assoc_with } )


def add_tier(self, tier_id, ling='default-lt', parent=None, locale=None,
                 part=None, ann=None, language=None, tier_dict=None):
        """Add a tier. When no linguistic type is given and the default
        linguistic type is unavailable then the assigned linguistic type will
        be the first in the list.

        :param str tier_id: Name of the tier.
        :param str ling: Linguistic type, if the type is not available it will
                         warn and pick the first available type.
        :param str parent: Parent tier name.
        :param str locale: Locale, if the locale is not present this option is
            ignored and the locale will not be set.
        :param str part: Participant.
        :param str ann: Annotator.
        :param str language: Language , if the language is not present this
            option is ignored and the language will not be set.
        :param dict tier_dict: TAG attributes, when this is not ``None`` it
                               will ignore all other options. Please only use
                               dictionaries coming from the
                               :func:`get_parameters_for_tier`
        :raises ValueError: If the tier_id is empty
        """
        if not tier_id:
            raise ValueError('Tier id is empty...')
        if ling not in self.linguistic_types:
            ling = sorted(self.linguistic_types.keys())[0]
        if locale and locale not in self.locales:
            locale = None
        if language and language not in self.languages:
            language = None
        if tier_dict is None:
            self.tiers[tier_id] = ({}, {}, {
                'TIER_ID': tier_id,
                'LINGUISTIC_TYPE_REF': ling,
                'PARENT_REF': parent,
                'PARTICIPANT': part,
                'DEFAULT_LOCALE': locale,
                'LANG_REF': language,
                'ANNOTATOR': ann}, len(self.tiers))
        else:
            self.tiers[tier_id] = ({}, {}, tier_dict, len(self.tiers))

def
add_tier
(
self
,
tier_id
,
ling
=
'default-lt'
,
parent
=
,
locale
=
,
part
=
,
ann
=
,
language
=
,
tier_dict
=
)
:
if
not
tier_id
:
raise
ValueError
(
'Tier id is empty...'
)
if
ling
not
in
self
.
linguistic_types
:
ling
=
sorted
(
self
.
linguistic_types
.
keys
(
)
)
[
0
]
if
locale
and
locale
not
in
self
.
locales
:
locale
=
if
language
and
language
not
in
self
.
languages
:
language
=
if
tier_dict
is
:
self
.
tiers
[
tier_id
]
=
(
{
}
,
{
}
,
{
'TIER_ID'
:
tier_id
,
'LINGUISTIC_TYPE_REF'
:
ling
,
'PARENT_REF'
:
parent
,
'PARTICIPANT'
:
part
,
'DEFAULT_LOCALE'
:
locale
,
'LANG_REF'
:
language
,
'ANNOTATOR'
:
ann
}
,
len
(
self
.
tiers
)
)
else
:
self
.
tiers
[
tier_id
]
=
(
{
}
,
{
}
,
tier_dict
,
len
(
self
.
tiers
)
)

Add a tier . When no linguistic type is given and the default linguistic type is unavailable then the assigned linguistic type will be the first in the list .


def clean_time_slots(self):
        """Clean up all unused timeslots.

        .. warning:: This can and will take time for larger tiers.

        When you want to do a lot of operations on a lot of tiers please unset
        the flags for cleaning in the functions so that the cleaning is only
        performed afterwards.
        """
        ts = ((a[0], a[1]) for t in self.tiers.values() for a in t[0].values())
        for a in {a for b in ts for a in b} ^ set(self.timeslots):
            del(self.timeslots[a])

def clean_time_slots ( self ) : ts = ( ( a [ 0 ] , a [ 1 ] ) for t in self . tiers . values ( ) for a in t [ 0 ] . values ( ) ) for a in { a for b in ts for a in b } ^ set ( self . timeslots ) : del ( self . timeslots [ a ] )


def extract(self, start, end):
        """Extracts the selected time frame as a new object.

        :param int start: Start time.
        :param int end: End time.
        :returns: class:`pympi.Elan.Eaf` object containing the extracted frame.
        """
        from copy import deepcopy
        eaf_out = deepcopy(self)
        for t in eaf_out.get_tier_names():
            for ab, ae, value in eaf_out.get_annotation_data_for_tier(t):
                if ab > end or ae < start:
                    eaf_out.remove_annotation(t, (start-end)//2, False)
        eaf_out.clean_time_slots()
        return eaf_out

def extract ( self , start , end ) : from copy import deepcopy eaf_out = deepcopy ( self ) for t in eaf_out . get_tier_names ( ) : for ab , ae , value in eaf_out . get_annotation_data_for_tier ( t ) : if ab > end or ae < start : eaf_out . remove_annotation ( t , ( start - end ) // 2 , False ) eaf_out . clean_time_slots ( ) return eaf_out

Extracts the selected time frame as a new object .


def generate_annotation_id(self):
        """Generate the next annotation id, this function is mainly used
        internally.
        """
        if not self.maxaid:
            valid_anns = [int(''.join(filter(str.isdigit, a)))
                          for a in self.timeslots]
            self.maxaid = max(valid_anns + [1])+1
        else:
            self.maxaid += 1
        return 'a{:d}'.format(self.maxaid)

def generate_annotation_id ( self ) : if not self . maxaid : valid_anns = [ int ( '' . join ( filter ( str . isdigit , a ) ) ) for a in self . timeslots ] self . maxaid = max ( valid_anns + [ 1 ] ) + 1 else : self . maxaid += 1 return 'a{:d}' . format ( self . maxaid )

Generate the next annotation id this function is mainly used internally .


def generate_ts_id(self, time=None):
        """Generate the next timeslot id, this function is mainly used
        internally

        :param int time: Initial time to assign to the timeslot.
        :raises ValueError: If the time is negative.
        """
        if time and time < 0:
            raise ValueError('Time is negative...')
        if not self.maxts:
            valid_ts = [int(''.join(filter(str.isdigit, a)))
                        for a in self.timeslots]
            self.maxts = max(valid_ts + [1])+1
        else:
            self.maxts += 1
        ts = 'ts{:d}'.format(self.maxts)
        self.timeslots[ts] = time
        return ts

def
generate_ts_id
(
self
,
time
=
)
:
if
time
and
time
<
0
:
raise
ValueError
(
'Time is negative...'
)
if
not
self
.
maxts
:
valid_ts
=
[
int
(
''
.
join
(
filter
(
str
.
isdigit
,
a
)
)
)
for
a
in
self
.
timeslots
]
self
.
maxts
=
max
(
valid_ts
+
[
1
]
)
+
1
else
:
self
.
maxts
+=
1
ts
=
'ts{:d}'
.
format
(
self
.
maxts
)
self
.
timeslots
[
ts
]
=
time
return
ts

Generate the next timeslot id this function is mainly used internally


def get_child_tiers_for(self, id_tier):
        """Give all child tiers for a tier.

        :param str id_tier: Name of the tier.
        :returns: List of all children
        :raises KeyError: If the tier is non existent.
        """
        self.tiers[id_tier]
        return [m for m in self.tiers if 'PARENT_REF' in self.tiers[m][2] and
                self.tiers[m][2]['PARENT_REF'] == id_tier]

def get_child_tiers_for ( self , id_tier ) : self . tiers [ id_tier ] return [ m for m in self . tiers if 'PARENT_REF' in self . tiers [ m ] [ 2 ] and self . tiers [ m ] [ 2 ] [ 'PARENT_REF' ] == id_tier ]


def get_full_time_interval(self):
        """Give the full time interval of the file. Note that the real interval
        can be longer because the sound file attached can be longer.

        :returns: Tuple of the form: ``(min_time, max_time)``.
        """
        return (0, 0) if not self.timeslots else\
            (min(self.timeslots.values()), max(self.timeslots.values()))

def get_full_time_interval ( self ) : return ( 0 , 0 ) if not self . timeslots else ( min ( self . timeslots . values ( ) ) , max ( self . timeslots . values ( ) ) )

Give the full time interval of the file . Note that the real interval can be longer because the sound file attached can be longer .


def get_ref_annotation_data_after_time(self, id_tier, time):
        """Give the ref annotation after a time. If an annotation overlaps
        with `ktime`` that annotation will be returned.

        :param str id_tier: Name of the tier.
        :param int time: Time to get the annotation after.
        :returns: Annotation after that time in a list
        :raises KeyError: If the tier is non existent.
        """
        befores = self.get_ref_annotation_data_between_times(
            id_tier, time, self.get_full_time_interval())
        if befores:
            return [min(befores, key=lambda x: x[0])]
        else:
            return []

def get_ref_annotation_data_after_time ( self , id_tier , time ) : befores = self . get_ref_annotation_data_between_times ( id_tier , time , self . get_full_time_interval ( ) ) if befores : return [ min ( befores , key = lambda x : x [ 0 ] ) ] else : return [ ]

Give the ref annotation after a time . If an annotation overlaps with ktime that annotation will be returned .


def get_ref_annotation_data_before_time(self, id_tier, time):
        """Give the ref annotation before a time. If an annotation overlaps
        with ``time`` that annotation will be returned.

        :param str id_tier: Name of the tier.
        :param int time: Time to get the annotation before.
        :returns: Annotation before that time in a list
        :raises KeyError: If the tier is non existent.
        """
        befores = self.get_ref_annotation_data_between_times(id_tier, 0, time)
        if befores:
            return [max(befores, key=lambda x: x[0])]
        else:
            return []

def get_ref_annotation_data_before_time ( self , id_tier , time ) : befores = self . get_ref_annotation_data_between_times ( id_tier , 0 , time ) if befores : return [ max ( befores , key = lambda x : x [ 0 ] ) ] else : return [ ]

Give the ref annotation before a time . If an annotation overlaps with time that annotation will be returned .


def get_tier_ids_for_linguistic_type(self, ling_type, parent=None):
        """Give a list of all tiers matching a linguistic type.

        :param str ling_type: Name of the linguistic type.
        :param str parent: Only match tiers from this parent, when ``None``
                           this option will be ignored.
        :returns: List of tiernames.
        :raises KeyError: If a tier or linguistic type is non existent.
        """
        return [t for t in self.tiers if
                self.tiers[t][2]['LINGUISTIC_TYPE_REF'] == ling_type and
                (parent is None or self.tiers[t][2]['PARENT_REF'] == parent)]

def get_tier_ids_for_linguistic_type ( self , ling_type , parent = None ) : return [ t for t in self . tiers if self . tiers [ t ] [ 2 ] [ 'LINGUISTIC_TYPE_REF' ] == ling_type and ( parent is None or self . tiers [ t ] [ 2 ] [ 'PARENT_REF' ] == parent ) ]

Give a list of all tiers matching a linguistic type .


def merge_tiers(self, tiers, tiernew=None, gapt=0, sep='_', safe=False):
        """Merge tiers into a new tier and when the gap is lower then the
        threshhold glue the annotations together.

        :param list tiers: List of tier names.
        :param str tiernew: Name for the new tier, if ``None`` the name will be
                            generated.
        :param int gapt: Threshhold for the gaps, if the this is set to 10 it
                         means that all gaps below 10 are ignored.
        :param str sep: Separator for the merged annotations.
        :param bool safe: Ignore zero length annotations(when working with
            possible malformed data).
        :returns: Name of the created tier.
        :raises KeyError: If a tier is non existent.
        """
        if tiernew is None:
            tiernew = u'{}_merged'.format('_'.join(tiers))
        self.add_tier(tiernew)
        aa = [(sys.maxsize, sys.maxsize, None)] + sorted((
            a for t in tiers for a in self.get_annotation_data_for_tier(t)),
            reverse=True)
        l = None
        while aa:
            begin, end, value = aa.pop()
            if l is None:
                l = [begin, end, [value]]
            elif begin - l[1] >= gapt:
                if not safe or l[1] > l[0]:
                    self.add_annotation(tiernew, l[0], l[1], sep.join(l[2]))
                l = [begin, end, [value]]
            else:
                if end > l[1]:
                    l[1] = end
                l[2].append(value)
        return tiernew

def merge_tiers ( self , tiers , tiernew = None , gapt = 0 , sep = '_' , safe = False ) : if tiernew is None : tiernew = u'{}_merged' . format ( '_' . join ( tiers ) ) self . add_tier ( tiernew ) aa = [ ( sys . maxsize , sys . maxsize , None ) ] + sorted ( ( a for t in tiers for a in self . get_annotation_data_for_tier ( t ) ) , reverse = True ) l = None while aa : begin , end , value = aa . pop ( ) if l is None : l = [ begin , end , [ value ] ] elif begin - l [ 1 ] >= gapt : if not safe or l [ 1 ] > l [ 0 ] : self . add_annotation ( tiernew , l [ 0 ] , l [ 1 ] , sep . join ( l [ 2 ] ) ) l = [ begin , end , [ value ] ] else : if end > l [ 1 ] : l [ 1 ] = end l [ 2 ] . append ( value ) return tiernew

Merge tiers into a new tier and when the gap is lower then the threshhold glue the annotations together .


def remove_all_annotations_from_tier(self, id_tier, clean=True):
        """remove all annotations from a tier

        :param str id_tier: Name of the tier.
        :raises KeyError: If the tier is non existent.
        """
        for aid in self.tiers[id_tier][0]:
            del(self.annotations[aid])
        for aid in self.tiers[id_tier][1]:
            del(self.annotations[aid])

        self.tiers[id_tier][0].clear()
        self.tiers[id_tier][1].clear()
        if clean:
            self.clean_time_slots()

def remove_all_annotations_from_tier ( self , id_tier , clean = True ) : for aid in self . tiers [ id_tier ] [ 0 ] : del ( self . annotations [ aid ] ) for aid in self . tiers [ id_tier ] [ 1 ] : del ( self . annotations [ aid ] ) self . tiers [ id_tier ] [ 0 ] . clear ( ) self . tiers [ id_tier ] [ 1 ] . clear ( ) if clean : self . clean_time_slots ( )


def remove_cv_description(self, cv_id, lang_ref):
        """Remove a controlled vocabulary description.

        :param str cv_id: Name of the controlled vocabulary.
        :paarm str cve_id: Name of the entry.
        :throws KeyError: If there is no controlled vocabulary with that name.
        """
        for i, (l, d) in reversed(enumerate(
                self.controlled_vocabularies[cv_id][1])):
            if l == lang_ref:
                del(self.controlled_vocabularies[cv_id][1][i])

def remove_cv_description ( self , cv_id , lang_ref ) : for i , ( l , d ) in reversed ( enumerate ( self . controlled_vocabularies [ cv_id ] [ 1 ] ) ) : if l == lang_ref : del ( self . controlled_vocabularies [ cv_id ] [ 1 ] [ i ] )

Remove a controlled vocabulary description .


def remove_license(self, name=None, url=None):
        """Remove all licenses matching both key and value.

        :param str name: Name of the license.
        :param str url: URL of the license.
        """
        for k, v in self.licenses[:]:
            if (name is None or name == k) and (url is None or url == v):
                del(self.licenses[self.licenses.index((k, v))])

def remove_license ( self , name = None , url = None ) : for k , v in self . licenses [ : ] : if ( name is None or name == k ) and ( url is None or url == v ) : del ( self . licenses [ self . licenses . index ( ( k , v ) ) ] )

Remove all licenses matching both key and value .


def remove_linked_files(self, file_path=None, relpath=None, mimetype=None,
                            time_origin=None, ex_from=None):
        """Remove all linked files that match all the criteria, criterias that
        are ``None`` are ignored.

        :param str file_path: Path of the file.
        :param str relpath: Relative filepath.
        :param str mimetype: Mimetype of the file.
        :param int time_origin: Time origin.
        :param str ex_from: Extracted from.
        """
        for attrib in self.media_descriptors[:]:
            if file_path is not None and attrib['MEDIA_URL'] != file_path:
                continue
            if relpath is not None and attrib['RELATIVE_MEDIA_URL'] != relpath:
                continue
            if mimetype is not None and attrib['MIME_TYPE'] != mimetype:
                continue
            if time_origin is not None and\
                    attrib['TIME_ORIGIN'] != time_origin:
                continue
            if ex_from is not None and attrib['EXTRACTED_FROM'] != ex_from:
                continue
            del(self.media_descriptors[self.media_descriptors.index(attrib)])

def remove_linked_files ( self , file_path = None , relpath = None , mimetype = None , time_origin = None , ex_from = None ) : for attrib in self . media_descriptors [ : ] : if file_path is not None and attrib [ 'MEDIA_URL' ] != file_path : continue if relpath is not None and attrib [ 'RELATIVE_MEDIA_URL' ] != relpath : continue if mimetype is not None and attrib [ 'MIME_TYPE' ] != mimetype : continue if time_origin is not None and attrib [ 'TIME_ORIGIN' ] != time_origin : continue if ex_from is not None and attrib [ 'EXTRACTED_FROM' ] != ex_from : continue del ( self . media_descriptors [ self . media_descriptors . index ( attrib ) ] )

Remove all linked files that match all the criteria criterias that are None are ignored .


def remove_property(self, key=None, value=None):
        """Remove all properties matching both key and value.

        :param str key: Key of the property.
        :param str value: Value of the property.
        """
        for k, v in self.properties[:]:
            if (key is None or key == k) and (value is None or value == v):
                del(self.properties[self.properties.index((k, v))])

def remove_property ( self , key = None , value = None ) : for k , v in self . properties [ : ] : if ( key is None or key == k ) and ( value is None or value == v ) : del ( self . properties [ self . properties . index ( ( k , v ) ) ] )

Remove all properties matching both key and value .


def remove_ref_annotation(self, id_tier, time):
        """Remove a reference annotation.

        :param str id_tier: Name of tier.
        :param int time: Time of the referenced annotation
        :raises KeyError: If the tier is non existent.
        :returns: Number of removed annotations.
        """
        removed = 0
        bucket = []
        for aid, (ref, value, _, _) in self.tiers[id_tier][1].items():
            begin, end, rvalue, _ = self.tiers[self.annotations[ref]][0][ref]
            begin = self.timeslots[begin]
            end = self.timeslots[end]
            if begin <= time and end >= time:
                removed += 1
                bucket.append(aid)
        for aid in bucket:
            del(self.tiers[id_tier][1][aid])
        return removed

def remove_ref_annotation ( self , id_tier , time ) : removed = 0 bucket = [ ] for aid , ( ref , value , _ , _ ) in self . tiers [ id_tier ] [ 1 ] . items ( ) : begin , end , rvalue , _ = self . tiers [ self . annotations [ ref ] ] [ 0 ] [ ref ] begin = self . timeslots [ begin ] end = self . timeslots [ end ] if begin <= time and end >= time : removed += 1 bucket . append ( aid ) for aid in bucket : del ( self . tiers [ id_tier ] [ 1 ] [ aid ] ) return removed


def remove_secondary_linked_files(self, file_path=None, relpath=None,
                                      mimetype=None, time_origin=None,
                                      assoc_with=None):
        """Remove all secondary linked files that match all the criteria,
        criterias that are ``None`` are ignored.

        :param str file_path: Path of the file.
        :param str relpath: Relative filepath.
        :param str mimetype: Mimetype of the file.
        :param int time_origin: Time origin.
        :param str ex_from: Extracted from.
        """
        for attrib in self.linked_file_descriptors[:]:
            if file_path is not None and attrib['LINK_URL'] != file_path:
                continue
            if relpath is not None and attrib['RELATIVE_LINK_URL'] != relpath:
                continue
            if mimetype is not None and attrib['MIME_TYPE'] != mimetype:
                continue
            if time_origin is not None and\
                    attrib['TIME_ORIGIN'] != time_origin:
                continue
            if assoc_with is not None and\
                    attrib['ASSOCIATED_WITH'] != assoc_with:
                continue
            del(self.linked_file_descriptors[
                self.linked_file_descriptors.index(attrib)])

def remove_secondary_linked_files ( self , file_path = None , relpath = None , mimetype = None , time_origin = None , assoc_with = None ) : for attrib in self . linked_file_descriptors [ : ] : if file_path is not None and attrib [ 'LINK_URL' ] != file_path : continue if relpath is not None and attrib [ 'RELATIVE_LINK_URL' ] != relpath : continue if mimetype is not None and attrib [ 'MIME_TYPE' ] != mimetype : continue if time_origin is not None and attrib [ 'TIME_ORIGIN' ] != time_origin : continue if assoc_with is not None and attrib [ 'ASSOCIATED_WITH' ] != assoc_with : continue del ( self . linked_file_descriptors [ self . linked_file_descriptors . index ( attrib ) ] )

Remove all secondary linked files that match all the criteria criterias that are None are ignored .


def remove_tier(self, id_tier, clean=True):
        """Remove a tier.

        :param str id_tier: Name of the tier.
        :param bool clean: Flag to also clean the timeslots.
        :raises KeyError: If tier is non existent.
        """
        del(self.tiers[id_tier])
        if clean:
            self.clean_time_slots()

def remove_tier ( self , id_tier , clean = True ) : del ( self . tiers [ id_tier ] ) if clean : self . clean_time_slots ( )


def remove_tiers(self, tiers):
        """Remove multiple tiers, note that this is a lot faster then removing
        them individually because of the delayed cleaning of timeslots.

        :param list tiers: Names of the tier to remove.
        :raises KeyError: If a tier is non existent.
        """
        for a in tiers:
            self.remove_tier(a, clean=False)
        self.clean_time_slots()

def remove_tiers ( self , tiers ) : for a in tiers : self . remove_tier ( a , clean = False ) self . clean_time_slots ( )

Remove multiple tiers note that this is a lot faster then removing them individually because of the delayed cleaning of timeslots .


def rename_tier(self, id_from, id_to):
        """Rename a tier. Note that this renames also the child tiers that have
        the tier as a parent.

        :param str id_from: Original name of the tier.
        :param str id_to: Target name of the tier.
        :throws KeyError: If the tier doesnt' exist.
        """
        childs = self.get_child_tiers_for(id_from)
        self.tiers[id_to] = self.tiers.pop(id_from)
        self.tiers[id_to][2]['TIER_ID'] = id_to
        for child in childs:
            self.tiers[child][2]['PARENT_REF'] = id_to

def rename_tier ( self , id_from , id_to ) : childs = self . get_child_tiers_for ( id_from ) self . tiers [ id_to ] = self . tiers . pop ( id_from ) self . tiers [ id_to ] [ 2 ] [ 'TIER_ID' ] = id_to for child in childs : self . tiers [ child ] [ 2 ] [ 'PARENT_REF' ] = id_to

Rename a tier . Note that this renames also the child tiers that have the tier as a parent .


def shift_annotations(self, time):
        """Shift all annotations in time. Annotations that are in the beginning
        and a left shift is applied can be squashed or discarded.

        :param int time: Time shift width, negative numbers make a left shift.
        :returns: Tuple of a list of squashed annotations and a list of removed
                  annotations in the format: ``(tiername, start, end, value)``.
        """
        total_re = []
        total_sq = []
        for name, tier in self.tiers.items():
            squashed = []
            for aid, (begin, end, value, _) in tier[0].items():
                if self.timeslots[end]+time <= 0:
                    squashed.append((name, aid))
                elif self.timeslots[begin]+time < 0:
                    total_sq.append((name, self.timeslots[begin],
                                     self.timeslots[end], value))
                    self.timeslots[begin] = 0
                else:
                    self.timeslots[begin] += time
                    self.timeslots[end] += time
            for name, aid in squashed:
                start, end, value, _ = self.tiers[name][0][aid]
                del(self.tiers[name][0][aid])
                del(self.annotations[aid])
                total_re.append(
                    (name, self.timeslots[start], self.timeslots[end], value))
        return total_sq, total_re

def shift_annotations ( self , time ) : total_re = [ ] total_sq = [ ] for name , tier in self . tiers . items ( ) : squashed = [ ] for aid , ( begin , end , value , _ ) in tier [ 0 ] . items ( ) : if self . timeslots [ end ] + time <= 0 : squashed . append ( ( name , aid ) ) elif self . timeslots [ begin ] + time < 0 : total_sq . append ( ( name , self . timeslots [ begin ] , self . timeslots [ end ] , value ) ) self . timeslots [ begin ] = 0 else : self . timeslots [ begin ] += time self . timeslots [ end ] += time for name , aid in squashed : start , end , value , _ = self . tiers [ name ] [ 0 ] [ aid ] del ( self . tiers [ name ] [ 0 ] [ aid ] ) del ( self . annotations [ aid ] ) total_re . append ( ( name , self . timeslots [ start ] , self . timeslots [ end ] , value ) ) return total_sq , total_re

Shift all annotations in time . Annotations that are in the beginning and a left shift is applied can be squashed or discarded .


def main():
    """Will be used to create the console script"""

    import optparse
    import sys
    import codecs
    import locale
    import six
    from .algorithm import get_display

    parser = optparse.OptionParser()

    parser.add_option('-e', '--encoding',
                      dest='encoding',
                      default='utf-8',
                      type='string',
                      help='Text encoding (default: utf-8)')

    parser.add_option('-u', '--upper-is-rtl',
                      dest='upper_is_rtl',
                      default=False,
                      action='store_true',
                      help="Treat upper case chars as strong 'R' "
                      'for debugging (default: False).')

    parser.add_option('-d', '--debug',
                      dest='debug',
                      default=False,
                      action='store_true',
                      help="Output to stderr steps taken with the algorithm")

    parser.add_option('-b', '--base-dir',
                      dest='base_dir',
                      default=None,
                      type='string',
                      help="Override base direction [L|R]")

    options, rest = parser.parse_args()

    if options.base_dir and options.base_dir not in 'LR':
        parser.error('option -b can be L or R')

    # allow unicode in sys.stdout.write
    if six.PY2:
        sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)

    if rest:
        lines = rest
    else:
        lines = sys.stdin

    for line in lines:
        display = get_display(line, options.encoding, options.upper_is_rtl,
                              options.base_dir, options.debug)
        # adjust the encoding as unicode, to match the output encoding
        if not isinstance(display, six.text_type):
            display = display.decode(options.encoding)

        six.print_(display, end='')


def debug_storage(storage, base_info=False, chars=True, runs=False):
    "Display debug information for the storage"

    import codecs
    import locale
    import sys

    if six.PY2:
        stderr = codecs.getwriter(locale.getpreferredencoding())(sys.stderr)
    else:
        stderr = sys.stderr

    caller = inspect.stack()[1][3]
    stderr.write('in %s\n' % caller)

    if base_info:
        stderr.write(u'  base level  : %d\n' % storage['base_level'])
        stderr.write(u'  base dir    : %s\n' % storage['base_dir'])

    if runs:
        stderr.write(u'  runs        : %s\n' % list(storage['runs']))

    if chars:
        output = u'  Chars       : '
        for _ch in storage['chars']:
            if _ch != '\n':
                output += _ch['ch']
            else:
                output += 'C'
        stderr.write(output + u'\n')

        output = u'  Res. levels : %s\n' % u''.join(
            [six.text_type(_ch['level']) for _ch in storage['chars']])
        stderr.write(output)

        _types = [_ch['type'].ljust(3) for _ch in storage['chars']]

        for i in range(3):
            if i:
                output = u'                %s\n'
            else:
                output = u'  Res. types  : %s\n'
            stderr.write(output % u''.join([_t[i] for _t in _types]))


def get_base_level(text, upper_is_rtl=False):
    """Get the paragraph base embedding level. Returns 0 for LTR,
    1 for RTL.

    `text` a unicode object.

    Set `upper_is_rtl` to True to treat upper case chars as strong 'R'
    for debugging (default: False).

    """

    base_level = None

    prev_surrogate = False
    # P2
    for _ch in text:
        # surrogate in case of ucs2
        if _IS_UCS2 and (_SURROGATE_MIN <= ord(_ch) <= _SURROGATE_MAX):
            prev_surrogate = _ch
            continue
        elif prev_surrogate:
            _ch = prev_surrogate + _ch
            prev_surrogate = False

        # treat upper as RTL ?
        if upper_is_rtl and _ch.isupper():
            base_level = 1
            break

        bidi_type = bidirectional(_ch)

        if bidi_type in ('AL', 'R'):
            base_level = 1
            break

        elif bidi_type == 'L':
            base_level = 0
            break

    # P3
    if base_level is None:
        base_level = 0

    return base_level

def get_base_level ( text , upper_is_rtl = False ) : base_level = None prev_surrogate = False for _ch in text : if _IS_UCS2 and ( _SURROGATE_MIN <= ord ( _ch ) <= _SURROGATE_MAX ) : prev_surrogate = _ch continue elif prev_surrogate : _ch = prev_surrogate + _ch prev_surrogate = False if upper_is_rtl and _ch . isupper ( ) : base_level = 1 break bidi_type = bidirectional ( _ch ) if bidi_type in ( 'AL' , 'R' ) : base_level = 1 break elif bidi_type == 'L' : base_level = 0 break if base_level is None : base_level = 0 return base_level

Get the paragraph base embedding level . Returns 0 for LTR 1 for RTL .


def get_embedding_levels(text, storage, upper_is_rtl=False, debug=False):
    """Get the paragraph base embedding level and direction,
    set the storage to the array of chars"""

    prev_surrogate = False
    base_level = storage['base_level']

    # preset the storage's chars
    for _ch in text:
        if _IS_UCS2 and (_SURROGATE_MIN <= ord(_ch) <= _SURROGATE_MAX):
            prev_surrogate = _ch
            continue
        elif prev_surrogate:
            _ch = prev_surrogate + _ch
            prev_surrogate = False

        if upper_is_rtl and _ch.isupper():
            bidi_type = 'R'
        else:
            bidi_type = bidirectional(_ch)

        storage['chars'].append({
            'ch': _ch,
            'level': base_level,
            'type': bidi_type,
            'orig': bidi_type
        })
    if debug:
        debug_storage(storage, base_info=True)

def get_embedding_levels ( text , storage , upper_is_rtl = False , debug = False ) : prev_surrogate = False base_level = storage [ 'base_level' ] for _ch in text : if _IS_UCS2 and ( _SURROGATE_MIN <= ord ( _ch ) <= _SURROGATE_MAX ) : prev_surrogate = _ch continue elif prev_surrogate : _ch = prev_surrogate + _ch prev_surrogate = False if upper_is_rtl and _ch . isupper ( ) : bidi_type = 'R' else : bidi_type = bidirectional ( _ch ) storage [ 'chars' ] . append ( { 'ch' : _ch , 'level' : base_level , 'type' : bidi_type , 'orig' : bidi_type } ) if debug : debug_storage ( storage , base_info = True )

Get the paragraph base embedding level and direction set the storage to the array of chars


def explicit_embed_and_overrides(storage, debug=False):
    """Apply X1 to X9 rules of the unicode algorithm.

    See http://unicode.org/reports/tr9/#Explicit_Levels_and_Directions

    """
    overflow_counter = almost_overflow_counter = 0
    directional_override = 'N'
    levels = deque()

    # X1
    embedding_level = storage['base_level']

    for _ch in storage['chars']:
        bidi_type = _ch['type']

        level_func, override = X2_X5_MAPPINGS.get(bidi_type, (None, None))

        if level_func:
            # So this is X2 to X5
            # if we've past EXPLICIT_LEVEL_LIMIT, note it and do nothing

            if overflow_counter != 0:
                overflow_counter += 1
                continue

            new_level = level_func(embedding_level)
            if new_level < EXPLICIT_LEVEL_LIMIT:
                levels.append((embedding_level, directional_override))
                embedding_level, directional_override = new_level, override

            elif embedding_level == EXPLICIT_LEVEL_LIMIT - 2:
                # The new level is invalid, but a valid level can still be
                # achieved if this level is 60 and we encounter an RLE or
                # RLO further on.  So record that we 'almost' overflowed.
                almost_overflow_counter += 1

            else:
                overflow_counter += 1
        else:
            # X6
            if bidi_type not in X6_IGNORED:
                _ch['level'] = embedding_level
                if directional_override != 'N':
                    _ch['type'] = directional_override

            # X7
            elif bidi_type == 'PDF':
                if overflow_counter:
                    overflow_counter -= 1
                elif almost_overflow_counter and \
                        embedding_level != EXPLICIT_LEVEL_LIMIT - 1:
                    almost_overflow_counter -= 1
                elif levels:
                    embedding_level, directional_override = levels.pop()

            # X8
            elif bidi_type == 'B':
                levels.clear()
                overflow_counter = almost_overflow_counter = 0
                embedding_level = _ch['level'] = storage['base_level']
                directional_override = 'N'

    # Removes the explicit embeds and overrides of types
    # RLE, LRE, RLO, LRO, PDF, and BN. Adjusts extended chars
    # next and prev as well

    # Applies X9. See http://unicode.org/reports/tr9/#X9
    storage['chars'] = [_ch for _ch in storage['chars']
                        if _ch['type'] not in X9_REMOVED]

    calc_level_runs(storage)

    if debug:
        debug_storage(storage, runs=True)

def explicit_embed_and_overrides ( storage , debug = False ) : overflow_counter = almost_overflow_counter = 0 directional_override = 'N' levels = deque ( ) embedding_level = storage [ 'base_level' ] for _ch in storage [ 'chars' ] : bidi_type = _ch [ 'type' ] level_func , override = X2_X5_MAPPINGS . get ( bidi_type , ( None , None ) ) if level_func : if overflow_counter != 0 : overflow_counter += 1 continue new_level = level_func ( embedding_level ) if new_level < EXPLICIT_LEVEL_LIMIT : levels . append ( ( embedding_level , directional_override ) ) embedding_level , directional_override = new_level , override elif embedding_level == EXPLICIT_LEVEL_LIMIT - 2 : almost_overflow_counter += 1 else : overflow_counter += 1 else : if bidi_type not in X6_IGNORED : _ch [ 'level' ] = embedding_level if directional_override != 'N' : _ch [ 'type' ] = directional_override elif bidi_type == 'PDF' : if overflow_counter : overflow_counter -= 1 elif almost_overflow_counter and embedding_level != EXPLICIT_LEVEL_LIMIT - 1 : almost_overflow_counter -= 1 elif levels : embedding_level , directional_override = levels . pop ( ) elif bidi_type == 'B' : levels . clear ( ) overflow_counter = almost_overflow_counter = 0 embedding_level = _ch [ 'level' ] = storage [ 'base_level' ] directional_override = 'N' storage [ 'chars' ] = [ _ch for _ch in storage [ 'chars' ] if _ch [ 'type' ] not in X9_REMOVED ] calc_level_runs ( storage ) if debug : debug_storage ( storage , runs = True )

Apply X1 to X9 rules of the unicode algorithm .


def calc_level_runs(storage):
    """Split the storage to run of char types at the same level.

    Applies X10. See http://unicode.org/reports/tr9/#X10
    """
    # run level depends on the higher of the two levels on either side of
    # the boundary If the higher level is odd, the type is R; otherwise,
    # it is L

    storage['runs'].clear()
    chars = storage['chars']

    # empty string ?
    if not chars:
        return

    def calc_level_run(b_l, b_r):
        return ['L', 'R'][max(b_l, b_r) % 2]

    first_char = chars[0]

    sor = calc_level_run(storage['base_level'], first_char['level'])
    eor = None

    run_start = run_length = 0

    prev_level, prev_type = first_char['level'], first_char['type']

    for _ch in chars:
        curr_level, curr_type = _ch['level'], _ch['type']

        if curr_level == prev_level:
            run_length += 1
        else:
            eor = calc_level_run(prev_level, curr_level)
            storage['runs'].append({'sor': sor, 'eor': eor, 'start': run_start,
                                    'type': prev_type, 'length': run_length})
            sor = eor
            run_start += run_length
            run_length = 1

        prev_level, prev_type = curr_level, curr_type

    # for the last char/runlevel
    eor = calc_level_run(curr_level, storage['base_level'])
    storage['runs'].append({'sor': sor, 'eor': eor, 'start': run_start,
                            'type': curr_type, 'length': run_length})

def calc_level_runs ( storage ) : storage [ 'runs' ] . clear ( ) chars = storage [ 'chars' ] if not chars : return def calc_level_run ( b_l , b_r ) : return [ 'L' , 'R' ] [ max ( b_l , b_r ) % 2 ] first_char = chars [ 0 ] sor = calc_level_run ( storage [ 'base_level' ] , first_char [ 'level' ] ) eor = None run_start = run_length = 0 prev_level , prev_type = first_char [ 'level' ] , first_char [ 'type' ] for _ch in chars : curr_level , curr_type = _ch [ 'level' ] , _ch [ 'type' ] if curr_level == prev_level : run_length += 1 else : eor = calc_level_run ( prev_level , curr_level ) storage [ 'runs' ] . append ( { 'sor' : sor , 'eor' : eor , 'start' : run_start , 'type' : prev_type , 'length' : run_length } ) sor = eor run_start += run_length run_length = 1 prev_level , prev_type = curr_level , curr_type eor = calc_level_run ( curr_level , storage [ 'base_level' ] ) storage [ 'runs' ] . append ( { 'sor' : sor , 'eor' : eor , 'start' : run_start , 'type' : curr_type , 'length' : run_length } )

Split the storage to run of char types at the same level .