def _create_link(self, act_node, name, instance):
        """Creates a link and checks if names are appropriate
        """

        act_node._links[name] = instance
        act_node._children[name] = instance

        full_name = instance.v_full_name
        if full_name not in self._root_instance._linked_by:
            self._root_instance._linked_by[full_name] = {}
        linking = self._root_instance._linked_by[full_name]
        if act_node.v_full_name not in linking:
            linking[act_node.v_full_name] = (act_node, set())
        linking[act_node.v_full_name][1].add(name)

        if name not in self._links_count:
            self._links_count[name] = 0
        self._links_count[name] = self._links_count[name] + 1

        self._logger.debug('Added link `%s` under `%s` pointing '
                           'to `%s`.' % (name, act_node.v_full_name,
                                         instance.v_full_name))
        return instance

def
_create_link
(
self
,
act_node
,
name
,
instance
)
:
act_node
.
_links
[
name
]
=
instance
act_node
.
_children
[
name
]
=
instance
full_name
=
instance
.
v_full_name
if
full_name
not
in
self
.
_root_instance
.
_linked_by
:
self
.
_root_instance
.
_linked_by
[
full_name
]
=
{
}
linking
=
self
.
_root_instance
.
_linked_by
[
full_name
]
if
act_node
.
v_full_name
not
in
linking
:
linking
[
act_node
.
v_full_name
]
=
(
act_node
,
set
(
)
)
linking
[
act_node
.
v_full_name
]
[
1
]
.
add
(
name
)
if
name
not
in
self
.
_links_count
:
self
.
_links_count
[
name
]
=
0
self
.
_links_count
[
name
]
=
self
.
_links_count
[
name
]
+
1
self
.
_logger
.
debug
(
'Added link `%s` under `%s` pointing '
'to `%s`.'
%
(
name
,
act_node
.
v_full_name
,
instance
.
v_full_name
)
)
return
instance


def _check_names(self, split_names, parent_node=None):
        """Checks if a list contains strings with invalid names.

        Returns a description of the name violations. If names are correct the empty
        string is returned.

        :param split_names: List of strings

        :param parent_node:

            The parental node from where to start (only applicable for node names)

        """

        faulty_names = ''

        if parent_node is not None and parent_node.v_is_root and split_names[0] == 'overview':
            faulty_names = '%s `overview` cannot be added directly under the root node ' \
                           'this is a reserved keyword,' % (faulty_names)

        for split_name in split_names:

            if len(split_name) == 0:
                faulty_names = '%s `%s` contains no characters, please use at least 1,' % (
                    faulty_names, split_name)

            elif split_name.startswith('_'):
                faulty_names = '%s `%s` starts with a leading underscore,' % (
                    faulty_names, split_name)

            elif re.match(CHECK_REGEXP, split_name) is None:
                faulty_names = '%s `%s` contains non-admissible characters ' \
                               '(use only [A-Za-z0-9_-]),' % \
                               (faulty_names, split_name)

            elif '$' in split_name:
                if split_name not in self._root_instance._wildcard_keys:
                    faulty_names = '%s `%s` contains `$` but has no associated ' \
                                   'wildcard function,' % (faulty_names, split_name)

            elif split_name in self._not_admissible_names:
                warnings.warn('`%s` is a method/attribute of the '
                              'trajectory/treenode/naminginterface, you may not be '
                              'able to access it via natural naming but only by using '
                              '`[]` square bracket notation. ' % split_name,
                              category=SyntaxWarning)

            elif split_name in self._python_keywords:
                warnings.warn('`%s` is a python keyword, you may not be '
                              'able to access it via natural naming but only by using '
                              '`[]` square bracket notation. ' % split_name,
                              category=SyntaxWarning)

        name = split_names[-1]
        if len(name) >= pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH:
            faulty_names = '%s `%s` is too long the name can only have %d characters but it has ' \
                           '%d,' % \
                           (faulty_names, name, len(name),
                            pypetconstants.HDF5_STRCOL_MAX_NAME_LENGTH)

        return faulty_names

Checks if a list contains strings with invalid names .


def _create_any_group(self, parent_node, name, type_name, instance=None, constructor=None,
                          args=None, kwargs=None):
        """Generically creates a new group inferring from the `type_name`."""

        if args is None:
            args = []

        if kwargs is None:
            kwargs = {}

        full_name = self._make_full_name(parent_node.v_full_name, name)

        if instance is None:
            if constructor is None:
                if type_name == RESULT_GROUP:
                    constructor = ResultGroup
                elif type_name == PARAMETER_GROUP:
                    constructor = ParameterGroup
                elif type_name == CONFIG_GROUP:
                    constructor = ConfigGroup
                elif type_name == DERIVED_PARAMETER_GROUP:
                    constructor = DerivedParameterGroup
                elif type_name == GROUP:
                    constructor = NNGroupNode
                else:
                    raise RuntimeError('You shall not pass!')
            instance = self._root_instance._construct_instance(constructor, full_name,
                                                               *args, **kwargs)
        else:
            instance._rename(full_name)
            # Check if someone tries to add a particular standard group to a branch where
            # it does not belong:
            if type_name == RESULT_GROUP:
                if type(instance) in (NNGroupNode,
                                   ParameterGroup,
                                   ConfigGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under results' %
                                    str(type(instance)))
            elif type_name == PARAMETER_GROUP:
                if type(instance) in (NNGroupNode,
                                   ResultGroup,
                                   ConfigGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under parameters' %
                                    str(type(instance)))
            elif type_name == CONFIG_GROUP:
                if type(instance) in (NNGroupNode,
                                   ParameterGroup,
                                   ResultGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under config' %
                                    str(type(instance)))
            elif type_name == DERIVED_PARAMETER_GROUP:
                if type(instance) in (NNGroupNode,
                                   ParameterGroup,
                                   ConfigGroup,
                                   ResultGroup):
                    raise TypeError('You cannot add a `%s` type of group under derived '
                                    'parameters' % str(type(instance)))
            elif type_name == GROUP:
                if type(instance) in (ResultGroup,
                                   ParameterGroup,
                                   ConfigGroup,
                                   DerivedParameterGroup):
                    raise TypeError('You cannot add a `%s` type of group under other data' %
                                    str(type(instance)))
            else:
                raise RuntimeError('You shall not pass!')

        self._set_details_tree_node(parent_node, name, instance)
        instance._nn_interface = self
        self._root_instance._all_groups[instance.v_full_name] = instance
        self._add_to_nodes_and_leaves(instance)
        parent_node._children[name] = instance
        parent_node._groups[name] = instance

        return instance

Generically creates a new group inferring from the type_name .


def _create_any_param_or_result(self, parent_node, name, type_name, instance, constructor,
                                    args, kwargs):
        """Generically creates a novel parameter or result instance inferring from the `type_name`.

        If the instance is already supplied it is NOT constructed new.

        :param parent_node:

            Parent trajectory node

        :param name:

            Name of the new result or parameter. Here the name no longer contains colons.

        :param type_name:

            Whether it is a parameter below parameters, config, derived parameters or whether
            it is a result.

        :param instance:

            The instance if it has been constructed somewhere else, otherwise None.

        :param constructor:

            A constructor used if instance needs to be constructed. If None the current standard
            constructor is chosen.

        :param args:

            Additional arguments passed to the constructor

        :param kwargs:

            Additional keyword arguments passed to the constructor

        :return: The new instance

        """
        root = self._root_instance
        full_name = self._make_full_name(parent_node.v_full_name, name)
        if instance is None:
            if constructor is None:
                if type_name == RESULT:
                    constructor = root._standard_result
                elif type_name in [PARAMETER, CONFIG, DERIVED_PARAMETER]:
                    constructor = root._standard_parameter
                else:
                    constructor = root._standard_leaf
            instance = root._construct_instance(constructor, full_name, *args, **kwargs)
        else:
            instance._rename(full_name)

        self._set_details_tree_node(parent_node, name, instance)

        where_dict = self._map_type_to_dict(type_name)
        full_name = instance._full_name

        if full_name in where_dict:
            raise AttributeError(full_name + ' is already part of trajectory,')

        if type_name != RESULT and full_name in root._changed_default_parameters:
            self._logger.info(
                'You have marked parameter %s for change before, so here you go!' %
                full_name)

            change_args, change_kwargs = root._changed_default_parameters.pop(full_name)
            instance.f_set(*change_args, **change_kwargs)

        where_dict[full_name] = instance
        self._add_to_nodes_and_leaves(instance)
        parent_node._children[name] = instance
        parent_node._leaves[name] = instance

        if full_name in self._root_instance._explored_parameters:
            instance._explored = True  # Mark this parameter as explored.
            self._root_instance._explored_parameters[full_name] = instance

        self._logger.debug('Added `%s` to trajectory.' % full_name)

        return instance

Generically creates a novel parameter or result instance inferring from the type_name .


def _set_details_tree_node(self, parent_node, name, instance):
        """Renames a given `instance` based on `parent_node` and `name`.

        Adds meta information like depth as well.

        """
        depth = parent_node._depth + 1
        if parent_node.v_is_root:
            branch = name  # We add below root
        else:
            branch = parent_node._branch
        if name in self._root_instance._run_information:
            run_branch = name
        else:
            run_branch = parent_node._run_branch

        instance._set_details(depth, branch, run_branch)

def _set_details_tree_node ( self , parent_node , name , instance ) : depth = parent_node . _depth + 1 if parent_node . v_is_root : branch = name else : branch = parent_node . _branch if name in self . _root_instance . _run_information : run_branch = name else : run_branch = parent_node . _run_branch instance . _set_details ( depth , branch , run_branch )

Renames a given instance based on parent_node and name .


def _iter_nodes(self, node, recursive=False, max_depth=float('inf'),
                    with_links=True, in_search=False, predicate=None):
        """Returns an iterator over nodes hanging below a given start node.

        :param node:

            Start node

        :param recursive:

            Whether recursively also iterate over the children of the start node's children

        :param max_depth:

            Maximum depth to search for

        :param in_search:

            if it is used during get search and if detailed info should be returned

        :param with_links:

            If links should be considered

        :param predicate:

            A predicate to filter nodes

        :return: Iterator

        """
        def _run_predicate(x, run_name_set):
            branch = x.v_run_branch
            return branch == 'trajectory' or branch in run_name_set

        if max_depth is None:
            max_depth = float('inf')

        if predicate is None:
            predicate = lambda x: True
        elif isinstance(predicate, (tuple, list)):
            # Create a predicate from a list of run names or run indices
            run_list = predicate
            run_name_set = set()
            for item in run_list:
                if item == -1:
                    run_name_set.add(self._root_instance.f_wildcard('$', -1))
                elif isinstance(item, int):
                    run_name_set.add(self._root_instance.f_idx_to_run(item))
                else:
                    run_name_set.add(item)
            predicate = lambda x: _run_predicate(x, run_name_set)

        if recursive:
            return NaturalNamingInterface._recursive_traversal_bfs(node,
                                            self._root_instance._linked_by,
                                            max_depth, with_links,
                                            in_search, predicate)
        else:
            iterator = (x for x in self._make_child_iterator(node, with_links) if
                        predicate(x[2]))
            if in_search:
                return iterator # Here we return tuples: (depth, name, object)
            else:
                return (x[2] for x in iterator)

def _iter_nodes ( self , node , recursive = False , max_depth = float ( 'inf' ) , with_links = True , in_search = False , predicate = None ) : def _run_predicate ( x , run_name_set ) : branch = x . v_run_branch return branch == 'trajectory' or branch in run_name_set if max_depth is None : max_depth = float ( 'inf' ) if predicate is None : predicate = lambda x : True elif isinstance ( predicate , ( tuple , list ) ) : run_list = predicate run_name_set = set ( ) for item in run_list : if item == - 1 : run_name_set . add ( self . _root_instance . f_wildcard ( '$' , - 1 ) ) elif isinstance ( item , int ) : run_name_set . add ( self . _root_instance . f_idx_to_run ( item ) ) else : run_name_set . add ( item ) predicate = lambda x : _run_predicate ( x , run_name_set ) if recursive : return NaturalNamingInterface . _recursive_traversal_bfs ( node , self . _root_instance . _linked_by , max_depth , with_links , in_search , predicate ) else : iterator = ( x for x in self . _make_child_iterator ( node , with_links ) if predicate ( x [ 2 ] ) ) if in_search : return iterator else : return ( x [ 2 ] for x in iterator )

Returns an iterator over nodes hanging below a given start node .


def _make_child_iterator(node, with_links, current_depth=0):
        """Returns an iterator over a node's children.

        In case of using a trajectory as a run (setting 'v_crun') some sub branches
        that do not belong to the run are blinded out.

        """
        cdp1 = current_depth + 1
        if with_links:
            iterator = ((cdp1, x[0], x[1]) for x in node._children.items())
        else:
            leaves = ((cdp1, x[0], x[1]) for x in node._leaves.items())
            groups = ((cdp1, y[0], y[1]) for y in node._groups.items())
            iterator = itools.chain(groups, leaves)
        return iterator

def _make_child_iterator ( node , with_links , current_depth = 0 ) : cdp1 = current_depth + 1 if with_links : iterator = ( ( cdp1 , x [ 0 ] , x [ 1 ] ) for x in node . _children . items ( ) ) else : leaves = ( ( cdp1 , x [ 0 ] , x [ 1 ] ) for x in node . _leaves . items ( ) ) groups = ( ( cdp1 , y [ 0 ] , y [ 1 ] ) for y in node . _groups . items ( ) ) iterator = itools . chain ( groups , leaves ) return iterator

Returns an iterator over a node s children .


def _recursive_traversal_bfs(node, linked_by=None,
                                 max_depth=float('inf'),
                                 with_links=True, in_search=False, predicate=None):
        """Iterator function traversing the tree below `node` in breadth first search manner.

        If `run_name` is given only sub branches of this run are considered and the rest is
        blinded out.

        """

        if predicate is None:
            predicate = lambda x: True

        iterator_queue = IteratorChain([(0, node.v_name, node)])
        #iterator_queue = iter([(0, node.v_name, node)])
        start = True
        visited_linked_nodes = set([])

        while True:
            try:
                depth, name, item = next(iterator_queue)
                full_name = item._full_name
                if start or predicate(item):
                    if full_name in visited_linked_nodes:
                        if in_search:
                            # We need to return the node again to check if a link to the node
                            # has to be found
                            yield depth, name, item
                    elif depth <= max_depth:
                        if start:
                            start = False
                        else:
                            if in_search:
                                yield depth, name, item
                            else:
                                yield item

                        if full_name in linked_by:
                            visited_linked_nodes.add(full_name)

                        if not item._is_leaf and depth < max_depth:
                            child_iterator = NaturalNamingInterface._make_child_iterator(item,
                                                                        with_links,
                                                                        current_depth=depth)
                            iterator_queue.add(child_iterator)
                            #iterator_queue = itools.chain(iterator_queue, child_iterator)
            except StopIteration:
                break

def _recursive_traversal_bfs ( node , linked_by = None , max_depth = float ( 'inf' ) , with_links = True , in_search = False , predicate = None ) : if predicate is None : predicate = lambda x : True iterator_queue = IteratorChain ( [ ( 0 , node . v_name , node ) ] ) start = True visited_linked_nodes = set ( [ ] ) while True : try : depth , name , item = next ( iterator_queue ) full_name = item . _full_name if start or predicate ( item ) : if full_name in visited_linked_nodes : if in_search : yield depth , name , item elif depth <= max_depth : if start : start = False else : if in_search : yield depth , name , item else : yield item if full_name in linked_by : visited_linked_nodes . add ( full_name ) if not item . _is_leaf and depth < max_depth : child_iterator = NaturalNamingInterface . _make_child_iterator ( item , with_links , current_depth = depth ) iterator_queue . add ( child_iterator ) except StopIteration : break

Iterator function traversing the tree below node in breadth first search manner .


def _very_fast_search(self, node, key, max_depth, with_links, crun):
        """Fast search for a node in the tree.

        The tree is not traversed but the reference dictionaries are searched.

        :param node:

            Parent node to start from

        :param key:

            Name of node to find

        :param max_depth:

            Maximum depth.

        :param with_links:

            If we work with links than we can only be sure to found the node in case we
            have a single match. Otherwise the other match might have been linked as well.

        :param crun:

            If given only nodes belonging to this particular run are searched and the rest
            is blinded out.

        :return: The found node and its depth

        :raises:

            TooManyGroupsError:

                If search cannot performed fast enough, an alternative search method is needed.

            NotUniqueNodeError:

                If several nodes match the key criterion

        """

        if key in self._links_count:
            return

        parent_full_name = node.v_full_name
        starting_depth = node.v_depth
        candidate_dict = self._get_candidate_dict(key, crun)

        # If there are to many potential candidates sequential search might be too slow
        if with_links:
            upper_bound = 1
        else:
            upper_bound = FAST_UPPER_BOUND
        if len(candidate_dict) > upper_bound:
            raise pex.TooManyGroupsError('Too many nodes')

        # Next check if the found candidates could be reached from the parent node
        result_node = None
        for goal_name in candidate_dict:

            # Check if we have found a matching node
            if goal_name.startswith(parent_full_name):
                candidate = candidate_dict[goal_name]
                if candidate.v_depth - starting_depth <= max_depth:
                    # In case of several solutions raise an error:
                    if result_node is not None:
                        raise pex.NotUniqueNodeError('Node `%s` has been found more than once, '
                                                     'full name of first occurrence is `%s` and of'
                                                     'second `%s`'
                                                     % (key, goal_name, result_node.v_full_name))
                    result_node = candidate

        if result_node is not None:
            return result_node, result_node.v_depth


def _search(self, node, key, max_depth=float('inf'), with_links=True, crun=None):
        """ Searches for an item in the tree below `node`

        :param node:

            The parent node below which the search is performed

        :param key:

            Name to search for. Can be the short name, the full name or parts of it

        :param max_depth:

            maximum search depth.

        :param with_links:

            If links should be considered

        :param crun:

            Used for very fast search if we know we operate in a single run branch

        :return: The found node and the depth it was found for

        """

        # If we find it directly there is no need for an exhaustive search
        if key in node._children and (with_links or key not in node._links):
            return node._children[key], 1

        # First the very fast search is tried that does not need tree traversal.
        try:
            result = self._very_fast_search(node, key, max_depth, with_links, crun)
            if result:
                return result
        except pex.TooManyGroupsError:
            pass
        except pex.NotUniqueNodeError:
            pass

        # Slowly traverse the entire tree
        nodes_iterator = self._iter_nodes(node, recursive=True,
                                          max_depth=max_depth, in_search=True,
                                          with_links=with_links)
        result_node = None
        result_depth = float('inf')
        for depth, name, child in nodes_iterator:

            if depth > result_depth:
                # We can break here because we enter a deeper stage of the tree and we
                # cannot find matching node of the same depth as the one we found
                break

            if key == name:
                # If result_node is not None means that we care about uniqueness and the search
                # has found more than a single solution.
                if result_node is not None:
                    raise pex.NotUniqueNodeError('Node `%s` has been found more than once within '
                                                 'the same depth %d. '
                                                 'Full name of first occurrence is `%s` and of '
                                                 'second `%s`'
                                                 % (key, child.v_depth, result_node.v_full_name,
                                                    child.v_full_name))

                result_node = child
                result_depth = depth

        return result_node, result_depth

Searches for an item in the tree below node


def _backwards_search(self, start_node, split_name, max_depth=float('inf'), shortcuts=True):
        """ Performs a backwards search from the terminal node back to the start node

        :param start_node:

            The node from where search starts, or here better way where backwards search should
            end.

        :param split_name:

            List of names

        :param max_depth:

            Maximum search depth where to look for

        :param shortcuts:

            If shortcuts are allowed

        """

        result_list = [] # Result list of all found items
        full_name_set = set() # Set containing full names of all found items to avoid finding items
        # twice due to links

        colon_name = '.'.join(split_name)
        key = split_name[-1]
        candidate_dict = self._get_candidate_dict(key, None, use_upper_bound=False)
        parent_full_name = start_node.v_full_name

        split_length = len(split_name)

        for candidate_name in candidate_dict:
            # Check if candidate startswith the parent's name
            candidate = candidate_dict[candidate_name]
            if key != candidate.v_name or candidate.v_full_name in full_name_set:
                # If this is not the case we do have link, that we need to skip
                continue

            if candidate_name.startswith(parent_full_name):
                if parent_full_name != '':
                    reduced_candidate_name = candidate_name[len(parent_full_name) + 1:]
                else:
                    reduced_candidate_name = candidate_name

                candidate_split_name = reduced_candidate_name.split('.')

                if len(candidate_split_name) > max_depth:
                    break

                if len(split_name) == 1 or reduced_candidate_name.endswith(colon_name):
                    result_list.append(candidate)
                    full_name_set.add(candidate.v_full_name)

                elif shortcuts:

                    candidate_set = set(candidate_split_name)
                    climbing = True
                    for name in split_name:
                        if name not in candidate_set:
                            climbing = False
                            break

                    if climbing:
                        count = 0
                        candidate_length = len(candidate_split_name)
                        for idx in range(candidate_length):

                            if idx + split_length - count > candidate_length:
                                break

                            if split_name[count] == candidate_split_name[idx]:
                                count += 1
                                if count == len(split_name):
                                    result_list.append(candidate)
                                    full_name_set.add(candidate.v_full_name)
                                    break

        return result_list

def _backwards_search ( self , start_node , split_name , max_depth = float ( 'inf' ) , shortcuts = True ) : result_list = [ ] full_name_set = set ( ) colon_name = '.' . join ( split_name ) key = split_name [ - 1 ] candidate_dict = self . _get_candidate_dict ( key , None , use_upper_bound = False ) parent_full_name = start_node . v_full_name split_length = len ( split_name ) for candidate_name in candidate_dict : candidate = candidate_dict [ candidate_name ] if key != candidate . v_name or candidate . v_full_name in full_name_set : continue if candidate_name . startswith ( parent_full_name ) : if parent_full_name != '' : reduced_candidate_name = candidate_name [ len ( parent_full_name ) + 1 : ] else : reduced_candidate_name = candidate_name candidate_split_name = reduced_candidate_name . split ( '.' ) if len ( candidate_split_name ) > max_depth : break if len ( split_name ) == 1 or reduced_candidate_name . endswith ( colon_name ) : result_list . append ( candidate ) full_name_set . add ( candidate . v_full_name ) elif shortcuts : candidate_set = set ( candidate_split_name ) climbing = True for name in split_name : if name not in candidate_set : climbing = False break if climbing : count = 0 candidate_length = len ( candidate_split_name ) for idx in range ( candidate_length ) : if idx + split_length - count > candidate_length : break if split_name [ count ] == candidate_split_name [ idx ] : count += 1 if count == len ( split_name ) : result_list . append ( candidate ) full_name_set . add ( candidate . v_full_name ) break return result_list


def kids(self):
        """Alternative naming, you can use `node.kids.name` instead of `node.name`
        for easier tab completion."""
        if self._kids is None:
            self._kids = NNTreeNodeKids(self)
        return self._kids

def kids ( self ) : if self . _kids is None : self . _kids = NNTreeNodeKids ( self ) return self . _kids

Alternative naming you can use node . kids . name instead of node . name for easier tab completion .


def _add_group_from_storage(self, args, kwargs):
        """Can be called from storage service to create a new group to bypass name checking"""
        return self._nn_interface._add_generic(self,
                                                type_name=GROUP,
                                                group_type_name=GROUP,
                                                args=args,
                                                kwargs=kwargs,
                                                add_prefix=False,
                                                check_naming=False)

def _add_group_from_storage ( self , args , kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = GROUP , group_type_name = GROUP , args = args , kwargs = kwargs , add_prefix = False , check_naming = False )


def _add_leaf_from_storage(self, args, kwargs):
        """Can be called from storage service to create a new leaf to bypass name checking"""
        return self._nn_interface._add_generic(self,
                                                type_name=LEAF,
                                                group_type_name=GROUP,
                                                args=args, kwargs=kwargs,
                                                add_prefix=False,
                                                check_naming=False)

def _add_leaf_from_storage ( self , args , kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = LEAF , group_type_name = GROUP , args = args , kwargs = kwargs , add_prefix = False , check_naming = False )


def f_dir_data(self):
        """Returns a list of all children names"""
        if (self._nn_interface is not None and
                    self._nn_interface._root_instance is not None
                        and self.v_root.v_auto_load):
            try:
                if self.v_is_root:
                    self.f_load(recursive=True, max_depth=1,
                                load_data=pypetconstants.LOAD_SKELETON,
                                with_meta_data=False,
                                with_run_information=False)
                else:
                    self.f_load(recursive=True, max_depth=1, load_data=pypetconstants.LOAD_SKELETON)
            except Exception as exc:
                pass
        return list(self._children.keys())

def f_dir_data ( self ) : if ( self . _nn_interface is not None and self . _nn_interface . _root_instance is not None and self . v_root . v_auto_load ) : try : if self . v_is_root : self . f_load ( recursive = True , max_depth = 1 , load_data = pypetconstants . LOAD_SKELETON , with_meta_data = False , with_run_information = False ) else : self . f_load ( recursive = True , max_depth = 1 , load_data = pypetconstants . LOAD_SKELETON ) except Exception as exc : pass return list ( self . _children . keys ( ) )


def _debug(self):
        """Creates a dummy object containing the whole tree to make unfolding easier.

        This method is only useful for debugging purposes.
        If you use an IDE and want to unfold the trajectory tree, you always need to
        open the private attribute `_children`. Use to this function to create a new
        object that contains the tree structure in its attributes.

        Manipulating the returned object does not change the original tree!

        """

        class Bunch(object):
            """Dummy container class"""
            pass

        debug_tree = Bunch()

        if not self.v_annotations.f_is_empty():
            debug_tree.v_annotations = self.v_annotations
        if not self.v_comment == '':
            debug_tree.v_comment = self.v_comment

        for leaf_name in self._leaves:
            leaf = self._leaves[leaf_name]
            setattr(debug_tree, leaf_name, leaf)

        for link_name in self._links:
            linked_node = self._links[link_name]
            setattr(debug_tree, link_name, 'Link to `%s`' % linked_node.v_full_name)

        for group_name in self._groups:
            group = self._groups[group_name]
            setattr(debug_tree, group_name, group._debug())

        return debug_tree

def
_debug
(
self
)
:
class
Bunch
(
object
)
:
pass
debug_tree
=
Bunch
(
)
if
not
self
.
v_annotations
.
f_is_empty
(
)
:
debug_tree
.
v_annotations
=
self
.
v_annotations
if
not
self
.
v_comment
==
''
:
debug_tree
.
v_comment
=
self
.
v_comment
for
leaf_name
in
self
.
_leaves
:
leaf
=
self
.
_leaves
[
leaf_name
]
setattr
(
debug_tree
,
leaf_name
,
leaf
)
for
link_name
in
self
.
_links
:
linked_node
=
self
.
_links
[
link_name
]
setattr
(
debug_tree
,
link_name
,
'Link to `%s`'
%
linked_node
.
v_full_name
)
for
group_name
in
self
.
_groups
:
group
=
self
.
_groups
[
group_name
]
setattr
(
debug_tree
,
group_name
,
group
.
_debug
(
)
)
return
debug_tree

Creates a dummy object containing the whole tree to make unfolding easier .


def f_get_parent(self):
        """Returns the parent of the node.

        Raises a TypeError if current node is root.

        """
        if self.v_is_root:
            raise TypeError('Root does not have a parent')
        elif self.v_location == '':
            return self.v_root
        else:
            return self.v_root.f_get(self.v_location, fast_access=False, shortcuts=False)

def
f_get_parent
(
self
)
:
if
self
.
v_is_root
:
raise
TypeError
(
'Root does not have a parent'
)
elif
self
.
v_location
==
''
:
return
self
.
v_root
else
:
return
self
.
v_root
.
f_get
(
self
.
v_location
,
fast_access
=
False
,
shortcuts
=
False
)


def f_add_group(self, *args, **kwargs):
        """Adds an empty generic group under the current node.

        You can add to a generic group anywhere you want. So you are free to build
        your parameter tree with any structure. You do not necessarily have to follow the
        four subtrees `config`, `parameters`, `derived_parameters`, `results`.

        If you are operating within these subtrees this simply calls the corresponding adding
        function.

        Be aware that if you are within a single run and you add items not below a group
        `run_XXXXXXXX` that you have to manually
        save the items. Otherwise they will be lost after the single run is completed.

        """

        return self._nn_interface._add_generic(self, type_name=GROUP,
                                               group_type_name=GROUP,
                                               args=args, kwargs=kwargs, add_prefix=False)

def f_add_group ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = GROUP , group_type_name = GROUP , args = args , kwargs = kwargs , add_prefix = False )

Adds an empty generic group under the current node .


def f_add_link(self, name_or_item, full_name_or_item=None):
        """Adds a link to an existing node.

        Can be called as ``node.f_add_link(other_node)`` this will add a link the `other_node`
        with the link name as the name of the node.

        Or can be called as ``node.f_add_link(name, other_node)`` to add a link to the
        `other_node` and the given `name` of the link.

        In contrast to addition of groups and leaves,  colon separated names
        are **not** allowed, i.e. ``node.f_add_link('mygroup.mylink', other_node)``
        does not work.

        """
        if isinstance(name_or_item, str):
            name = name_or_item
            if isinstance(full_name_or_item, str):
                instance = self.v_root.f_get(full_name_or_item)
            else:
                instance =  full_name_or_item
        else:
            instance = name_or_item
            name = instance.v_name

        return self._nn_interface._add_generic(self, type_name=LINK,
                                               group_type_name=GROUP, args=(name, instance),
                                               kwargs={},
                                               add_prefix=False)

def f_add_link ( self , name_or_item , full_name_or_item = None ) : if isinstance ( name_or_item , str ) : name = name_or_item if isinstance ( full_name_or_item , str ) : instance = self . v_root . f_get ( full_name_or_item ) else : instance = full_name_or_item else : instance = name_or_item name = instance . v_name return self . _nn_interface . _add_generic ( self , type_name = LINK , group_type_name = GROUP , args = ( name , instance ) , kwargs = { } , add_prefix = False )


def f_remove_link(self, name):
        """ Removes a link from from the current group node with a given name.

        Does not delete the link from the hard drive. If you want to do this,
        checkout :func:`~pypet.trajectory.Trajectory.f_delete_links`

        """
        if name not in self._links:
            raise ValueError('No link with name `%s` found under `%s`.' % (name, self._full_name))

        self._nn_interface._remove_link(self, name)

Removes a link from from the current group node with a given name .


def f_add_leaf(self, *args, **kwargs):
        """Adds an empty generic leaf under the current node.

        You can add to a generic leaves anywhere you want. So you are free to build
        your trajectory tree with any structure. You do not necessarily have to follow the
        four subtrees `config`, `parameters`, `derived_parameters`, `results`.

        If you are operating within these subtrees this simply calls the corresponding adding
        function.

        Be aware that if you are within a single run and you add items not below a group
        `run_XXXXXXXX` that you have to manually
        save the items. Otherwise they will be lost after the single run is completed.

        """

        return self._nn_interface._add_generic(self, type_name=LEAF,
                                               group_type_name=GROUP,
                                               args=args, kwargs=kwargs,
                                               add_prefix=False)

def f_add_leaf ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = LEAF , group_type_name = GROUP , args = args , kwargs = kwargs , add_prefix = False )

Adds an empty generic leaf under the current node .


def f_remove(self, recursive=True, predicate=None):
        """Recursively removes the group and all it's children.

        :param recursive:

            If removal should be applied recursively. If not, node can only be removed
            if it has no children.

        :param predicate:

            In case of recursive removal, you can selectively remove nodes in the tree.
            Predicate which can evaluate for each node to ``True`` in order to remove the node or
            ``False`` if the node should be kept. Leave ``None`` if you want to remove all nodes.

        """
        parent = self.f_get_parent()
        parent.f_remove_child(self.v_name, recursive=recursive, predicate=predicate)

def f_remove ( self , recursive = True , predicate = None ) : parent = self . f_get_parent ( ) parent . f_remove_child ( self . v_name , recursive = recursive , predicate = predicate )

Recursively removes the group and all it s children .


def f_remove_child(self, name, recursive=False, predicate=None):
        """Removes a child of the group.

        Note that groups and leaves are only removed from the current trajectory in RAM.
        If the trajectory is stored to disk, this data is not affected. Thus, removing children
        can be only be used to free RAM memory!

        If you want to free memory on disk via your storage service,
        use :func:`~pypet.trajectory.Trajectory.f_delete_items` of your trajectory.

        :param name:

            Name of child, naming by grouping is NOT allowed ('groupA.groupB.childC'),
            child must be direct successor of current node.

        :param recursive:

            Must be true if child is a group that has children. Will remove
            the whole subtree in this case. Otherwise a Type Error is thrown.

        :param predicate:

            Predicate which can evaluate for each node to ``True`` in order to remove the node or
            ``False`` if the node should be kept. Leave ``None`` if you want to remove all nodes.

        :raises:

            TypeError if recursive is false but there are children below the node.

            ValueError if child does not exist.

        """
        if name not in self._children:
            raise ValueError('Your group `%s` does not contain the child `%s`.' %
                             (self.v_full_name, name))
        else:
            child = self._children[name]
            if (name not in self._links and
                    not child.v_is_leaf and
                    child.f_has_children() and
                    not recursive):
                raise TypeError('Cannot remove child. It is a group with children. Use'
                                ' f_remove with ``recursive = True``')
            else:
                self._nn_interface._remove_subtree(self, name, predicate)


def f_contains(self, item, with_links=True, shortcuts=False, max_depth=None):
        """Checks if the node contains a specific parameter or result.

        It is checked if the item can be found via the
        :func:`~pypet.naturalnaming.NNGroupNode.f_get` method.

        :param item: Parameter/Result name or instance.

            If a parameter or result instance is supplied it is also checked if
            the provided item and the found item are exactly the same instance, i.e.
            `id(item)==id(found_item)`.

        :param with_links:

            If links are considered.

        :param shortcuts:

            Shortcuts is `False` the name you supply must
            be found in the tree WITHOUT hopping over nodes in between.
            If `shortcuts=False` and you supply a
            non colon separated (short) name, than the name must be found
            in the immediate children of your current node.
            Otherwise searching via shortcuts is allowed.

        :param max_depth:

            If shortcuts is `True` than the maximum search depth
            can be specified. `None` means no limit.

        :return: True or False

        """

        # Check if an instance or a name was supplied by the user
        try:
            search_string = item.v_full_name
            parent_full_name = self.v_full_name

            if not search_string.startswith(parent_full_name):
                return False

            if parent_full_name != '':
                search_string = search_string[len(parent_full_name) + 1:]
            else:
                search_string = search_string

            shortcuts = False # if we search for a particular item we do not allow shortcuts

        except AttributeError:
            search_string = item
            item = None

        if search_string == '':
            return False # To allow to search for nodes wit name = '', which are never part
            # of the trajectory

        try:
            result = self.f_get(search_string,
                                shortcuts=shortcuts, max_depth=max_depth, with_links=with_links)
        except AttributeError:
            return False

        if item is not None:
            return id(item) == id(result)
        else:
            return True

def f_contains ( self , item , with_links = True , shortcuts = False , max_depth = None ) : try : search_string = item . v_full_name parent_full_name = self . v_full_name if not search_string . startswith ( parent_full_name ) : return False if parent_full_name != '' : search_string = search_string [ len ( parent_full_name ) + 1 : ] else : search_string = search_string shortcuts = False except AttributeError : search_string = item item = None if search_string == '' : return False try : result = self . f_get ( search_string , shortcuts = shortcuts , max_depth = max_depth , with_links = with_links ) except AttributeError : return False if item is not None : return id ( item ) == id ( result ) else : return True

Checks if the node contains a specific parameter or result .


def f_get_default(self, name, default=None, fast_access=True, with_links=True,
              shortcuts=True, max_depth=None, auto_load=False):
        """ Similar to `f_get`, but returns the default value if `name` is not found in the
        trajectory.

        This function uses the `f_get` method and will return the default value
        in case `f_get` raises an AttributeError or a DataNotInStorageError.
        Other errors are not handled.

        In contrast to `f_get`, fast access is True by default.

        """
        try:
            return self.f_get(name, fast_access=fast_access,
                           shortcuts=shortcuts,
                           max_depth=max_depth,
                           auto_load=auto_load,
                           with_links=with_links)

        except (AttributeError, pex.DataNotInStorageError):
            return default

def f_get_default ( self , name , default = None , fast_access = True , with_links = True , shortcuts = True , max_depth = None , auto_load = False ) : try : return self . f_get ( name , fast_access = fast_access , shortcuts = shortcuts , max_depth = max_depth , auto_load = auto_load , with_links = with_links ) except ( AttributeError , pex . DataNotInStorageError ) : return default

Similar to f_get but returns the default value if name is not found in the trajectory .


def f_get_children(self, copy=True):
        """Returns a children dictionary.

        :param copy:

            Whether the group's original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!

        :returns: Dictionary of nodes

        """
        if copy:
            return self._children.copy()
        else:
            return self._children

def f_get_children ( self , copy = True ) : if copy : return self . _children . copy ( ) else : return self . _children


def f_get_groups(self, copy=True):
        """Returns a dictionary of groups hanging immediately below this group.

        :param copy:

            Whether the group's original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!

        :returns: Dictionary of nodes

        """
        if copy:
            return self._groups.copy()
        else:
            return self._groups

def f_get_groups ( self , copy = True ) : if copy : return self . _groups . copy ( ) else : return self . _groups

Returns a dictionary of groups hanging immediately below this group .


def f_get_leaves(self, copy=True):
        """Returns a dictionary of all leaves hanging immediately below this group.

        :param copy:

            Whether the group's original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!

        :returns: Dictionary of nodes

        """
        if copy:
            return self._leaves.copy()
        else:
            return self._leaves

def f_get_leaves ( self , copy = True ) : if copy : return self . _leaves . copy ( ) else : return self . _leaves

Returns a dictionary of all leaves hanging immediately below this group .


def f_get_links(self, copy=True):
        """Returns a link dictionary.

        :param copy:

            Whether the group's original dictionary or a shallow copy is returned.
            If you want the real dictionary please do not modify it at all!

        :returns: Dictionary of nodes

        """
        if copy:
            return self._links.copy()
        else:
            return self._links

def f_get_links ( self , copy = True ) : if copy : return self . _links . copy ( ) else : return self . _links


def f_store_child(self, name, recursive=False, store_data=pypetconstants.STORE_DATA,
                      max_depth=None):
        """Stores a child or recursively a subtree to disk.

        :param name:

            Name of child to store. If grouped ('groupA.groupB.childC') the path along the way
            to last node in the chain is stored. Shortcuts are NOT allowed!

        :param recursive:

            Whether recursively all children's children should be stored too.

        :param store_data:

            For how to choose 'store_data' see :ref:`more-on-storing`.

        :param max_depth:

            In case `recursive` is `True`, you can specify the maximum depth to store
            data relative from current node. Leave `None` if you don't want to limit
            the depth.

        :raises: ValueError if the child does not exist.

        """
        if not self.f_contains(name, shortcuts=False):
            raise ValueError('Your group `%s` does not (directly) contain the child `%s`. '
                             'Please not that shortcuts are not allowed for `f_store_child`.' %
                             (self.v_full_name, name))

        traj = self._nn_interface._root_instance
        storage_service = traj.v_storage_service

        storage_service.store(pypetconstants.TREE, self, name,
                              trajectory_name=traj.v_name,
                              recursive=recursive,
                              store_data=store_data,
                              max_depth=max_depth)

Stores a child or recursively a subtree to disk .


def f_store(self, recursive=True, store_data=pypetconstants.STORE_DATA,
                max_depth=None):
        """Stores a group node to disk

        :param recursive:

            Whether recursively all children should be stored too. Default is ``True``.

        :param store_data:

            For how to choose 'store_data' see :ref:`more-on-storing`.

        :param max_depth:

            In case `recursive` is `True`, you can specify the maximum depth to store
            data relative from current node. Leave `None` if you don't want to limit
            the depth.

        """
        traj = self._nn_interface._root_instance
        storage_service = traj.v_storage_service

        storage_service.store(pypetconstants.GROUP, self,
                              trajectory_name=traj.v_name,
                              recursive=recursive,
                              store_data=store_data,
                              max_depth=max_depth)

def f_store ( self , recursive = True , store_data = pypetconstants . STORE_DATA , max_depth = None ) : traj = self . _nn_interface . _root_instance storage_service = traj . v_storage_service storage_service . store ( pypetconstants . GROUP , self , trajectory_name = traj . v_name , recursive = recursive , store_data = store_data , max_depth = max_depth )


def f_load_child(self, name, recursive=False, load_data=pypetconstants.LOAD_DATA,
                     max_depth=None):
        """Loads a child or recursively a subtree from disk.

        :param name:

            Name of child to load. If grouped ('groupA.groupB.childC') the path along the way
            to last node in the chain is loaded. Shortcuts are NOT allowed!

        :param recursive:

            Whether recursively all nodes below the last child should be loaded, too.
            Note that links are never evaluated recursively. Only the linked node
            will be loaded if it does not exist in the tree, yet. Any nodes or links
            of this linked node are not loaded.

        :param load_data:

            Flag how to load the data.
            For how to choose 'load_data' see :ref:`more-on-loading`.

        :param max_depth:

            In case `recursive` is `True`, you can specify the maximum depth to load
            load data relative from current node. Leave `None` if you don't want to limit
            the depth.

        :returns:

            The loaded child, in case of grouping ('groupA.groupB.childC') the last
            node (here 'childC') is returned.

        """

        traj = self._nn_interface._root_instance
        storage_service = traj.v_storage_service

        storage_service.load(pypetconstants.TREE, self, name,
                             trajectory_name=traj.v_name,
                             load_data=load_data,
                             recursive=recursive,
                             max_depth=max_depth)

        return self.f_get(name, shortcuts=False)

def f_load_child ( self , name , recursive = False , load_data = pypetconstants . LOAD_DATA , max_depth = None ) : traj = self . _nn_interface . _root_instance storage_service = traj . v_storage_service storage_service . load ( pypetconstants . TREE , self , name , trajectory_name = traj . v_name , load_data = load_data , recursive = recursive , max_depth = max_depth ) return self . f_get ( name , shortcuts = False )

Loads a child or recursively a subtree from disk .


def f_load(self, recursive=True, load_data=pypetconstants.LOAD_DATA,
               max_depth=None):
        """Loads a group from disk.

        :param recursive:

            Default is ``True``.
            Whether recursively all nodes below the current node should be loaded, too.
            Note that links are never evaluated recursively. Only the linked node
            will be loaded if it does not exist in the tree, yet. Any nodes or links
            of this linked node are not loaded.

        :param load_data:

            Flag how to load the data.
            For how to choose 'load_data' see :ref:`more-on-loading`.

        :param max_depth:

            In case `recursive` is `True`, you can specify the maximum depth to load
            load data relative from current node.

        :returns:

            The node itself.

        """

        traj = self._nn_interface._root_instance
        storage_service = traj.v_storage_service

        storage_service.load(pypetconstants.GROUP, self,
                             trajectory_name=traj.v_name,
                             load_data=load_data,
                             recursive=recursive,
                             max_depth=max_depth)
        return self

def f_load ( self , recursive = True , load_data = pypetconstants . LOAD_DATA , max_depth = None ) : traj = self . _nn_interface . _root_instance storage_service = traj . v_storage_service storage_service . load ( pypetconstants . GROUP , self , trajectory_name = traj . v_name , load_data = load_data , recursive = recursive , max_depth = max_depth ) return self


def f_add_parameter_group(self, *args, **kwargs):
        """Adds an empty parameter group under the current node.

        Can be called with ``f_add_parameter_group('MyName', 'this is an informative comment')``
        or ``f_add_parameter_group(name='MyName', comment='This is an informative comment')``
        or with a given new group instance:
        ``f_add_parameter_group(ParameterGroup('MyName', comment='This is a comment'))``.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is the trajectory (root), the prefix `'parameters'`
        is added to the full name.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        created.

        """
        return self._nn_interface._add_generic(self, type_name=PARAMETER_GROUP,
                                               group_type_name=PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_parameter_group ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = PARAMETER_GROUP , group_type_name = PARAMETER_GROUP , args = args , kwargs = kwargs )

Adds an empty parameter group under the current node .


def f_add_parameter(self, *args, **kwargs):
        """ Adds a parameter under the current node.

        There are two ways to add a new parameter either by adding a parameter instance:

        >>> new_parameter = Parameter('group1.group2.myparam', data=42, comment='Example!')
        >>> traj.f_add_parameter(new_parameter)

        Or by passing the values directly to the function, with the name being the first
        (non-keyword!) argument:

        >>> traj.f_add_parameter('group1.group2.myparam', 42, comment='Example!')

        If you want to create a different parameter than the standard parameter, you can
        give the constructor as the first (non-keyword!) argument followed by the name
        (non-keyword!):

        >>> traj.f_add_parameter(PickleParameter,'group1.group2.myparam', data=42, comment='Example!')

        The full name of the current node is added as a prefix to the given parameter name.
        If the current node is the trajectory the prefix `'parameters'` is added to the name.

        Note, all non-keyword and keyword parameters apart from the optional constructor
        are passed on as is to the constructor.

        Moreover, you always should specify a default data value of a parameter,
        even if you want to explore it later.

        """
        return self._nn_interface._add_generic(self, type_name=PARAMETER,
                                               group_type_name=PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_parameter ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = PARAMETER , group_type_name = PARAMETER_GROUP , args = args , kwargs = kwargs )

Adds a parameter under the current node .


def f_add_result_group(self, *args, **kwargs):
        """Adds an empty result group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is a single run (root) adds the prefix `'results.runs.run_08%d%'` to the
        full name where `'08%d'` is replaced by the index of the current run.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """

        return self._nn_interface._add_generic(self, type_name=RESULT_GROUP,
                                               group_type_name=RESULT_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_result_group ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = RESULT_GROUP , group_type_name = RESULT_GROUP , args = args , kwargs = kwargs )

Adds an empty result group under the current node .


def f_add_result(self, *args, **kwargs):
        """Adds a result under the current node.

        There are two ways to add a new result either by adding a result instance:

        >>> new_result = Result('group1.group2.myresult', 1666, x=3, y=4, comment='Example!')
        >>> traj.f_add_result(new_result)

        Or by passing the values directly to the function, with the name being the first
        (non-keyword!) argument:

        >>> traj.f_add_result('group1.group2.myresult', 1666, x=3, y=3,comment='Example!')

        If you want to create a different result than the standard result, you can
        give the constructor as the first (non-keyword!) argument followed by the name
        (non-keyword!):

        >>> traj.f_add_result(PickleResult,'group1.group2.myresult', 1666, x=3, y=3, comment='Example!')

        Additional arguments (here `1666`) or keyword arguments (here `x=3, y=3`) are passed
        onto the constructor of the result.

        Adds the full name of the current node as prefix to the name of the result.
        If current node is a single run (root) adds the prefix `'results.runs.run_08%d%'` to the
        full name where `'08%d'` is replaced by the index of the current run.

        """
        return self._nn_interface._add_generic(self, type_name=RESULT,
                                               group_type_name=RESULT_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_result ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = RESULT , group_type_name = RESULT_GROUP , args = args , kwargs = kwargs )


def f_add_derived_parameter_group(self, *args, **kwargs):
        """Adds an empty derived parameter group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is a single run (root) adds the prefix `'derived_parameters.runs.run_08%d%'`
        to the full name where `'08%d'` is replaced by the index of the current run.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """

        return self._nn_interface._add_generic(self, type_name=DERIVED_PARAMETER_GROUP,
                                               group_type_name=DERIVED_PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_derived_parameter_group ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = DERIVED_PARAMETER_GROUP , group_type_name = DERIVED_PARAMETER_GROUP , args = args , kwargs = kwargs )

Adds an empty derived parameter group under the current node .


def f_add_derived_parameter(self, *args, **kwargs):
        """Adds a derived parameter under the current group.

        Similar to
        :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`

        Naming prefixes are added as in
        :func:`~pypet.naturalnaming.DerivedParameterGroup.f_add_derived_parameter_group`

        """
        return self._nn_interface._add_generic(self, type_name=DERIVED_PARAMETER,
                                               group_type_name=DERIVED_PARAMETER_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_derived_parameter ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = DERIVED_PARAMETER , group_type_name = DERIVED_PARAMETER_GROUP , args = args , kwargs = kwargs )

Adds a derived parameter under the current group .


def f_add_config_group(self, *args, **kwargs):
        """Adds an empty config group under the current node.

        Adds the full name of the current node as prefix to the name of the group.
        If current node is the trajectory (root), the prefix `'config'` is added to the full name.

        The `name` can also contain subgroups separated via colons, for example:
        `name=subgroup1.subgroup2.subgroup3`. These other parent groups will be automatically
        be created.

        """
        return self._nn_interface._add_generic(self, type_name=CONFIG_GROUP,
                                               group_type_name=CONFIG_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_config_group ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = CONFIG_GROUP , group_type_name = CONFIG_GROUP , args = args , kwargs = kwargs )

Adds an empty config group under the current node .


def f_add_config(self, *args, **kwargs):
        """Adds a config parameter under the current group.

        Similar to
        :func:`~pypet.naturalnaming.ParameterGroup.f_add_parameter`.

        If current group is the trajectory the prefix `'config'` is added to the name.

        """
        return self._nn_interface._add_generic(self, type_name=CONFIG,
                                               group_type_name=CONFIG_GROUP,
                                               args=args, kwargs=kwargs)

def f_add_config ( self , * args , ** kwargs ) : return self . _nn_interface . _add_generic ( self , type_name = CONFIG , group_type_name = CONFIG_GROUP , args = args , kwargs = kwargs )

Adds a config parameter under the current group .


def eval_one_max(traj, individual):
    """The fitness function"""
    traj.f_add_result('$set.$.individual', list(individual))
    fitness = sum(individual)
    traj.f_add_result('$set.$.fitness', fitness)
    traj.f_store()
    return (fitness,)

def eval_one_max ( traj , individual ) : traj . f_add_result ( '$set.$.individual' , list ( individual ) ) fitness = sum ( individual ) traj . f_add_result ( '$set.$.fitness' , fitness ) traj . f_store ( ) return ( fitness , )


def add_commit_variables(traj, commit):
    """Adds commit information to the trajectory."""

    git_time_value = time.strftime('%Y_%m_%d_%Hh%Mm%Ss', time.localtime(commit.committed_date))

    git_short_name = str(commit.hexsha[0:7])
    git_commit_name = 'commit_%s_' % git_short_name
    git_commit_name = 'git.' + git_commit_name + git_time_value

    if not traj.f_contains('config.'+git_commit_name, shortcuts=False):

        git_commit_name += '.'
        # Add the hexsha
        traj.f_add_config(git_commit_name+'hexsha', commit.hexsha,
                          comment='SHA-1 hash of commit')

        # Add the description string
        traj.f_add_config(git_commit_name+'name_rev', commit.name_rev,
                          comment='String describing the commits hex sha based on '
                                  'the closest Reference')

        # Add unix epoch
        traj.f_add_config(git_commit_name+'committed_date',
                          commit.committed_date, comment='Date of commit as unix epoch seconds')

        # Add commit message
        traj.f_add_config(git_commit_name+'message', str(commit.message),
                          comment='The commit message')

Adds commit information to the trajectory .


def make_git_commit(environment, git_repository, user_message, git_fail):
    """ Makes a commit and returns if a new commit was triggered and the SHA_1 code of the commit.

    If `git_fail` is `True` program fails instead of triggering a new commit given
    not committed changes. Then a `GitDiffError` is raised.

    """

    # Import GitPython, we do it here to allow also users not having GitPython installed
    # to use the normal environment

    # Open the repository
    repo = git.Repo(git_repository)
    index = repo.index

    traj = environment.v_trajectory

    # Create the commit message and append the trajectory name and comment
    if traj.v_comment:
        commentstr = ', Comment: `%s`' % traj.v_comment
    else:
        commentstr = ''

    if user_message:
        user_message += ' -- '

    message = '%sTrajectory: `%s`, Time: `%s`, %s' % \
              (user_message, traj.v_name, traj.v_time, commentstr)

    # Detect changes:
    diff = index.diff(None)

    if diff:
        if git_fail:
            # User requested fail instead of a new commit
            raise pex.GitDiffError('Found not committed changes!')
        # Make the commit
        repo.git.add('-u')
        commit = index.commit(message)
        new_commit = True

    else:
        # Take old commit
        commit = repo.commit(None)
        new_commit = False

    # Add the commit info to the trajectory
    add_commit_variables(traj, commit)

    return new_commit, commit.hexsha

def
make_git_commit
(
environment
,
git_repository
,
user_message
,
git_fail
)
:
repo
=
git
.
Repo
(
git_repository
)
index
=
repo
.
index
traj
=
environment
.
v_trajectory
if
traj
.
v_comment
:
commentstr
=
', Comment: `%s`'
%
traj
.
v_comment
else
:
commentstr
=
''
if
user_message
:
user_message
+=
' -- '
message
=
'%sTrajectory: `%s`, Time: `%s`, %s'
%
(
user_message
,
traj
.
v_name
,
traj
.
v_time
,
commentstr
)
diff
=
index
.
diff
(
)
if
diff
:
if
git_fail
:
raise
pex
.
GitDiffError
(
'Found not committed changes!'
)
repo
.
git
.
add
(
'-u'
)
commit
=
index
.
commit
(
message
)
new_commit
=
True
else
:
commit
=
repo
.
commit
(
)
new_commit
=
False
add_commit_variables
(
traj
,
commit
)
return
new_commit
,
commit
.
hexsha

Makes a commit and returns if a new commit was triggered and the SHA_1 code of the commit .


def flatten_dictionary(nested_dict, separator):
    """Flattens a nested dictionary.

    New keys are concatenations of nested keys with the `separator` in between.

    """
    flat_dict = {}
    for key, val in nested_dict.items():
        if isinstance(val, dict):
            new_flat_dict = flatten_dictionary(val, separator)
            for flat_key, inval in new_flat_dict.items():
                new_key = key + separator + flat_key
                flat_dict[new_key] = inval
        else:
            flat_dict[key] = val

    return flat_dict

def flatten_dictionary ( nested_dict , separator ) : flat_dict = { } for key , val in nested_dict . items ( ) : if isinstance ( val , dict ) : new_flat_dict = flatten_dictionary ( val , separator ) for flat_key , inval in new_flat_dict . items ( ) : new_key = key + separator + flat_key flat_dict [ new_key ] = inval else : flat_dict [ key ] = val return flat_dict


def nest_dictionary(flat_dict, separator):
    """ Nests a given flat dictionary.

    Nested keys are created by splitting given keys around the `separator`.

    """
    nested_dict = {}
    for key, val in flat_dict.items():
        split_key = key.split(separator)
        act_dict = nested_dict
        final_key = split_key.pop()
        for new_key in split_key:
            if not new_key in act_dict:
                act_dict[new_key] = {}

            act_dict = act_dict[new_key]

        act_dict[final_key] = val
    return nested_dict

def nest_dictionary ( flat_dict , separator ) : nested_dict = { } for key , val in flat_dict . items ( ) : split_key = key . split ( separator ) act_dict = nested_dict final_key = split_key . pop ( ) for new_key in split_key : if not new_key in act_dict : act_dict [ new_key ] = { } act_dict = act_dict [ new_key ] act_dict [ final_key ] = val return nested_dict


def progressbar(index, total, percentage_step=10, logger='print', log_level=logging.INFO,
                 reprint=True, time=True, length=20, fmt_string=None, reset=False):
    """Plots a progress bar to the given `logger` for large for loops.

    To be used inside a for-loop at the end of the loop:

    .. code-block:: python

        for irun in range(42):
            my_costly_job() # Your expensive function
            progressbar(index=irun, total=42, reprint=True) # shows a growing progressbar

    There is no initialisation of the progressbar necessary before the for-loop.
    The progressbar will be reset automatically if used in another for-loop.

    :param index: Current index of for-loop
    :param total: Total size of for-loop
    :param percentage_step: Steps with which the bar should be plotted
    :param logger:

        Logger to write to - with level INFO. If string 'print' is given, the print statement is
        used. Use ``None`` if you don't want to print or log the progressbar statement.

    :param log_level: Log level with which to log.
    :param reprint:

        If no new line should be plotted but carriage return (works only for printing)

    :param time: If the remaining time should be estimated and displayed
    :param length: Length of the bar in `=` signs.
    :param fmt_string:

        A string which contains exactly one `%s` in order to incorporate the progressbar.
        If such a string is given, ``fmt_string % progressbar`` is printed/logged.

    :param reset:

        If the progressbar should be restarted. If progressbar is called with a lower
        index than the one before, the progressbar is automatically restarted.

    :return:

        The progressbar string or `None` if the string has not been updated.

    """
    return _progressbar(index=index, total=total, percentage_step=percentage_step,
                        logger=logger, log_level=log_level, reprint=reprint,
                        time=time, length=length, fmt_string=fmt_string, reset=reset)

def progressbar ( index , total , percentage_step = 10 , logger = 'print' , log_level = logging . INFO , reprint = True , time = True , length = 20 , fmt_string = None , reset = False ) : return _progressbar ( index = index , total = total , percentage_step = percentage_step , logger = logger , log_level = log_level , reprint = reprint , time = time , length = length , fmt_string = fmt_string , reset = reset )

Plots a progress bar to the given logger for large for loops .


def _get_argspec(func):
    """Helper function to support both Python versions"""
    if inspect.isclass(func):
        func = func.__init__
    if not inspect.isfunction(func):
        # Init function not existing
        return [], False
    parameters = inspect.signature(func).parameters
    args = []
    uses_starstar = False
    for par in parameters.values():
        if (par.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD or
                    par.kind == inspect.Parameter.KEYWORD_ONLY):
            args.append(par.name)
        elif par.kind == inspect.Parameter.VAR_KEYWORD:
            uses_starstar = True
    return args, uses_starstar

def _get_argspec ( func ) : if inspect . isclass ( func ) : func = func . __init__ if not inspect . isfunction ( func ) : return [ ] , False parameters = inspect . signature ( func ) . parameters args = [ ] uses_starstar = False for par in parameters . values ( ) : if ( par . kind == inspect . Parameter . POSITIONAL_OR_KEYWORD or par . kind == inspect . Parameter . KEYWORD_ONLY ) : args . append ( par . name ) elif par . kind == inspect . Parameter . VAR_KEYWORD : uses_starstar = True return args , uses_starstar


def get_matching_kwargs(func, kwargs):
    """Takes a function and keyword arguments and returns the ones that can be passed."""
    args, uses_startstar = _get_argspec(func)
    if uses_startstar:
        return kwargs.copy()
    else:
        matching_kwargs = dict((k, kwargs[k]) for k in args if k in kwargs)
        return matching_kwargs

def get_matching_kwargs ( func , kwargs ) : args , uses_startstar = _get_argspec ( func ) if uses_startstar : return kwargs . copy ( ) else : matching_kwargs = dict ( ( k , kwargs [ k ] ) for k in args if k in kwargs ) return matching_kwargs

Takes a function and keyword arguments and returns the ones that can be passed .


def format_time(timestamp):
    """Formats timestamp to human readable format"""
    format_string = '%Y_%m_%d_%Hh%Mm%Ss'
    formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime(format_string)
    return formatted_time

def format_time ( timestamp ) : format_string = '%Y_%m_%d_%Hh%Mm%Ss' formatted_time = datetime . datetime . fromtimestamp ( timestamp ) . strftime ( format_string ) return formatted_time


def port_to_tcp(port=None):
    """Returns local tcp address for a given `port`, automatic port if `None`"""
    #address = 'tcp://' + socket.gethostbyname(socket.getfqdn())
    domain_name = socket.getfqdn()
    try:
        addr_list = socket.getaddrinfo(domain_name, None)
    except Exception:
        addr_list = socket.getaddrinfo('127.0.0.1', None)
    family, socktype, proto, canonname, sockaddr = addr_list[0]
    host = convert_ipv6(sockaddr[0])
    address =  'tcp://' + host
    if port is None:
        port = ()
    if not isinstance(port, int):
        # determine port automatically
        context = zmq.Context()
        try:
            socket_ = context.socket(zmq.REP)
            socket_.ipv6 = is_ipv6(address)
            port = socket_.bind_to_random_port(address, *port)
        except Exception:
            print('Could not connect to {} using {}'.format(address, addr_list))
            pypet_root_logger = logging.getLogger('pypet')
            pypet_root_logger.exception('Could not connect to {}'.format(address))
            raise
        socket_.close()
        context.term()
    return address + ':' + str(port)

def
port_to_tcp
(
port
=
)
:
domain_name
=
socket
.
getfqdn
(
)
try
:
addr_list
=
socket
.
getaddrinfo
(
domain_name
,
)
except
Exception
:
addr_list
=
socket
.
getaddrinfo
(
'127.0.0.1'
,
)
family
,
socktype
,
proto
,
canonname
,
sockaddr
=
addr_list
[
0
]
host
=
convert_ipv6
(
sockaddr
[
0
]
)
address
=
'tcp://'
+
host
if
port
is
:
port
=
(
)
if
not
isinstance
(
port
,
int
)
:
context
=
zmq
.
Context
(
)
try
:
socket_
=
context
.
socket
(
zmq
.
REP
)
socket_
.
ipv6
=
is_ipv6
(
address
)
port
=
socket_
.
bind_to_random_port
(
address
,
*
port
)
except
Exception
:
print
(
'Could not connect to {} using {}'
.
format
(
address
,
addr_list
)
)
pypet_root_logger
=
logging
.
getLogger
(
'pypet'
)
pypet_root_logger
.
exception
(
'Could not connect to {}'
.
format
(
address
)
)
raise
socket_
.
close
(
)
context
.
term
(
)
return
address
+
':'
+
str
(
port
)

Returns local tcp address for a given port automatic port if None


def racedirs(path):
    """Like os.makedirs but takes care about race conditions"""
    if os.path.isfile(path):
        raise IOError('Path `%s` is already a file not a directory')
    while True:
        try:
            if os.path.isdir(path):
                # only break if full path has been created or exists
                break
            os.makedirs(path)
        except EnvironmentError as exc:
            # Part of the directory path already exist
            if exc.errno != 17:
                # This error won't be any good
                raise

Like os . makedirs but takes care about race conditions


def _reset(self, index, total, percentage_step, length):
        """Resets to the progressbar to start a new one"""
        self._start_time = datetime.datetime.now()
        self._start_index = index
        self._current_index = index
        self._percentage_step = percentage_step
        self._total = float(total)
        self._total_minus_one = total - 1
        self._length = length
        self._norm_factor = total * percentage_step / 100.0
        self._current_interval = int((index + 1.0) / self._norm_factor)

def _reset ( self , index , total , percentage_step , length ) : self . _start_time = datetime . datetime . now ( ) self . _start_index = index self . _current_index = index self . _percentage_step = percentage_step self . _total = float ( total ) self . _total_minus_one = total - 1 self . _length = length self . _norm_factor = total * percentage_step / 100.0 self . _current_interval = int ( ( index + 1.0 ) / self . _norm_factor )


def _get_remaining(self, index):
        """Calculates remaining time as a string"""
        try:
            current_time = datetime.datetime.now()
            time_delta = current_time - self._start_time
            try:
                total_seconds = time_delta.total_seconds()
            except AttributeError:
                # for backwards-compatibility
                # Python 2.6 does not support `total_seconds`
                total_seconds = ((time_delta.microseconds +
                                    (time_delta.seconds +
                                     time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6)
            remaining_seconds = int((self._total - self._start_index - 1.0) *
                                    total_seconds / float(index - self._start_index) -
                                    total_seconds)
            remaining_delta = datetime.timedelta(seconds=remaining_seconds)
            remaining_str = ', remaining: ' + str(remaining_delta)
        except ZeroDivisionError:
            remaining_str = ''
        return remaining_str

def
_get_remaining
(
self
,
index
)
:
try
:
current_time
=
datetime
.
datetime
.
now
(
)
time_delta
=
current_time
self
.
_start_time
try
:
total_seconds
=
time_delta
.
total_seconds
(
)
except
AttributeError
:
total_seconds
=
(
(
time_delta
.
microseconds
+
(
time_delta
.
seconds
+
time_delta
.
days
*
24
*
3600
)
*
10
**
6
)
/
10.0
**
6
)
remaining_seconds
=
int
(
(
self
.
_total
self
.
_start_index
1.0
)
*
total_seconds
/
float
(
index
self
.
_start_index
)
total_seconds
)
remaining_delta
=
datetime
.
timedelta
(
seconds
=
remaining_seconds
)
remaining_str
=
', remaining: '
+
str
(
remaining_delta
)
except
ZeroDivisionError
:
remaining_str
=
''
return
remaining_str


def f_to_dict(self, copy=True):
        """Returns annotations as dictionary.

        :param copy: Whether to return a shallow copy or the real thing (aka _dict).

        """
        if copy:
            return self._dict.copy()
        else:
            return self._dict

def f_to_dict ( self , copy = True ) : if copy : return self . _dict . copy ( ) else : return self . _dict


def f_remove(self, key):
        """Removes `key` from annotations"""
        key = self._translate_key(key)
        try:
            del self._dict[key]
        except KeyError:
            raise AttributeError('Your annotations do not contain %s' % key)

def
f_remove
(
self
,
key
)
:
key
=
self
.
_translate_key
(
key
)
try
:
del
self
.
_dict
[
key
]
except
KeyError
:
raise
AttributeError
(
'Your annotations do not contain %s'
%
key
)


def f_ann_to_str(self):
        """Returns all annotations lexicographically sorted as a concatenated string."""
        resstr = ''
        for key in sorted(self._dict.keys()):
            resstr += '%s=%s; ' % (key, str(self._dict[key]))
        return resstr[:-2]

def
f_ann_to_str
(
self
)
:
resstr
=
''
for
key
in
sorted
(
self
.
_dict
.
keys
(
)
)
:
resstr
+=
'%s=%s; '
%
(
key
,
str
(
self
.
_dict
[
key
]
)
)
return
resstr
[
:
2
]

Returns all annotations lexicographically sorted as a concatenated string .


def make_ordinary_result(result, key, trajectory=None, reload=True):
    """Turns a given shared data item into a an ordinary one.

    :param result: Result container with shared data
    :param key: The name of the shared data
    :param trajectory:

        The trajectory, only needed if shared data has
        no access to the trajectory, yet.

    :param reload: If data should be reloaded after conversion

    :return: The result

    """
    shared_data = result.f_get(key)
    if trajectory is not None:
        shared_data.traj = trajectory
    shared_data._request_data('make_ordinary')
    result.f_remove(key)
    if reload:
        trajectory.f_load_item(result, load_data=pypetconstants.OVERWRITE_DATA)
    return result

def make_ordinary_result ( result , key , trajectory = None , reload = True ) : shared_data = result . f_get ( key ) if trajectory is not None : shared_data . traj = trajectory shared_data . _request_data ( 'make_ordinary' ) result . f_remove ( key ) if reload : trajectory . f_load_item ( result , load_data = pypetconstants . OVERWRITE_DATA ) return result

Turns a given shared data item into a an ordinary one .


def make_shared_result(result, key, trajectory, new_class=None):
    """Turns an ordinary data item into a shared one.

    Removes the old result from the trajectory and replaces it.
    Empties the given result.

    :param result: The result containing ordinary data
    :param key: Name of ordinary data item
    :param trajectory: Trajectory container
    :param new_class:

        Class of new shared data item.
        Leave `None` for automatic detection.

    :return: The `result`

    """
    data = result.f_get(key)
    if new_class is None:
        if isinstance(data, ObjectTable):
            new_class = SharedTable
        elif isinstance(data, pd.DataFrame):
            new_class = SharedPandasFrame
        elif isinstance(data, (tuple, list)):
            new_class = SharedArray
        elif isinstance(data, (np.ndarray, np.matrix)):
            new_class = SharedCArray
        else:
            raise RuntimeError('Your data `%s` is not understood.' % key)
    shared_data = new_class(result.f_translate_key(key), result, trajectory=trajectory)
    result[key] = shared_data
    shared_data._request_data('make_shared')
    return result

def
make_shared_result
(
result
,
key
,
trajectory
,
new_class
=
)
:
data
=
result
.
f_get
(
key
)
if
new_class
is
:
if
isinstance
(
data
,
ObjectTable
)
:
new_class
=
SharedTable
elif
isinstance
(
data
,
pd
.
DataFrame
)
:
new_class
=
SharedPandasFrame
elif
isinstance
(
data
,
(
tuple
,
list
)
)
:
new_class
=
SharedArray
elif
isinstance
(
data
,
(
np
.
ndarray
,
np
.
matrix
)
)
:
new_class
=
SharedCArray
else
:
raise
RuntimeError
(
'Your data `%s` is not understood.'
%
key
)
shared_data
=
new_class
(
result
.
f_translate_key
(
key
)
,
result
,
trajectory
=
trajectory
)
result
[
key
]
=
shared_data
shared_data
.
_request_data
(
'make_shared'
)
return
result

Turns an ordinary data item into a shared one .


def create_shared_data(self, **kwargs):
        """Creates shared data on disk with a StorageService on disk.

        Needs to be called before shared data can be used later on.

        Actual arguments of ``kwargs`` depend on the type of data to be
        created. For instance, creating an array one can use the keyword
        ``obj`` to pass a numpy array (``obj=np.zeros((10,20,30))``).
        Whereas for a PyTables table may need a description dictionary
        (``description={'column_1': pt.StringCol(2, pos=0),'column_2': pt.FloatCol( pos=1)}``)
        Refer to the PyTables documentation on how to create tables.

        """
        if 'flag' not in kwargs:
            kwargs['flag'] = self.FLAG
        if 'data' in kwargs:
            kwargs['obj'] = kwargs.pop('data')
        if 'trajectory' in kwargs:
            self.traj = kwargs.pop('trajectory')
        if 'traj' in kwargs:
            self.traj = kwargs.pop('traj')
        if 'name' in kwargs:
            self.name = kwargs.pop['name']
        if 'parent' in kwargs:
            self.parent = kwargs.pop('parent')
            if self.name is not None:
                self.parent[self.name] = self
        return self._request_data('create_shared_data', kwargs=kwargs)

def create_shared_data ( self , ** kwargs ) : if 'flag' not in kwargs : kwargs [ 'flag' ] = self . FLAG if 'data' in kwargs : kwargs [ 'obj' ] = kwargs . pop ( 'data' ) if 'trajectory' in kwargs : self . traj = kwargs . pop ( 'trajectory' ) if 'traj' in kwargs : self . traj = kwargs . pop ( 'traj' ) if 'name' in kwargs : self . name = kwargs . pop [ 'name' ] if 'parent' in kwargs : self . parent = kwargs . pop ( 'parent' ) if self . name is not None : self . parent [ self . name ] = self return self . _request_data ( 'create_shared_data' , kwargs = kwargs )

Creates shared data on disk with a StorageService on disk .


def _request_data(self, request, args=None, kwargs=None):
        """Interface with the underlying storage.

        Passes request to the StorageService that performs the appropriate action.
        For example, given a shared table ``t``.
        ``t.remove_row(4)`` is parsed into ``request='remove_row', args=(4,)`` and
        passed onto the storage service. In case of the HDF5StorageService,
        this is again translated back into ``hdf5_table_node.remove_row(4)``.

        """
        return self._storage_service.store(pypetconstants.ACCESS_DATA,
                                           self.parent.v_full_name,
                                           self.name,
                                           request, args, kwargs,
                                           trajectory_name=self.traj.v_name)

def _request_data ( self , request , args = None , kwargs = None ) : return self . _storage_service . store ( pypetconstants . ACCESS_DATA , self . parent . v_full_name , self . name , request , args , kwargs , trajectory_name = self . traj . v_name )


def get_data_node(self):
        """Returns the actula node of the underlying data.

        In case one uses HDF5 this will be the HDF5 leaf node.

        """
        if not self._storage_service.is_open:
            warnings.warn('You requesting the data item but your store is not open, '
                          'the item itself will be closed, too!',
                          category=RuntimeWarning)
        return self._request_data('__thenode__')

Returns the actula node of the underlying data .


def _supports(self, item):
        """Checks if outer data structure is supported."""
        result = super(SharedResult, self)._supports(item)
        result = result or type(item) in SharedResult.SUPPORTED_DATA
        return result

def _supports ( self , item ) : result = super ( SharedResult , self ) . _supports ( item ) result = result or type ( item ) in SharedResult . SUPPORTED_DATA return result

Checks if outer data structure is supported .


def create_shared_data(self, name=None, **kwargs):
        """Calls the corresponding function of the shared data item"""
        if name is None:
            item = self.f_get()
        else:
            item = self.f_get(name)
        return item.create_shared_data(**kwargs)

def create_shared_data ( self , name = None , ** kwargs ) : if name is None : item = self . f_get ( ) else : item = self . f_get ( name ) return item . create_shared_data ( ** kwargs )


def manipulate_multiproc_safe(traj):
    """ Target function that manipulates the trajectory.

    Stores the current name of the process into the trajectory and
    **overwrites** previous settings.

    :param traj:

        Trajectory container with multiprocessing safe storage service

    """

    # Manipulate the data in the trajectory
    traj.last_process_name = mp.current_process().name
    # Store the manipulated data
    traj.results.f_store(store_data=3)

def manipulate_multiproc_safe ( traj ) : traj . last_process_name = mp . current_process ( ) . name traj . results . f_store ( store_data = 3 )

Target function that manipulates the trajectory .


def _lock(self, name, client_id, request_id):
        """Hanldes locking of locks

        If a lock is already locked sends a WAIT command,
        else LOCKs it and sends GO.

        Complains if a given client re-locks a lock without releasing it before.

        """
        if name in self._locks:
            other_client_id, other_request_id = self._locks[name]
            if other_client_id == client_id:
                response = (self.LOCK_ERROR + self.DELIMITER +
                            'Re-request of lock `%s` (old request id `%s`) by `%s` '
                            '(request id `%s`)' % (name, client_id, other_request_id, request_id))
                self._logger.warning(response)
                return response
            else:
                return self.WAIT
        else:
            self._locks[name] = (client_id, request_id)
            return self.GO


def send_done(self):
        """Notifies the Server to shutdown"""
        self.start(test_connection=False)
        self._logger.debug('Sending shutdown signal')
        self._req_rep(ZMQServer.DONE)

def
send_done
(
self
)
:
self
.
start
(
test_connection
=
False
)
self
.
_logger
.
debug
(
'Sending shutdown signal'
)
self
.
_req_rep
(
ZMQServer
.
DONE
)


def finalize(self):
        """Closes socket and terminates context

        NO-OP if already closed.

        """
        if self._context is not None:
            if self._socket is not None:
                self._close_socket(confused=False)
            self._context.term()
            self._context = None
            self._poll = None

def finalize ( self ) : if self . _context is not None : if self . _socket is not None : self . _close_socket ( confused = False ) self . _context . term ( ) self . _context = None self . _poll = None


def start(self, test_connection=True):
        """Starts connection to server if not existent.

        NO-OP if connection is already established.
        Makes ping-pong test as well if desired.

        """
        if self._context is None:
            self._logger.debug('Starting Client')
            self._context = zmq.Context()
            self._poll = zmq.Poller()
            self._start_socket()
            if test_connection:
                self.test_ping()

def
start
(
self
,
test_connection
=
True
)
:
if
self
.
_context
is
:
self
.
_logger
.
debug
(
'Starting Client'
)
self
.
_context
=
zmq
.
Context
(
)
self
.
_poll
=
zmq
.
Poller
(
)
self
.
_start_socket
(
)
if
test_connection
:
self
.
test_ping
(
)

Starts connection to server if not existent .


def _req_rep_retry(self, request):
        """Returns response and number of retries"""
        retries_left = self.RETRIES
        while retries_left:
            self._logger.log(1, 'Sending REQ `%s`', request)
            self._send_request(request)
            socks = dict(self._poll.poll(self.TIMEOUT))
            if socks.get(self._socket) == zmq.POLLIN:
                response = self._receive_response()
                self._logger.log(1, 'Received REP `%s`', response)
                return response, self.RETRIES - retries_left
            else:
                self._logger.debug('No response from server (%d retries left)' %
                                   retries_left)
                self._close_socket(confused=True)
                retries_left -= 1
                if retries_left == 0:
                    raise RuntimeError('Server seems to be offline!')
                time.sleep(self.SLEEP)
                self._start_socket()


def acquire(self):
        """Acquires lock and returns `True`

        Blocks until lock is available.

        """
        self.start(test_connection=False)
        while True:
            str_response, retries = self._req_rep_retry(LockerServer.LOCK)
            response = str_response.split(LockerServer.DELIMITER)
            if response[0] == LockerServer.GO:
                return True
            elif response[0] == LockerServer.LOCK_ERROR and retries > 0:
                # Message was sent but Server response was lost and we tried again
                self._logger.error(str_response + '; Probably due to retry')
                return True
            elif response[0] == LockerServer.WAIT:
                time.sleep(self.SLEEP)
            else:
                raise RuntimeError('Response `%s` not understood' % response)

def
acquire
(
self
)
:
self
.
start
(
test_connection
=
False
)
while
True
:
str_response
,
retries
=
self
.
_req_rep_retry
(
LockerServer
.
LOCK
)
response
=
str_response
.
split
(
LockerServer
.
DELIMITER
)
if
response
[
0
]
==
LockerServer
.
GO
:
return
True
elif
response
[
0
]
==
LockerServer
.
LOCK_ERROR
and
retries
>
0
:
self
.
_logger
.
error
(
str_response
+
'; Probably due to retry'
)
return
True
elif
response
[
0
]
==
LockerServer
.
WAIT
:
time
.
sleep
(
self
.
SLEEP
)
else
:
raise
RuntimeError
(
'Response `%s` not understood'
%
response
)


def listen(self):
        """ Handles listening requests from the client.

        There are 4 types of requests:

        1- Check space in the queue
        2- Tests the socket
        3- If there is a space, it sends data
        4- after data is sent, puts it to queue for storing

        """
        count = 0
        self._start()
        while True:
            result = self._socket.recv_pyobj()

            if isinstance(result, tuple):
                request, data = result
            else:
                request = result
                data = None

            if request == self.SPACE:
                if self.queue.qsize() + count < self.queue_maxsize:
                    self._socket.send_string(self.SPACE_AVAILABLE)
                    count += 1
                else:
                    self._socket.send_string(self.SPACE_NOT_AVAILABLE)

            elif request == self.PING:
                self._socket.send_string(self.PONG)

            elif request == self.DATA:
                self._socket.send_string(self.STORING)
                self.queue.put(data)
                count -= 1

            elif request == self.DONE:
                self._socket.send_string(ZMQServer.CLOSED)
                self.queue.put(('DONE', [], {}))
                self._close()
                break

            else:
                raise RuntimeError('I did not understand your request %s' % request)

def
listen
(
self
)
:
count
=
0
self
.
_start
(
)
while
True
:
result
=
self
.
_socket
.
recv_pyobj
(
)
if
isinstance
(
result
,
tuple
)
:
request
,
data
=
result
else
:
request
=
result
data
=
if
request
==
self
.
SPACE
:
if
self
.
queue
.
qsize
(
)
+
count
<
self
.
queue_maxsize
:
self
.
_socket
.
send_string
(
self
.
SPACE_AVAILABLE
)
count
+=
1
else
:
self
.
_socket
.
send_string
(
self
.
SPACE_NOT_AVAILABLE
)
elif
request
==
self
.
PING
:
self
.
_socket
.
send_string
(
self
.
PONG
)
elif
request
==
self
.
DATA
:
self
.
_socket
.
send_string
(
self
.
STORING
)
self
.
queue
.
put
(
data
)
count
-=
1
elif
request
==
self
.
DONE
:
self
.
_socket
.
send_string
(
ZMQServer
.
CLOSED
)
self
.
queue
.
put
(
(
'DONE'
,
[
]
,
{
}
)
)
self
.
_close
(
)
break
else
:
raise
RuntimeError
(
'I did not understand your request %s'
%
request
)

Handles listening requests from the client .