def _all_get_or_create_table(self, where, tablename, description, expectedrows=None):
        """Creates a new table, or if the table already exists, returns it."""
        where_node = self._hdf5file.get_node(where)

        if not tablename in where_node:
            if not expectedrows is None:
                table = self._hdf5file.create_table(where=where_node, name=tablename,
                                              description=description, title=tablename,
                                              expectedrows=expectedrows,
                                              filters=self._all_get_filters())
            else:
                table = self._hdf5file.create_table(where=where_node, name=tablename,
                                              description=description, title=tablename,
                                              filters=self._all_get_filters())
        else:
            table = where_node._f_get_child(tablename)

        return table

def _all_get_or_create_table ( self , where , tablename , description , expectedrows = None ) : where_node = self . _hdf5file . get_node ( where ) if not tablename in where_node : if not expectedrows is None : table = self . _hdf5file . create_table ( where = where_node , name = tablename , description = description , title = tablename , expectedrows = expectedrows , filters = self . _all_get_filters ( ) ) else : table = self . _hdf5file . create_table ( where = where_node , name = tablename , description = description , title = tablename , filters = self . _all_get_filters ( ) ) else : table = where_node . _f_get_child ( tablename ) return table

Creates a new table or if the table already exists returns it .


def _all_get_node_by_name(self, name):
        """Returns an HDF5 node by the path specified in `name`"""
        path_name = name.replace('.', '/')
        where = '/%s/%s' % (self._trajectory_name, path_name)
        return self._hdf5file.get_node(where=where)

def _all_get_node_by_name ( self , name ) : path_name = name . replace ( '.' , '/' ) where = '/%s/%s' % ( self . _trajectory_name , path_name ) return self . _hdf5file . get_node ( where = where )

Returns an HDF5 node by the path specified in name


def _all_set_attributes_to_recall_natives(data, ptitem, prefix):
        """Stores original data type to hdf5 node attributes for preserving the data type.

        :param data:

            Data to be stored

        :param ptitem:

            HDF5 node to store data types as attributes. Can also be just a PTItemMock.

        :param prefix:

            String prefix to label and name data in HDF5 attributes

        """

        # If `data` is a container, remember the container type
        if type(data) is tuple:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_TUPLE)

        elif type(data) is list:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_LIST)

        elif type(data) is np.ndarray:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_NDARRAY)

        elif type(data) is np.matrix:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_MATRIX)

        elif type(data) in pypetconstants.PARAMETER_SUPPORTED_DATA:
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_SCALAR)

            strtype = type(data).__name__

            if not strtype in pypetconstants.PARAMETERTYPEDICT:
                raise TypeError('I do not know how to handle `%s` its type is `%s`.' %
                                (str(data), repr(type(data))))

            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE, strtype)

        elif type(data) is dict:
            if len(data) > 0:
                HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_DICT)
            else:
                HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.COLL_TYPE,
                                   HDF5StorageService.COLL_EMPTY_DICT)
        else:
            raise TypeError('I do not know how to handle `%s` its type is `%s`.' %
                            (str(data), repr(type(data))))

        if type(data) in (list, tuple):
            # If data is a list or tuple we need to remember the data type of the elements
            # in the list or tuple.
            # We do NOT need to remember the elements of `dict` explicitly, though.
            # `dict` is stored
            # as an `ObjectTable` and thus types are already conserved.
            if len(data) > 0:
                strtype = type(data[0]).__name__

                if not strtype in pypetconstants.PARAMETERTYPEDICT:
                    raise TypeError('I do not know how to handle `%s` its type is '
                                    '`%s`.' % (str(data), strtype))

                HDF5StorageService._all_set_attr(ptitem, prefix +
                                       HDF5StorageService.SCALAR_TYPE, strtype)
        elif (type(data) in (np.ndarray, np.matrix) and
                  np.issubdtype(data.dtype, str)):
            HDF5StorageService._all_set_attr(ptitem, prefix + HDF5StorageService.SCALAR_TYPE,
                                   str.__name__)

Stores original data type to hdf5 node attributes for preserving the data type .


def _all_recall_native_type(self, data, ptitem, prefix):
        """Checks if loaded data has the type it was stored in. If not converts it.

        :param data: Data item to be checked and converted
        :param ptitem: HDf5 Node or Leaf from where data was loaded
        :param prefix: Prefix for recalling the data type from the hdf5 node attributes

        :return:

            Tuple, first item is the (converted) `data` item, second boolean whether
            item was converted or not.

        """
        typestr = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.SCALAR_TYPE)
        colltype = self._all_get_from_attrs(ptitem, prefix + HDF5StorageService.COLL_TYPE)
        type_changed = False

        # Check what the original data type was from the hdf5 node attributes
        if colltype == HDF5StorageService.COLL_SCALAR:
            # Here data item was a scalar

            if isinstance(data, np.ndarray):
                # If we recall a numpy scalar, pytables loads a 1d array :-/
                # So we have to change it to a real scalar value
                data = np.array([data])[0]
                type_changed = True

            if not typestr is None:
                # Check if current type and stored type match
                # if not convert the data
                if typestr != type(data).__name__:

                    if typestr == str.__name__:
                        data = data.decode(self._encoding)
                    else:
                        try:
                            data = pypetconstants.PARAMETERTYPEDICT[typestr](data)
                        except KeyError:
                            # For compatibility with files from older pypet versions
                            data = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](data)

                    type_changed = True

        elif (colltype == HDF5StorageService.COLL_TUPLE or
                        colltype == HDF5StorageService.COLL_LIST):
            # Here data item was originally a tuple or a list

            if type(data) is not list and type is not tuple:
                # If the original type cannot be recalled, first convert it to a list
                type_changed = True
                data = list(data)

            if len(data) > 0:
                first_item = data[0]
                # Check if the type of the first item was conserved
                if not typestr == type(first_item).__name__:

                    if not isinstance(data, list):
                        data = list(data)

                    # If type was not conserved we need to convert all items
                    # in the list or tuple
                    for idx, item in enumerate(data):
                        if typestr == str.__name__:
                            data[idx] = data[idx].decode(self._encoding)
                        else:
                            try:
                                data[idx] = pypetconstants.PARAMETERTYPEDICT[typestr](item)
                            except KeyError:
                                # For compatibility with files from older pypet versions:
                                data[idx] = pypetconstants.COMPATPARAMETERTYPEDICT[typestr](item)
                        type_changed = True

            if colltype == HDF5StorageService.COLL_TUPLE:
                # If it was originally a tuple we need to convert it back to tuple
                if type(data) is not tuple:
                    data = tuple(data)
                    type_changed = True
        elif colltype == HDF5StorageService.COLL_EMPTY_DICT:
            data = {}
            type_changed = True
        elif isinstance(data, np.ndarray):

            if typestr == str.__name__:
                data = np.core.defchararray.decode(data, self._encoding)
                type_changed = True

            if colltype == HDF5StorageService.COLL_MATRIX:
                # Here data item was originally a matrix
                data = np.matrix(data)
                type_changed = True

        return data, type_changed

def _all_recall_native_type ( self , data , ptitem , prefix ) : typestr = self . _all_get_from_attrs ( ptitem , prefix + HDF5StorageService . SCALAR_TYPE ) colltype = self . _all_get_from_attrs ( ptitem , prefix + HDF5StorageService . COLL_TYPE ) type_changed = False if colltype == HDF5StorageService . COLL_SCALAR : if isinstance ( data , np . ndarray ) : data = np . array ( [ data ] ) [ 0 ] type_changed = True if not typestr is None : if typestr != type ( data ) . __name__ : if typestr == str . __name__ : data = data . decode ( self . _encoding ) else : try : data = pypetconstants . PARAMETERTYPEDICT [ typestr ] ( data ) except KeyError : data = pypetconstants . COMPATPARAMETERTYPEDICT [ typestr ] ( data ) type_changed = True elif ( colltype == HDF5StorageService . COLL_TUPLE or colltype == HDF5StorageService . COLL_LIST ) : if type ( data ) is not list and type is not tuple : type_changed = True data = list ( data ) if len ( data ) > 0 : first_item = data [ 0 ] if not typestr == type ( first_item ) . __name__ : if not isinstance ( data , list ) : data = list ( data ) for idx , item in enumerate ( data ) : if typestr == str . __name__ : data [ idx ] = data [ idx ] . decode ( self . _encoding ) else : try : data [ idx ] = pypetconstants . PARAMETERTYPEDICT [ typestr ] ( item ) except KeyError : data [ idx ] = pypetconstants . COMPATPARAMETERTYPEDICT [ typestr ] ( item ) type_changed = True if colltype == HDF5StorageService . COLL_TUPLE : if type ( data ) is not tuple : data = tuple ( data ) type_changed = True elif colltype == HDF5StorageService . COLL_EMPTY_DICT : data = { } type_changed = True elif isinstance ( data , np . ndarray ) : if typestr == str . __name__ : data = np . core . defchararray . decode ( data , self . _encoding ) type_changed = True if colltype == HDF5StorageService . COLL_MATRIX : data = np . matrix ( data ) type_changed = True return data , type_changed

Checks if loaded data has the type it was stored in . If not converts it .


def _all_add_or_modify_row(self, item_name, insert_dict, table, index=None, condition=None,
                               condvars=None,
                               flags=(ADD_ROW, MODIFY_ROW,)):
        """Adds or changes a row in a pytable.

        :param item_name: Name of item, the row is about, only important for throwing errors.

        :param insert_dict:

            Dictionary of data that is about to be inserted into the pytables row.

        :param table:

            The table to insert or modify a row in

        :param index:

            Index of row to be modified. Instead of an index a search condition can be
            used as well, see below.

        :param condition:

            Condition to search for in the table

        :param condvars:

            Variables for the search condition

        :param flags:

            Flags whether to add, modify, or remove a row in the table

        """
        if len(flags) == 0:
            # No flags means no-op
            return

        # You can only specify either an index or a condition not both
        if index is not None and condition is not None:
            raise ValueError('Please give either a condition or an index or none!')
        elif condition is not None:
            row_iterator = table.where(condition, condvars=condvars)
        elif index is not None:
            row_iterator = table.iterrows(index, index + 1)
        else:
            row_iterator = None

        try:
            row = next(row_iterator)
        except TypeError:
            row = None
        except StopIteration:
            row = None

        # multiple_entries = []

        if ((HDF5StorageService.MODIFY_ROW in flags or HDF5StorageService.ADD_ROW in flags) and
                    HDF5StorageService.REMOVE_ROW in flags):
            # You cannot remove and modify or add at the same time
            raise ValueError('You cannot add or modify and remove a row at the same time.')

        if row is None and HDF5StorageService.ADD_ROW in flags:
            # Here we add a new row

            row = table.row

            self._all_insert_into_row(row, insert_dict)

            row.append()

        elif row is not None and HDF5StorageService.MODIFY_ROW in flags:
            # Here we modify an existing row

            self._all_insert_into_row(row, insert_dict)

            row.update()

        elif HDF5StorageService.REMOVE_ROW in flags:
            # Here we delete an existing row

            if row is not None:
                # Only delete if the row does exist otherwise we do not have to do anything
                row_number = row.nrow
                try:
                    table.remove_rows(start=row_number, stop=row_number+1)
                except NotImplementedError:
                    pass
                    # We get here if we try to remove the last row of a table
                    # there is nothing we can do but keep it :-(
        else:
            raise ValueError('Something is wrong, you might not have found '
                             'a row, or your flags are not set appropriately')

        self._all_kill_iterator(row_iterator)
        table.flush()

        if HDF5StorageService.REMOVE_ROW not in flags and row is None:
            raise RuntimeError('Could not add or modify entries of `%s` in '
                               'table %s' % (item_name, table._v_name))


def _all_insert_into_row(self, row, insert_dict):
        """Copies data from `insert_dict` into a pytables `row`."""
        for key, val in insert_dict.items():
            try:
                row[key] = val
            except KeyError as ke:
                self._logger.warning('Could not write `%s` into a table, ' % key + repr(ke))

def
_all_insert_into_row
(
self
,
row
,
insert_dict
)
:
for
key
,
val
in
insert_dict
.
items
(
)
:
try
:
row
[
key
]
=
val
except
KeyError
as
ke
:
self
.
_logger
.
warning
(
'Could not write `%s` into a table, '
%
key
+
repr
(
ke
)
)

Copies data from insert_dict into a pytables row .


def _all_extract_insert_dict(self, item, colnames, additional_info=None):
        """Extracts information from a given item to be stored into a pytable row.

        Items can be a variety of things here, trajectories, single runs, group node,
        parameters, results.

        :param item: Item from which data should be extracted

        :param colnames: Names of the columns in the pytable

        :param additional_info: (dict)

            Additional information that should be stored into the pytable row that cannot be
            read out from `item`.

        :return: Dictionary containing the data to be inserted into a row

        """
        insert_dict = {}

        if 'length' in colnames:
            insert_dict['length'] = len(item)

        if 'comment' in colnames:
            comment = self._all_cut_string(item.v_comment.encode('utf-8'),
                                           pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH,
                                           self._logger)

            insert_dict['comment'] = comment

        if 'location' in colnames:
            insert_dict['location'] = item.v_location.encode('utf-8')

        if 'name' in colnames:
            name = item._name if (not item.v_is_root or not item.v_is_run) else item._crun
            insert_dict['name'] = name.encode('utf-8')

        if 'class_name' in colnames:
            insert_dict['class_name'] = item.f_get_class_name().encode('utf-8')

        if 'value' in colnames:
            insert_dict['value'] = self._all_cut_string(
                item.f_val_to_str().encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_VALUE_LENGTH,
                self._logger)

        if 'hexdigest' in colnames:
            insert_dict['hexdigest'] = additional_info['hexdigest']

        if 'idx' in colnames:
            insert_dict['idx'] = item.v_idx

        if 'time' in colnames:
            time_ = item._time
            insert_dict['time'] = time_.encode('utf-8')

        if 'timestamp' in colnames:
            timestamp = item._timestamp
            insert_dict['timestamp'] = timestamp

        if 'range' in colnames:
            third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // 3 + 10
            item_range = itools.islice(item.f_get_range(copy=False), 0, third_length)
            range_string = ', '.join([repr(x) for x in item_range])
            insert_dict['range'] = self._all_cut_string(
                range_string.encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,
                self._logger)

        # To allow backwards compatibility
        if 'array' in colnames:
            third_length = pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH // 3 + 10
            item_range = itools.islice(item.f_get_range(copy=False), 0, third_length)
            range_string = ', '.join([repr(x) for x in item_range])
            insert_dict['array'] = self._all_cut_string(
                range_string.encode('utf-8'),
                pypetconstants.HDF5_STRCOL_MAX_RANGE_LENGTH,
                self._logger)

        if 'version' in colnames:
            insert_dict['version'] = item.v_version.encode('utf-8')

        if 'python' in colnames:
            insert_dict['python'] = item.v_python.encode('utf-8')

        if 'finish_timestamp' in colnames:
            insert_dict['finish_timestamp'] = item._finish_timestamp_run

        return insert_dict

def
_all_extract_insert_dict
(
self
,
item
,
colnames
,
additional_info
=
)
:
insert_dict
=
{
}
if
'length'
in
colnames
:
insert_dict
[
'length'
]
=
len
(
item
)
if
'comment'
in
colnames
:
comment
=
self
.
_all_cut_string
(
item
.
v_comment
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_COMMENT_LENGTH
,
self
.
_logger
)
insert_dict
[
'comment'
]
=
comment
if
'location'
in
colnames
:
insert_dict
[
'location'
]
=
item
.
v_location
.
encode
(
'utf-8'
)
if
'name'
in
colnames
:
name
=
item
.
_name
if
(
not
item
.
v_is_root
or
not
item
.
v_is_run
)
else
item
.
_crun
insert_dict
[
'name'
]
=
name
.
encode
(
'utf-8'
)
if
'class_name'
in
colnames
:
insert_dict
[
'class_name'
]
=
item
.
f_get_class_name
(
)
.
encode
(
'utf-8'
)
if
'value'
in
colnames
:
insert_dict
[
'value'
]
=
self
.
_all_cut_string
(
item
.
f_val_to_str
(
)
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_VALUE_LENGTH
,
self
.
_logger
)
if
'hexdigest'
in
colnames
:
insert_dict
[
'hexdigest'
]
=
additional_info
[
'hexdigest'
]
if
'idx'
in
colnames
:
insert_dict
[
'idx'
]
=
item
.
v_idx
if
'time'
in
colnames
:
time_
=
item
.
_time
insert_dict
[
'time'
]
=
time_
.
encode
(
'utf-8'
)
if
'timestamp'
in
colnames
:
timestamp
=
item
.
_timestamp
insert_dict
[
'timestamp'
]
=
timestamp
if
'range'
in
colnames
:
third_length
=
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
//
3
+
10
item_range
=
itools
.
islice
(
item
.
f_get_range
(
copy
=
False
)
,
0
,
third_length
)
range_string
=
', '
.
join
(
[
repr
(
x
)
for
x
in
item_range
]
)
insert_dict
[
'range'
]
=
self
.
_all_cut_string
(
range_string
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
,
self
.
_logger
)
if
'array'
in
colnames
:
third_length
=
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
//
3
+
10
item_range
=
itools
.
islice
(
item
.
f_get_range
(
copy
=
False
)
,
0
,
third_length
)
range_string
=
', '
.
join
(
[
repr
(
x
)
for
x
in
item_range
]
)
insert_dict
[
'array'
]
=
self
.
_all_cut_string
(
range_string
.
encode
(
'utf-8'
)
,
pypetconstants
.
HDF5_STRCOL_MAX_RANGE_LENGTH
,
self
.
_logger
)
if
'version'
in
colnames
:
insert_dict
[
'version'
]
=
item
.
v_version
.
encode
(
'utf-8'
)
if
'python'
in
colnames
:
insert_dict
[
'python'
]
=
item
.
v_python
.
encode
(
'utf-8'
)
if
'finish_timestamp'
in
colnames
:
insert_dict
[
'finish_timestamp'
]
=
item
.
_finish_timestamp_run
return
insert_dict

Extracts information from a given item to be stored into a pytable row .


def _all_cut_string(string, max_length, logger):
        """Cuts string data to the maximum length allowed in a pytables column
        if string is too long.

        :param string: String to be cut
        :param max_length: Maximum allowed string length
        :param logger: Logger where messages about truncating should be written

        :return: String, cut if too long

        """
        if len(string) > max_length:
            logger.debug('The string `%s` was too long I truncated it to'
                         ' %d characters' %
                         (string, max_length))
            string = string[0:max_length - 3] + '...'.encode('utf-8')

        return string

Cuts string data to the maximum length allowed in a pytables column if string is too long .


def _all_create_or_get_group(self, name, parent_hdf5_group=None):
        """Creates or returns a group"""
        if not name in parent_hdf5_group:
            new_hdf5_group = self._hdf5file.create_group(where=parent_hdf5_group,
                                                      name=name,
                                                      title=name,
                                                      filters=self._all_get_filters())
            return new_hdf5_group, True
        else:
            new_hdf5_group = parent_hdf5_group._f_get_child(name)
            return new_hdf5_group, False

def _all_create_or_get_group ( self , name , parent_hdf5_group = None ) : if not name in parent_hdf5_group : new_hdf5_group = self . _hdf5file . create_group ( where = parent_hdf5_group , name = name , title = name , filters = self . _all_get_filters ( ) ) return new_hdf5_group , True else : new_hdf5_group = parent_hdf5_group . _f_get_child ( name ) return new_hdf5_group , False


def _all_create_or_get_groups(self, key, start_hdf5_group=None):
        """Creates new or follows existing group nodes along a given colon separated `key`.

        :param key:

            Colon separated path along hdf5 file, e.g. `parameters.mobiles.cars`.

        :param start_hdf5_group:

            HDF5 group from where to start, leave `None` for the trajectory group.

        :return:

            Final group node, e.g. group node with name `cars`.

        """
        if start_hdf5_group is None:
            newhdf5_group = self._trajectory_group
        else:
            newhdf5_group = start_hdf5_group
        created = False
        if key == '':
            return newhdf5_group, created
        split_key = key.split('.')
        for name in split_key:
            newhdf5_group, created = self._all_create_or_get_group(name, newhdf5_group)

        return newhdf5_group, created

def _all_create_or_get_groups ( self , key , start_hdf5_group = None ) : if start_hdf5_group is None : newhdf5_group = self . _trajectory_group else : newhdf5_group = start_hdf5_group created = False if key == '' : return newhdf5_group , created split_key = key . split ( '.' ) for name in split_key : newhdf5_group , created = self . _all_create_or_get_group ( name , newhdf5_group ) return newhdf5_group , created

Creates new or follows existing group nodes along a given colon separated key .


def _ann_store_annotations(self, item_with_annotations, node, overwrite=False):
        """Stores annotations into an hdf5 file."""

        # If we overwrite delete all annotations first
        if overwrite is True or overwrite == 'v_annotations':
            annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)
            if annotated:
                current_attrs = node._v_attrs
                for attr_name in current_attrs._v_attrnames:
                    if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):
                        delattr(current_attrs, attr_name)
                delattr(current_attrs, HDF5StorageService.ANNOTATED)
                self._hdf5file.flush()

        # Only store annotations if the item has some
        if not item_with_annotations.v_annotations.f_is_empty():

            anno_dict = item_with_annotations.v_annotations._dict

            current_attrs = node._v_attrs

            changed = False

            for field_name in anno_dict:
                val = anno_dict[field_name]

                field_name_with_prefix = HDF5StorageService.ANNOTATION_PREFIX + field_name
                if field_name_with_prefix not in current_attrs:
                    # Only store *new* annotations, if they already exist on disk, skip storage
                    setattr(current_attrs, field_name_with_prefix, val)
                    changed = True

            if changed:
                setattr(current_attrs, HDF5StorageService.ANNOTATED, True)
                self._hdf5file.flush()

def _ann_store_annotations ( self , item_with_annotations , node , overwrite = False ) : if overwrite is True or overwrite == 'v_annotations' : annotated = self . _all_get_from_attrs ( node , HDF5StorageService . ANNOTATED ) if annotated : current_attrs = node . _v_attrs for attr_name in current_attrs . _v_attrnames : if attr_name . startswith ( HDF5StorageService . ANNOTATION_PREFIX ) : delattr ( current_attrs , attr_name ) delattr ( current_attrs , HDF5StorageService . ANNOTATED ) self . _hdf5file . flush ( ) if not item_with_annotations . v_annotations . f_is_empty ( ) : anno_dict = item_with_annotations . v_annotations . _dict current_attrs = node . _v_attrs changed = False for field_name in anno_dict : val = anno_dict [ field_name ] field_name_with_prefix = HDF5StorageService . ANNOTATION_PREFIX + field_name if field_name_with_prefix not in current_attrs : setattr ( current_attrs , field_name_with_prefix , val ) changed = True if changed : setattr ( current_attrs , HDF5StorageService . ANNOTATED , True ) self . _hdf5file . flush ( )


def _ann_load_annotations(self, item_with_annotations, node):
        """Loads annotations from disk."""

        annotated = self._all_get_from_attrs(node, HDF5StorageService.ANNOTATED)

        if annotated:

            annotations = item_with_annotations.v_annotations

            # You can only load into non-empty annotations, to prevent overwriting data in RAM
            if not annotations.f_is_empty():
                raise TypeError('Loading into non-empty annotations!')

            current_attrs = node._v_attrs

            for attr_name in current_attrs._v_attrnames:

                if attr_name.startswith(HDF5StorageService.ANNOTATION_PREFIX):
                    key = attr_name
                    key = key.replace(HDF5StorageService.ANNOTATION_PREFIX, '')

                    data = getattr(current_attrs, attr_name)
                    setattr(annotations, key, data)

def
_ann_load_annotations
(
self
,
item_with_annotations
,
node
)
:
annotated
=
self
.
_all_get_from_attrs
(
node
,
HDF5StorageService
.
ANNOTATED
)
if
annotated
:
annotations
=
item_with_annotations
.
v_annotations
if
not
annotations
.
f_is_empty
(
)
:
raise
TypeError
(
'Loading into non-empty annotations!'
)
current_attrs
=
node
.
_v_attrs
for
attr_name
in
current_attrs
.
_v_attrnames
:
if
attr_name
.
startswith
(
HDF5StorageService
.
ANNOTATION_PREFIX
)
:
key
=
attr_name
key
=
key
.
replace
(
HDF5StorageService
.
ANNOTATION_PREFIX
,
''
)
data
=
getattr
(
current_attrs
,
attr_name
)
setattr
(
annotations
,
key
,
data
)


def _grp_store_group(self, traj_group, store_data=pypetconstants.STORE_DATA,
                         with_links=True, recursive=False, max_depth=None,
                         _hdf5_group=None, _newly_created=False):
        """Stores a group node.

        For group nodes only annotations and comments need to be stored.

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return
        elif store_data == pypetconstants.STORE_DATA_SKIPPING and traj_group._stored:
            self._logger.debug('Already found `%s` on disk I will not store it!' %
                                   traj_group.v_full_name)
        elif not recursive:
            if _hdf5_group is None:
                _hdf5_group, _newly_created = self._all_create_or_get_groups(traj_group.v_full_name)

            overwrite = store_data == pypetconstants.OVERWRITE_DATA

            if (traj_group.v_comment != '' and
                    (HDF5StorageService.COMMENT not in _hdf5_group._v_attrs or overwrite)):
                setattr(_hdf5_group._v_attrs, HDF5StorageService.COMMENT, traj_group.v_comment)

            if ((_newly_created or overwrite) and
                type(traj_group) not in (nn.NNGroupNode, nn.ConfigGroup, nn.ParameterGroup,
                                             nn.DerivedParameterGroup, nn.ResultGroup)):
                # We only store the name of the class if it is not one of the standard groups,
                # that are always used.
                setattr(_hdf5_group._v_attrs, HDF5StorageService.CLASS_NAME,
                        traj_group.f_get_class_name())

            self._ann_store_annotations(traj_group, _hdf5_group, overwrite=overwrite)
            self._hdf5file.flush()
            traj_group._stored = True

            # Signal completed node loading
            self._node_processing_timer.signal_update()

        if recursive:
            parent_traj_group = traj_group.f_get_parent()
            parent_hdf5_group = self._all_create_or_get_groups(parent_traj_group.v_full_name)[0]

            self._tree_store_nodes_dfs(parent_traj_group, traj_group.v_name, store_data=store_data,
                                       with_links=with_links, recursive=recursive,
                                       max_depth=max_depth, current_depth=0,
                                       parent_hdf5_group=parent_hdf5_group)


def _grp_load_group(self, traj_group, load_data=pypetconstants.LOAD_DATA, with_links=True,
                        recursive=False, max_depth=None,
                        _traj=None, _as_new=False, _hdf5_group=None):
        """Loads a group node and potentially everything recursively below"""
        if _hdf5_group is None:
            _hdf5_group = self._all_get_node_by_name(traj_group.v_full_name)
            _traj = traj_group.v_root

        if recursive:
            parent_traj_node = traj_group.f_get_parent()
            self._tree_load_nodes_dfs(parent_traj_node, load_data=load_data, with_links=with_links,
                                  recursive=recursive, max_depth=max_depth,
                                  current_depth=0,
                                  trajectory=_traj, as_new=_as_new,
                                  hdf5_group=_hdf5_group)
        else:
            if load_data == pypetconstants.LOAD_NOTHING:
                return

            elif load_data == pypetconstants.OVERWRITE_DATA:
                traj_group.v_annotations.f_empty()
                traj_group.v_comment = ''

            self._all_load_skeleton(traj_group, _hdf5_group)
            traj_group._stored = not _as_new

            # Signal completed node loading
            self._node_processing_timer.signal_update()


def _all_load_skeleton(self, traj_node, hdf5_group):
        """Reloads skeleton data of a tree node"""
        if traj_node.v_annotations.f_is_empty():
            self._ann_load_annotations(traj_node, hdf5_group)
        if traj_node.v_comment == '':
            comment = self._all_get_from_attrs(hdf5_group, HDF5StorageService.COMMENT)
            if comment is None:
                comment = ''
            traj_node.v_comment = comment

def _all_load_skeleton ( self , traj_node , hdf5_group ) : if traj_node . v_annotations . f_is_empty ( ) : self . _ann_load_annotations ( traj_node , hdf5_group ) if traj_node . v_comment == '' : comment = self . _all_get_from_attrs ( hdf5_group , HDF5StorageService . COMMENT ) if comment is None : comment = '' traj_node . v_comment = comment


def _prm_extract_missing_flags(data_dict, flags_dict):
        """Extracts storage flags for data in `data_dict`
        if they were not specified in `flags_dict`.

        See :const:`~pypet.storageservice.HDF5StorageService.TYPE_FLAG_MAPPING`
        for how to store different types of data per default.

        """
        for key, data in data_dict.items():
            if not key in flags_dict:
                dtype = type(data)
                if (dtype is np.ndarray or dtype is dict) and len(data) == 0:
                        # Empty containers are stored as an Array
                        # No need to ask for tuple or list, because they are always
                        # stored as arrays.
                        flags_dict[key] = HDF5StorageService.ARRAY
                        continue
                else:
                    try:
                        flags_dict[key] = HDF5StorageService.TYPE_FLAG_MAPPING[dtype]
                    except KeyError:
                        raise pex.NoSuchServiceError('I cannot store `%s`, I do not understand the'
                                                     'type `%s`.' % (key, str(dtype)))

Extracts storage flags for data in data_dict if they were not specified in flags_dict .


def _prm_meta_add_summary(self, instance):
        """Adds data to the summary tables and returns if `instance`s comment has to be stored.

        Also moves comments upwards in the hierarchy if purge_duplicate_comments is true
        and a lower index run has completed. Only necessary for *multiprocessing*.

        :return: Tuple

            * String specifying the subtree

            * Boolean whether to store the comment to `instance`s hdf5 node

        """
        if instance.v_comment == '':
            return False

        where = instance.v_branch
        definitely_store_comment = True

        # Get the hexdigest of the comment to see if such a comment has been stored before
        bytes_comment = instance.v_comment.encode('utf-8')
        hexdigest = hashlib.sha1(bytes_comment).hexdigest()
        hexdigest = hexdigest.encode('utf-8')

        # Get the overview table
        table_name = where + '_summary'

        # Check if the overview table exists, otherwise skip the rest of
        # the meta adding
        if table_name in self._overview_group:
            table = getattr(self._overview_group, table_name)
        else:
            return definitely_store_comment

        try:
            condvars = {'hexdigestcol': table.cols.hexdigest,
                'hexdigest': hexdigest}

            condition = """(hexdigestcol == hexdigest)"""

            row_iterator = table.where(condition, condvars=condvars)

            row = None
            try:
                row = next(row_iterator)
            except StopIteration:
                pass

            if row is None:
                self._all_store_param_or_result_table_entry(instance, table,
                                                            flags=(
                                                                HDF5StorageService.ADD_ROW,),
                                                            additional_info={
                                                                'hexdigest': hexdigest})

                definitely_store_comment = True
            else:
                definitely_store_comment = False
                self._all_kill_iterator(row_iterator)

        except pt.NoSuchNodeError:
            definitely_store_comment = True

        return definitely_store_comment

def _prm_meta_add_summary ( self , instance ) : if instance . v_comment == '' : return False where = instance . v_branch definitely_store_comment = True bytes_comment = instance . v_comment . encode ( 'utf-8' ) hexdigest = hashlib . sha1 ( bytes_comment ) . hexdigest ( ) hexdigest = hexdigest . encode ( 'utf-8' ) table_name = where + '_summary' if table_name in self . _overview_group : table = getattr ( self . _overview_group , table_name ) else : return definitely_store_comment try : condvars = { 'hexdigestcol' : table . cols . hexdigest , 'hexdigest' : hexdigest } condition = row_iterator = table . where ( condition , condvars = condvars ) row = None try : row = next ( row_iterator ) except StopIteration : pass if row is None : self . _all_store_param_or_result_table_entry ( instance , table , flags = ( HDF5StorageService . ADD_ROW , ) , additional_info = { 'hexdigest' : hexdigest } ) definitely_store_comment = True else : definitely_store_comment = False self . _all_kill_iterator ( row_iterator ) except pt . NoSuchNodeError : definitely_store_comment = True return definitely_store_comment

Adds data to the summary tables and returns if instance s comment has to be stored .


def _prm_add_meta_info(self, instance, group, overwrite=False):
        """Adds information to overview tables and meta information to
        the `instance`s hdf5 `group`.

        :param instance: Instance to store meta info about
        :param group: HDF5 group of instance
        :param overwrite: If data should be explicitly overwritten

        """

        if overwrite:
            flags = ()
        else:
            flags = (HDF5StorageService.ADD_ROW,)

        definitely_store_comment = True
        try:
            # Check if we need to store the comment. Maybe update the overview tables
            # accordingly if the current run index is lower than the one in the table.
            definitely_store_comment = self._prm_meta_add_summary(instance)

            try:
                # Update the summary overview table
                table_name = instance.v_branch + '_overview'

                table = getattr(self._overview_group, table_name)
                if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:

                    self._all_store_param_or_result_table_entry(instance, table,
                                                                flags=flags)
            except pt.NoSuchNodeError:
                pass
        except Exception as exc:
            self._logger.error('Could not store information table due to `%s`.' % repr(exc))

        if ((not self._purge_duplicate_comments or definitely_store_comment) and
                    instance.v_comment != ''):
            # Only add the comment if necessary
            setattr(group._v_attrs, HDF5StorageService.COMMENT, instance.v_comment)

        # Add class name and whether node is a leaf to the HDF5 attributes
        setattr(group._v_attrs, HDF5StorageService.CLASS_NAME, instance.f_get_class_name())
        setattr(group._v_attrs, HDF5StorageService.LEAF, True)

        if instance.v_is_parameter and instance.v_explored:
            # If the stored parameter was an explored one we need to mark this in the
            # explored overview table
            try:
                tablename = 'explored_parameters_overview'
                table = getattr(self._overview_group, tablename)

                if len(table) < pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH:
                    self._all_store_param_or_result_table_entry(instance, table,
                                                                flags=flags)
            except pt.NoSuchNodeError:
                pass
            except Exception as exc:
                self._logger.error('Could not store information '
                                   'table due to `%s`.' % repr(exc))

Adds information to overview tables and meta information to the instance s hdf5 group .


def _prm_store_from_dict(self, fullname, store_dict, hdf5_group, store_flags, kwargs):
        """Stores a `store_dict`"""
        for key, data_to_store in store_dict.items():
            # self._logger.log(1, 'SUB-Storing %s [%s]', key, str(store_dict[key]))
            original_hdf5_group = None

            flag = store_flags[key]

            if '.' in key:
                original_hdf5_group = hdf5_group
                split_key = key.split('.')
                key = split_key.pop()
                for inner_key in split_key:
                    hdf5_group, newly_created =  self._all_create_or_get_group(inner_key,
                                                                                hdf5_group)
                    if newly_created:
                        setattr(hdf5_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                                HDF5StorageService.NESTED_GROUP)
                    else:
                        store_type = self._all_get_from_attrs(hdf5_group, HDF5StorageService.STORAGE_TYPE)
                        if store_type != HDF5StorageService.NESTED_GROUP:
                            raise ValueError('You want to nested results but `%s` is already '
                                             'of type `%s`!' % (hdf5_group._v_name, store_type))

            # Iterate through the data and store according to the storage flags
            if key in hdf5_group:
                # We won't change any data that is found on disk
                self._logger.debug(
                    'Found %s already in hdf5 node of %s, so I will ignore it.' %
                    (key, fullname))
                continue

            if flag == HDF5StorageService.TABLE:
                # self._logger.log(1, 'SUB-Storing %s TABLE', key)
                self._prm_write_into_pytable(key, data_to_store, hdf5_group, fullname,
                                             **kwargs)
            elif flag == HDF5StorageService.DICT:
                # self._logger.log(1, 'SUB-Storing %s DICT', key)
                self._prm_write_dict_as_table(key, data_to_store, hdf5_group, fullname,
                                              **kwargs)
            elif flag == HDF5StorageService.ARRAY:
                # self._logger.log(1, 'SUB-Storing %s ARRAY', key)
                self._prm_write_into_array(key, data_to_store, hdf5_group, fullname,
                                           **kwargs)
            elif flag in (HDF5StorageService.CARRAY,
                          HDF5StorageService.EARRAY,
                          HDF5StorageService.VLARRAY):
                self._prm_write_into_other_array(key, data_to_store,
                                                 hdf5_group, fullname,
                                                 flag=flag, **kwargs)
            elif flag in (HDF5StorageService.SERIES,
                          HDF5StorageService.FRAME,
                          #  HDF5StorageService.PANEL
                          ):
                # self._logger.log(1, 'SUB-Storing %s PANDAS', key)
                self._prm_write_pandas_data(key, data_to_store, hdf5_group, fullname,
                                            flag, **kwargs)
            elif flag == HDF5StorageService.SHARED_DATA:
                pass  # Shared data needs to be explicitly created and is not stored on
                # the fly
            else:
                raise RuntimeError('You shall not pass!')

            if original_hdf5_group is not None:
                hdf5_group = original_hdf5_group


def _prm_store_parameter_or_result(self,
                                       instance,
                                       store_data=pypetconstants.STORE_DATA,
                                       store_flags=None,
                                       overwrite=None,
                                       with_links=False,
                                       recursive=False,
                                       _hdf5_group=None,
                                       _newly_created=False,
                                       **kwargs):
        """Stores a parameter or result to hdf5.

        :param instance:

            The instance to be stored

        :param store_data:

            How to store data

        :param store_flags:

            Dictionary containing how to store individual data, usually empty.

        :param overwrite:

            Instructions how to overwrite data

        :param with_links:

            Placeholder because leaves have no links

        :param recursive:

            Placeholder, because leaves have no children

        :param _hdf5_group:

            The hdf5 group for storing the parameter or result

        :param _newly_created:

            If should be created in a new form

        """
        if store_data == pypetconstants.STORE_NOTHING:
            return
        elif store_data == pypetconstants.STORE_DATA_SKIPPING and instance._stored:
            self._logger.debug('Already found `%s` on disk I will not store it!' %
                                   instance.v_full_name)
            return
        elif store_data == pypetconstants.OVERWRITE_DATA:
            if not overwrite:
                overwrite = True

        fullname = instance.v_full_name
        self._logger.debug('Storing `%s`.' % fullname)

        if _hdf5_group is None:
            # If no group is provided we might need to create one
            _hdf5_group, _newly_created = self._all_create_or_get_groups(fullname)

        # kwargs_flags = {} # Dictionary to change settings
        # old_kwargs = {}
        store_dict = {}
        # If the user did not supply storage flags, we need to set it to the empty dictionary
        if store_flags is None:
            store_flags = {}

        try:
            # Get the data to store from the instance
            if not instance.f_is_empty():
                store_dict = instance._store()
            try:
                # Ask the instance for storage flags
                instance_flags = instance._store_flags().copy() # copy to avoid modifying the
                # original data
            except AttributeError:
                # If it does not provide any, set it to the empty dictionary
                instance_flags = {}

            # User specified flags have priority over the flags from the instance
            instance_flags.update(store_flags)
            store_flags = instance_flags

            # If we still have data in `store_dict` about which we do not know how to store
            # it, pick default storage flags
            self._prm_extract_missing_flags(store_dict, store_flags)

            if overwrite:
                if isinstance(overwrite, str):
                    overwrite = [overwrite]

                if overwrite is True:
                    to_delete = [key for key in store_dict.keys() if key in _hdf5_group]
                    self._all_delete_parameter_or_result_or_group(instance,
                                                                  delete_only=to_delete,
                                                                  _hdf5_group=_hdf5_group)
                elif isinstance(overwrite, (list, tuple)):
                    overwrite_set = set(overwrite)
                    key_set = set(store_dict.keys())

                    stuff_not_to_be_overwritten = overwrite_set - key_set

                    if overwrite!='v_annotations' and len(stuff_not_to_be_overwritten) > 0:
                        self._logger.warning('Cannot overwrite `%s`, these items are not supposed to '
                                             'be stored by the leaf node.' %
                                             str(stuff_not_to_be_overwritten))

                    stuff_to_overwrite = overwrite_set & key_set
                    if len(stuff_to_overwrite) > 0:
                        self._all_delete_parameter_or_result_or_group(instance,
                                                                      delete_only=list(
                                                                          stuff_to_overwrite))
                else:
                    raise ValueError('Your value of overwrite `%s` is not understood. '
                                     'Please pass `True` of a list of strings to fine grain '
                                     'overwriting.' % str(overwrite))

            self._prm_store_from_dict(fullname, store_dict, _hdf5_group, store_flags, kwargs)

            # Store annotations
            self._ann_store_annotations(instance, _hdf5_group, overwrite=overwrite)

            if _newly_created or overwrite is True:
                # If we created a new group or the parameter was extended we need to
                # update the meta information and summary tables
                self._prm_add_meta_info(instance, _hdf5_group,
                                        overwrite=not _newly_created)

            instance._stored = True

            #self._logger.debug('Finished Storing `%s`.' % fullname)
            # Signal completed node loading
            self._node_processing_timer.signal_update()

        except:
            # I anything fails, we want to remove the data of the parameter again
            self._logger.error(
                'Failed storing leaf `%s`. I will remove the hdf5 data I added  again.' % fullname)
            # Delete data
            for key in store_dict.keys():
                if key in  _hdf5_group:
                    hdf5_child = _hdf5_group._f_get_child(key)
                    hdf5_child._f_remove(recursive=True)
            # If no data left delete the whole parameter
            if _hdf5_group._v_nchildren == 0:
                _hdf5_group._f_remove(recursive=True)
            raise


def _prm_write_shared_array(self, key, data, hdf5_group, full_name, flag, **kwargs):
        """Creates and array that can be used with an HDF5 array object"""

        if flag == HDF5StorageService.ARRAY:
            self._prm_write_into_array(key, data, hdf5_group, full_name, **kwargs)
        elif flag in (HDF5StorageService.CARRAY,
                    HDF5StorageService.EARRAY,
                    HDF5StorageService.VLARRAY):
            self._prm_write_into_other_array(key, data, hdf5_group, full_name,
                                             flag=flag, **kwargs)
        else:
            raise RuntimeError('Flag `%s` of hdf5 data `%s` of `%s` not understood' %
                               (flag, key, full_name))

        self._hdf5file.flush()


def _prm_write_shared_table(self, key, hdf5_group, fullname, **kwargs):
        """Creates a new empty table"""
        first_row = None
        description = None
        if 'first_row' in kwargs:
            first_row = kwargs.pop('first_row')
            if not 'description' in kwargs:
                description = {}
                for colname in first_row:
                    data = first_row[colname]
                    column = self._all_get_table_col(key, [data], fullname)
                    description[colname] = column

        if 'description' in kwargs:
            description = kwargs.pop('description')

        if 'filters' in kwargs:
            filters = kwargs.pop('filters')
        else:
            filters = self._all_get_filters(kwargs)

        table = self._hdf5file.create_table(where=hdf5_group, name=key,
                                          description=description,
                                          filters=filters,
                                          **kwargs)
        table.flush()

        if first_row is not None:
            row = table.row
            for key in description:
                row[key] = first_row[key]

            row.append()
            table.flush()

def _prm_write_shared_table ( self , key , hdf5_group , fullname , ** kwargs ) : first_row = None description = None if 'first_row' in kwargs : first_row = kwargs . pop ( 'first_row' ) if not 'description' in kwargs : description = { } for colname in first_row : data = first_row [ colname ] column = self . _all_get_table_col ( key , [ data ] , fullname ) description [ colname ] = column if 'description' in kwargs : description = kwargs . pop ( 'description' ) if 'filters' in kwargs : filters = kwargs . pop ( 'filters' ) else : filters = self . _all_get_filters ( kwargs ) table = self . _hdf5file . create_table ( where = hdf5_group , name = key , description = description , filters = filters , ** kwargs ) table . flush ( ) if first_row is not None : row = table . row for key in description : row [ key ] = first_row [ key ] row . append ( ) table . flush ( )


def _prm_write_dict_as_table(self, key, data_to_store, group, fullname, **kwargs):
        """Stores a python dictionary as pytable

        :param key:

            Name of data item to store

        :param data_to_store:

            Dictionary to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """
        if key in group:
            raise ValueError(
                'Dictionary `%s` already exists in `%s`. Appending is not supported (yet).')

        if key in group:
            raise ValueError('Dict `%s` already exists in `%s`. Appending is not supported (yet).')

        temp_dict = {}
        for innerkey in data_to_store:
            val = data_to_store[innerkey]
            temp_dict[innerkey] = [val]

        # Convert dictionary to object table
        objtable = ObjectTable(data=temp_dict)

        # Then store the object table
        self._prm_write_into_pytable(key, objtable, group, fullname, **kwargs)

        new_table = group._f_get_child(key)

        # Remember that the Object Table represents a dictionary
        self._all_set_attributes_to_recall_natives(temp_dict, new_table,
                                                   HDF5StorageService.DATA_PREFIX)

        setattr(new_table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                HDF5StorageService.DICT)

        self._hdf5file.flush()


def _prm_write_pandas_data(self, key, data, group, fullname, flag, **kwargs):
        """Stores a pandas DataFrame into hdf5.

        :param key:

            Name of data item to store

        :param data:

            Pandas Data to Store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param flag:

            If it is a series, frame or panel

        """
        try:
            if 'filters' not in kwargs:
                filters = self._all_get_filters(kwargs)
                kwargs['filters'] = filters
            if 'format' not in kwargs:
                kwargs['format'] = self.pandas_format
            if 'encoding' not in kwargs:
                kwargs['encoding'] = self.encoding

            overwrite = kwargs.pop('overwrite', False)

            if key in group and not (overwrite or kwargs.get('append', False)):
                raise ValueError(
                    'DataFrame `%s` already exists in `%s`. '
                    'To append pass ``append=`True```.' % (key, fullname))
            else:
                self._logger.debug('Appending to pandas data `%s` in `%s`' % (key, fullname))

            if data is not None and (kwargs['format'] == 'f' or kwargs['format'] == 'fixed'):
                kwargs['expectedrows'] = data.shape[0]

            name = group._v_pathname + '/' + key
            self._hdf5store.put(name, data, **kwargs)
            self._hdf5store.flush()
            self._hdf5file.flush()

            frame_group = group._f_get_child(key)
            setattr(frame_group._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            self._hdf5file.flush()

        except:
            self._logger.error('Failed storing pandas data `%s` of `%s`.' % (key, fullname))
            raise


def _prm_write_into_other_array(self, key, data, group, fullname,
                                    flag, **kwargs):
        """Stores data as carray, earray or vlarray depending on `flag`.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        :param flag:

            How to store:
                CARRAY, EARRAY, VLARRAY

        """
        try:

            if flag == HDF5StorageService.CARRAY:
                factory = self._hdf5file.create_carray
            elif flag == HDF5StorageService.EARRAY:
                factory = self._hdf5file.create_earray
            elif flag == HDF5StorageService.VLARRAY:
                factory = self._hdf5file.create_vlarray
            else:
                raise RuntimeError('You shall not pass!')

            if key in group:
                raise ValueError(
                    'CArray `%s` already exists in `%s`. Appending is not supported (yet).')

            if 'filters' in kwargs:
                filters = kwargs.pop('filters')
            else:
                filters = self._all_get_filters(kwargs)

            try:
                other_array = factory(where=group, name=key, obj=data,
                                                filters=filters, **kwargs)
            except (ValueError, TypeError) as exc:
                try:
                    conv_data = data[:]
                    conv_data = np.core.defchararray.encode(conv_data, self.encoding)
                    other_array = factory(where=group, name=key,
                                                obj=conv_data,
                                                filters=filters, **kwargs)
                except Exception:
                    # Re-raise original Error
                    raise exc

            if data is not None:
                # Remember the types of the original data to recall them on loading
                self._all_set_attributes_to_recall_natives(data, other_array,
                                                       HDF5StorageService.DATA_PREFIX)
            setattr(other_array._v_attrs, HDF5StorageService.STORAGE_TYPE, flag)
            self._hdf5file.flush()
        except:
            self._logger.error('Failed storing %s `%s` of `%s`.' % (flag, key, fullname))
            raise

Stores data as carray earray or vlarray depending on flag .


def _prm_write_into_array(self, key, data, group, fullname, **kwargs):
        """Stores data as array.

        :param key:

            Name of data item to store

        :param data:

            Data to store

        :param group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        :param recall:

            If container type and data type for perfect recall should be stored

        """

        try:
            if key in group:
                raise ValueError(
                    'Array `%s` already exists in `%s`. Appending is not supported (yet).')

            try:

                array = self._hdf5file.create_array(where=group,
                                              name=key, obj=data, **kwargs)
            except (TypeError, ValueError) as exc:
                try:
                    if type(data) is dict and len(data) == 0:
                        # We cannot store an empty dictionary,
                        # but we can use an empty tuple as a dummy.
                        conv_data = ()
                    elif isinstance(data, str):
                        conv_data = data.encode(self._encoding)
                    elif isinstance(data, int):
                        conv_data = np.int64(data)
                    else:
                        conv_data = []
                        for string in data:
                            conv_data.append(string.encode(self._encoding))
                    array = self._hdf5file.create_array(where=group,
                                                  name=key, obj=conv_data, **kwargs)
                except Exception:
                    # Re-raise original error
                    raise exc

            if data is not None:
                # Remember the types of the original data to recall them on loading
                self._all_set_attributes_to_recall_natives(data, array,
                                                           HDF5StorageService.DATA_PREFIX)
            setattr(array._v_attrs, HDF5StorageService.STORAGE_TYPE,
                    HDF5StorageService.ARRAY)
            self._hdf5file.flush()
        except:
            self._logger.error('Failed storing array `%s` of `%s`.' % (key, fullname))
            raise


def _lnk_delete_link(self, link_name):
        """Removes a link from disk"""
        translated_name = '/' + self._trajectory_name + '/' + link_name.replace('.','/')
        link = self._hdf5file.get_node(where=translated_name)
        link._f_remove()

def _lnk_delete_link ( self , link_name ) : translated_name = '/' + self . _trajectory_name + '/' + link_name . replace ( '.' , '/' ) link = self . _hdf5file . get_node ( where = translated_name ) link . _f_remove ( )


def _all_delete_parameter_or_result_or_group(self, instance,
                                                 delete_only=None,
                                                 remove_from_item=False,
                                                 recursive=False,
                                                 _hdf5_group=None):
        """Removes a parameter or result or group from the hdf5 file.

        :param instance: Instance to be removed

        :param delete_only:

            List of elements if you only want to delete parts of a leaf node. Note that this
            needs to list the names of the hdf5 subnodes. BE CAREFUL if you erase parts of a leaf.
            Erasing partly happens at your own risk, it might be the case that you can
            no longer reconstruct the leaf from the leftovers!

        :param remove_from_item:

            If using `delete_only` and `remove_from_item=True` after deletion the data item is
            also removed from the `instance`.

        :param recursive:

            If a group node has children, you will can delete it if recursive is True.

        """
        split_name = instance.v_location.split('.')
        if _hdf5_group is None:
            where = '/' + self._trajectory_name + '/' + '/'.join(split_name)
            node_name = instance.v_name
            _hdf5_group = self._hdf5file.get_node(where=where, name=node_name)

        if delete_only is None:
            if instance.v_is_group and not recursive and len(_hdf5_group._v_children) != 0:
                    raise TypeError('You cannot remove the group `%s`, it has children, please '
                                    'use `recursive=True` to enforce removal.' %
                                    instance.v_full_name)
            _hdf5_group._f_remove(recursive=True)
        else:
            if not instance.v_is_leaf:
                raise ValueError('You can only choose `delete_only` mode for leafs.')

            if isinstance(delete_only, str):
                delete_only = [delete_only]

            for delete_item in delete_only:
                if (remove_from_item and
                        hasattr(instance, '__contains__') and
                        hasattr(instance, '__delattr__') and
                            delete_item in instance):
                    delattr(instance, delete_item)
                try:
                    _hdf5_sub_group = self._hdf5file.get_node(where=_hdf5_group,
                                                        name=delete_item)
                    _hdf5_sub_group._f_remove(recursive=True)
                except pt.NoSuchNodeError:
                    self._logger.warning('Could not delete `%s` from `%s`. HDF5 node not found!' %
                                         (delete_item, instance.v_full_name))

Removes a parameter or result or group from the hdf5 file .


def _prm_write_into_pytable(self, tablename, data, hdf5_group, fullname, **kwargs):
        """Stores data as pytable.

        :param tablename:

            Name of the data table

        :param data:

            Data to store

        :param hdf5_group:

            Group node where to store data in hdf5 file

        :param fullname:

            Full name of the `data_to_store`s original container, only needed for throwing errors.

        """

        datasize = data.shape[0]

        try:

            # Get a new pytables description from the data and create a new table
            description_dict, data_type_dict = self._prm_make_description(data, fullname)
            description_dicts = [{}]

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # For optimization we want to store the original data types into another table
                # and split the tables into several ones
                new_table_group = self._hdf5file.create_group(where=hdf5_group,
                                                    name=tablename,
                                                    filters=self._all_get_filters(kwargs.copy()))

                count = 0
                for innerkey in description_dict:
                    val = description_dict[innerkey]
                    if count == ptpa.MAX_COLUMNS:
                        description_dicts.append({})
                        count = 0
                    description_dicts[-1][innerkey] = val
                    count += 1

                setattr(new_table_group._v_attrs, HDF5StorageService.STORAGE_TYPE,
                        HDF5StorageService.TABLE)
                setattr(new_table_group._v_attrs, HDF5StorageService.SPLIT_TABLE, 1)

                hdf5_group = new_table_group
            else:
                description_dicts = [description_dict]

            for idx, descr_dict in enumerate(description_dicts):

                if idx == 0:
                    tblname = tablename
                else:
                    tblname = tablename + '_%d' % idx

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=datasize,
                                              filters=self._all_get_filters(kwargs.copy()))

                row = table.row
                for n in range(datasize):
                    # Fill the columns with data, note if the parameter was extended nstart!=0

                    for key in descr_dict:
                        row[key] = data[key][n]

                    row.append()

                # Remember the original types of the data for perfect recall
                if idx == 0 and len(description_dict) <= ptpa.MAX_COLUMNS:
                    # We only have a single table and
                    # we can store the original data types as attributes
                    for field_name in data_type_dict:
                        type_description = data_type_dict[field_name]
                        self._all_set_attr(table, field_name, type_description)

                    setattr(table._v_attrs, HDF5StorageService.STORAGE_TYPE,
                            HDF5StorageService.TABLE)

                table.flush()
                self._hdf5file.flush()

            if len(description_dict) > ptpa.MAX_COLUMNS:
                # We have potentially many split tables and the data types are
                # stored into an additional table for performance reasons
                tblname = tablename + '__' + HDF5StorageService.STORAGE_TYPE
                field_names, data_types = list(zip(*data_type_dict.items()))
                data_type_table_dict = {'field_name': field_names, 'data_type': data_types}
                descr_dict, _ = self._prm_make_description(data_type_table_dict, fullname)

                table = self._hdf5file.create_table(where=hdf5_group, name=tblname,
                                              description=descr_dict,
                                              title=tblname,
                                              expectedrows=len(field_names),
                                              filters=self._all_get_filters(kwargs))

                row = table.row

                for n in range(len(field_names)):
                    # Fill the columns with data

                    for key in data_type_table_dict:
                        row[key] = data_type_table_dict[key][n]

                    row.append()

                setattr(table._v_attrs, HDF5StorageService.DATATYPE_TABLE, 1)

                table.flush()
                self._hdf5file.flush()

        except:
            self._logger.error('Failed storing table `%s` of `%s`.' % (tablename, fullname))
            raise

def
_prm_write_into_pytable
(
self
,
tablename
,
data
,
hdf5_group
,
fullname
,
**
kwargs
)
:
datasize
=
data
.
shape
[
0
]
try
:
description_dict
,
data_type_dict
=
self
.
_prm_make_description
(
data
,
fullname
)
description_dicts
=
[
{
}
]
if
len
(
description_dict
)
>
ptpa
.
MAX_COLUMNS
:
new_table_group
=
self
.
_hdf5file
.
create_group
(
where
=
hdf5_group
,
name
=
tablename
,
filters
=
self
.
_all_get_filters
(
kwargs
.
copy
(
)
)
)
count
=
0
for
innerkey
in
description_dict
:
val
=
description_dict
[
innerkey
]
if
count
==
ptpa
.
MAX_COLUMNS
:
description_dicts
.
append
(
{
}
)
count
=
0
description_dicts
[
1
]
[
innerkey
]
=
val
count
+=
1
setattr
(
new_table_group
.
_v_attrs
,
HDF5StorageService
.
STORAGE_TYPE
,
HDF5StorageService
.
TABLE
)
setattr
(
new_table_group
.
_v_attrs
,
HDF5StorageService
.
SPLIT_TABLE
,
1
)
hdf5_group
=
new_table_group
else
:
description_dicts
=
[
description_dict
]
for
idx
,
descr_dict
in
enumerate
(
description_dicts
)
:
if
idx
==
0
:
tblname
=
tablename
else
:
tblname
=
tablename
+
'_%d'
%
idx
table
=
self
.
_hdf5file
.
create_table
(
where
=
hdf5_group
,
name
=
tblname
,
description
=
descr_dict
,
title
=
tblname
,
expectedrows
=
datasize
,
filters
=
self
.
_all_get_filters
(
kwargs
.
copy
(
)
)
)
row
=
table
.
row
for
n
in
range
(
datasize
)
:
for
key
in
descr_dict
:
row
[
key
]
=
data
[
key
]
[
n
]
row
.
append
(
)
if
idx
==
0
and
len
(
description_dict
)
<=
ptpa
.
MAX_COLUMNS
:
for
field_name
in
data_type_dict
:
type_description
=
data_type_dict
[
field_name
]
self
.
_all_set_attr
(
table
,
field_name
,
type_description
)
setattr
(
table
.
_v_attrs
,
HDF5StorageService
.
STORAGE_TYPE
,
HDF5StorageService
.
TABLE
)
table
.
flush
(
)
self
.
_hdf5file
.
flush
(
)
if
len
(
description_dict
)
>
ptpa
.
MAX_COLUMNS
:
tblname
=
tablename
+
'__'
+
HDF5StorageService
.
STORAGE_TYPE
field_names
,
data_types
=
list
(
zip
(
*
data_type_dict
.
items
(
)
)
)
data_type_table_dict
=
{
'field_name'
:
field_names
,
'data_type'
:
data_types
}
descr_dict
,
_
=
self
.
_prm_make_description
(
data_type_table_dict
,
fullname
)
table
=
self
.
_hdf5file
.
create_table
(
where
=
hdf5_group
,
name
=
tblname
,
description
=
descr_dict
,
title
=
tblname
,
expectedrows
=
len
(
field_names
)
,
filters
=
self
.
_all_get_filters
(
kwargs
)
)
row
=
table
.
row
for
n
in
range
(
len
(
field_names
)
)
:
for
key
in
data_type_table_dict
:
row
[
key
]
=
data_type_table_dict
[
key
]
[
n
]
row
.
append
(
)
setattr
(
table
.
_v_attrs
,
HDF5StorageService
.
DATATYPE_TABLE
,
1
)
table
.
flush
(
)
self
.
_hdf5file
.
flush
(
)
except
:
self
.
_logger
.
error
(
'Failed storing table `%s` of `%s`.'
%
(
tablename
,
fullname
)
)
raise


def _prm_make_description(self, data, fullname):
        """ Returns a description dictionary for pytables table creation"""

        def _convert_lists_and_tuples(series_of_data):
            """Converts lists and tuples to numpy arrays"""
            if isinstance(series_of_data[0],
                          (list, tuple)):  # and not isinstance(series_of_data[0], np.ndarray):
                # If the first data item is a list, the rest must be as well, since
                # data has to be homogeneous
                for idx, item in enumerate(series_of_data):
                    series_of_data[idx] = np.array(item)

        descriptiondict = {}  # dictionary containing the description to build a pytables table
        original_data_type_dict = {}  # dictionary containing the original data types

        for key in data:
            val = data[key]

            # remember the original data types
            self._all_set_attributes_to_recall_natives(val[0], PTItemMock(original_data_type_dict),
                                                       HDF5StorageService.FORMATTED_COLUMN_PREFIX %
                                                       key)

            _convert_lists_and_tuples(val)

            # get a pytables column from the data
            col = self._all_get_table_col(key, val, fullname)

            descriptiondict[key] = col

        return descriptiondict, original_data_type_dict

def _prm_make_description ( self , data , fullname ) : def _convert_lists_and_tuples ( series_of_data ) : if isinstance ( series_of_data [ 0 ] , ( list , tuple ) ) : for idx , item in enumerate ( series_of_data ) : series_of_data [ idx ] = np . array ( item ) descriptiondict = { } original_data_type_dict = { } for key in data : val = data [ key ] self . _all_set_attributes_to_recall_natives ( val [ 0 ] , PTItemMock ( original_data_type_dict ) , HDF5StorageService . FORMATTED_COLUMN_PREFIX % key ) _convert_lists_and_tuples ( val ) col = self . _all_get_table_col ( key , val , fullname ) descriptiondict [ key ] = col return descriptiondict , original_data_type_dict


def _all_get_table_col(self, key, column, fullname):
        """ Creates a pytables column instance.

        The type of column depends on the type of `column[0]`.
        Note that data in `column` must be homogeneous!

        """
        val = column[0]

        try:

            # # We do not want to loose int_
            if type(val) is int:
                return pt.IntCol()

            if isinstance(val, (str, bytes)):
                itemsize = int(self._prm_get_longest_stringsize(column))
                return pt.StringCol(itemsize)

            if isinstance(val, np.ndarray):
                if (np.issubdtype(val.dtype, str) or
                        np.issubdtype(val.dtype, bytes)):
                    itemsize = int(self._prm_get_longest_stringsize(column))
                    return pt.StringCol(itemsize, shape=val.shape)
                else:
                    return pt.Col.from_dtype(np.dtype((val.dtype, val.shape)))
            else:
                return pt.Col.from_dtype(np.dtype(type(val)))
        except Exception:
            self._logger.error('Failure in storing `%s` of Parameter/Result `%s`.'
                               ' Its type was `%s`.' % (key, fullname, repr(type(val))))
            raise


def _prm_get_longest_stringsize(string_list):
        """ Returns the longest string size for a string entry across data."""
        maxlength = 1

        for stringar in string_list:
            if isinstance(stringar, np.ndarray):
                if stringar.ndim > 0:
                    for string in stringar.ravel():
                        maxlength = max(len(string), maxlength)
                else:
                    maxlength = max(len(stringar.tolist()), maxlength)
            else:
                maxlength = max(len(stringar), maxlength)

        # Make the string Col longer than needed in order to allow later on slightly larger strings
        return int(maxlength * 1.5)

def _prm_get_longest_stringsize ( string_list ) : maxlength = 1 for stringar in string_list : if isinstance ( stringar , np . ndarray ) : if stringar . ndim > 0 : for string in stringar . ravel ( ) : maxlength = max ( len ( string ) , maxlength ) else : maxlength = max ( len ( stringar . tolist ( ) ) , maxlength ) else : maxlength = max ( len ( stringar ) , maxlength ) return int ( maxlength * 1.5 )

Returns the longest string size for a string entry across data .


def _prm_load_into_dict(self, full_name, load_dict, hdf5_group, instance,
                            load_only, load_except, load_flags, _prefix = ''):
        """Loads into dictionary"""
        for node in hdf5_group:

            load_type = self._all_get_from_attrs(node, HDF5StorageService.STORAGE_TYPE)

            if _prefix:
                load_name = '%s.%s' % (_prefix, node._v_name)
            else:
                load_name = node._v_name

            if load_type == HDF5StorageService.NESTED_GROUP:
                self._prm_load_into_dict(full_name=full_name,
                                         load_dict=load_dict,
                                         hdf5_group=node,
                                         instance=instance,
                                         load_only=load_only,
                                         load_except=load_except,
                                         load_flags=load_flags,
                                         _prefix=load_name)
                continue

            if load_only is not None:

                if load_name not in load_only:
                    continue
                else:
                    load_only.remove(load_name)

            elif load_except is not None:
                if load_name in load_except:
                    load_except.remove(load_name)
                    continue

            # Recall from the hdf5 node attributes how the data was stored and reload accordingly
            if load_name in load_flags:
                load_type = load_flags[load_name]

            if load_type == HDF5StorageService.DICT:
                to_load = self._prm_read_dictionary(node, full_name)
            elif load_type == HDF5StorageService.TABLE:
                to_load = self._prm_read_table(node, full_name)
            elif load_type in (HDF5StorageService.ARRAY, HDF5StorageService.CARRAY,
                                HDF5StorageService.EARRAY, HDF5StorageService.VLARRAY):
                to_load = self._prm_read_array(node, full_name)
            elif load_type in (HDF5StorageService.FRAME,
                               HDF5StorageService.SERIES,
                               #  HDF5StorageService.PANEL
                               ):
                to_load = self._prm_read_pandas(node, full_name)
            elif load_type.startswith(HDF5StorageService.SHARED_DATA):
                to_load = self._prm_read_shared_data(node, instance)
            else:
                raise pex.NoSuchServiceError('Cannot load %s, do not understand the hdf5 file '
                                             'structure of %s [%s].' %
                                             (full_name, str(node), str(load_type)))

            if to_load is None:
                raise RuntimeError('You shall not pass!')

            load_dict[load_name] = to_load


def _prm_read_dictionary(self, leaf, full_name):
        """Loads data that was originally a dictionary when stored

        :param leaf:

            PyTables table containing the dictionary data

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded

        """
        try:
            # Load as Pbject Table
            temp_table = self._prm_read_table(leaf, full_name)
            # Turn the ObjectTable into a dictionary of lists (with length 1).
            temp_dict = temp_table.to_dict('list')

            innder_dict = {}

            # Turn the dictionary of lists into a normal dictionary
            for innerkey, vallist in temp_dict.items():
                innder_dict[innerkey] = vallist[0]

            return innder_dict
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (leaf._v_name, full_name))
            raise

def
_prm_read_dictionary
(
self
,
leaf
,
full_name
)
:
try
:
temp_table
=
self
.
_prm_read_table
(
leaf
,
full_name
)
temp_dict
=
temp_table
.
to_dict
(
'list'
)
innder_dict
=
{
}
for
innerkey
,
vallist
in
temp_dict
.
items
(
)
:
innder_dict
[
innerkey
]
=
vallist
[
0
]
return
innder_dict
except
:
self
.
_logger
.
error
(
'Failed loading `%s` of `%s`.'
%
(
leaf
.
_v_name
,
full_name
)
)
raise


def _prm_read_shared_data(self, shared_node, instance):
        """Reads shared data and constructs the appropraite class.

        :param shared_node:

            hdf5 node storing the pandas DataFrame

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load

        """
        try:
            data_type = self._all_get_from_attrs(shared_node,
                                                 HDF5StorageService.SHARED_DATA_TYPE)
            constructor = shared.FLAG_CLASS_MAPPING[data_type]
            name = shared_node._v_name
            result = constructor(name=name, parent=instance)
            return result
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (shared_node._v_name,
                                                                 instance.v_full_name))
            raise

def
_prm_read_shared_data
(
self
,
shared_node
,
instance
)
:
try
:
data_type
=
self
.
_all_get_from_attrs
(
shared_node
,
HDF5StorageService
.
SHARED_DATA_TYPE
)
constructor
=
shared
.
FLAG_CLASS_MAPPING
[
data_type
]
name
=
shared_node
.
_v_name
result
=
constructor
(
name
=
name
,
parent
=
instance
)
return
result
except
:
self
.
_logger
.
error
(
'Failed loading `%s` of `%s`.'
%
(
shared_node
.
_v_name
,
instance
.
v_full_name
)
)
raise

Reads shared data and constructs the appropraite class .


def _prm_read_table(self, table_or_group, full_name):
        """Reads a non-nested PyTables table column by column and created a new ObjectTable for
        the loaded data.

        :param table_or_group:

            PyTables table to read from or a group containing subtables.

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to be loaded

        """
        try:
            result_table = None

            if self._all_get_from_attrs(table_or_group, HDF5StorageService.SPLIT_TABLE):
                table_name = table_or_group._v_name

                data_type_table_name = table_name + '__' + HDF5StorageService.STORAGE_TYPE

                data_type_table = table_or_group._v_children[data_type_table_name]
                data_type_dict = {}

                for row in data_type_table:
                    fieldname = row['field_name'].decode('utf-8')
                    data_type_dict[fieldname] = row['data_type'].decode('utf-8')

                for sub_table in table_or_group:
                    sub_table_name = sub_table._v_name

                    if sub_table_name == data_type_table_name:
                        continue

                    for colname in sub_table.colnames:
                        # Read Data column by column
                        col = sub_table.col(colname)
                        data_list = list(col)

                        prefix = HDF5StorageService.FORMATTED_COLUMN_PREFIX % colname
                        for idx, data in enumerate(data_list):
                            # Recall original type of data
                            data, type_changed = self._all_recall_native_type(data,
                                                                              PTItemMock(
                                                                                data_type_dict),
                                                                              prefix)
                            if type_changed:
                                data_list[idx] = data
                            else:
                                break

                        # Construct or insert into an ObjectTable
                        if result_table is None:
                            result_table = ObjectTable(data={colname: data_list})
                        else:
                            result_table[colname] = data_list

            else:
                for colname in table_or_group.colnames:
                    # Read Data column by column
                    col = table_or_group.col(colname)
                    data_list = list(col)

                    prefix = HDF5StorageService.FORMATTED_COLUMN_PREFIX % colname
                    for idx, data in enumerate(data_list):
                        # Recall original type of data
                        data, type_changed = self._all_recall_native_type(data, table_or_group,
                                                                          prefix)
                        if type_changed:
                            data_list[idx] = data
                        else:
                            break

                    # Construct or insert into an ObjectTable
                    if result_table is None:
                        result_table = ObjectTable(data={colname: data_list})
                    else:
                        result_table[colname] = data_list

            return result_table
        except:
            self._logger.error(
                'Failed loading `%s` of `%s`.' % (table_or_group._v_name, full_name))
            raise

def
_prm_read_table
(
self
,
table_or_group
,
full_name
)
:
try
:
result_table
=
if
self
.
_all_get_from_attrs
(
table_or_group
,
HDF5StorageService
.
SPLIT_TABLE
)
:
table_name
=
table_or_group
.
_v_name
data_type_table_name
=
table_name
+
'__'
+
HDF5StorageService
.
STORAGE_TYPE
data_type_table
=
table_or_group
.
_v_children
[
data_type_table_name
]
data_type_dict
=
{
}
for
row
in
data_type_table
:
fieldname
=
row
[
'field_name'
]
.
decode
(
'utf-8'
)
data_type_dict
[
fieldname
]
=
row
[
'data_type'
]
.
decode
(
'utf-8'
)
for
sub_table
in
table_or_group
:
sub_table_name
=
sub_table
.
_v_name
if
sub_table_name
==
data_type_table_name
:
continue
for
colname
in
sub_table
.
colnames
:
col
=
sub_table
.
col
(
colname
)
data_list
=
list
(
col
)
prefix
=
HDF5StorageService
.
FORMATTED_COLUMN_PREFIX
%
colname
for
idx
,
data
in
enumerate
(
data_list
)
:
data
,
type_changed
=
self
.
_all_recall_native_type
(
data
,
PTItemMock
(
data_type_dict
)
,
prefix
)
if
type_changed
:
data_list
[
idx
]
=
data
else
:
break
if
result_table
is
:
result_table
=
ObjectTable
(
data
=
{
colname
:
data_list
}
)
else
:
result_table
[
colname
]
=
data_list
else
:
for
colname
in
table_or_group
.
colnames
:
col
=
table_or_group
.
col
(
colname
)
data_list
=
list
(
col
)
prefix
=
HDF5StorageService
.
FORMATTED_COLUMN_PREFIX
%
colname
for
idx
,
data
in
enumerate
(
data_list
)
:
data
,
type_changed
=
self
.
_all_recall_native_type
(
data
,
table_or_group
,
prefix
)
if
type_changed
:
data_list
[
idx
]
=
data
else
:
break
if
result_table
is
:
result_table
=
ObjectTable
(
data
=
{
colname
:
data_list
}
)
else
:
result_table
[
colname
]
=
data_list
return
result_table
except
:
self
.
_logger
.
error
(
'Failed loading `%s` of `%s`.'
%
(
table_or_group
.
_v_name
,
full_name
)
)
raise

Reads a non - nested PyTables table column by column and created a new ObjectTable for the loaded data .


def _prm_read_array(self, array, full_name):
        """Reads data from an array or carray

        :param array:

            PyTables array or carray to read from

        :param full_name:

            Full name of the parameter or result whose data is to be loaded

        :return:

            Data to load

        """
        try:
            result = self._svrc_read_array(array)
            # Recall original data types
            result, dummy = self._all_recall_native_type(result, array,
                                                         HDF5StorageService.DATA_PREFIX)

            return result
        except:
            self._logger.error('Failed loading `%s` of `%s`.' % (array._v_name, full_name))
            raise

def
_prm_read_array
(
self
,
array
,
full_name
)
:
try
:
result
=
self
.
_svrc_read_array
(
array
)
result
,
dummy
=
self
.
_all_recall_native_type
(
result
,
array
,
HDF5StorageService
.
DATA_PREFIX
)
return
result
except
:
self
.
_logger
.
error
(
'Failed loading `%s` of `%s`.'
%
(
array
.
_v_name
,
full_name
)
)
raise


def load_trajectory(name=None,
                    index=None,
                    as_new=False,
                    load_parameters=pypetconstants.LOAD_DATA,
                    load_derived_parameters=pypetconstants.LOAD_SKELETON,
                    load_results=pypetconstants.LOAD_SKELETON,
                    load_other_data=pypetconstants.LOAD_SKELETON,
                    recursive=True,
                    load_data=None,
                    max_depth=None,
                    force=False,
                    dynamic_imports=None,
                    new_name='my_trajectory',
                    add_time=True,
                    wildcard_functions=None,
                    with_run_information=True,
                    storage_service=storage.HDF5StorageService,
                    **kwargs):
    """Helper function that creates a novel trajectory and loads it from disk.

    For the parameters see :func:`~pypet.trajectory.Trajectory.f_load`.

    ``new_name`` and ``add_time`` are only used in case ``as_new`` is ``True``.
    Accordingly, they determine the new name of trajectory.

    """
    if name is None and index is None:
        raise ValueError('Please specify either a name or an index')
    elif name is not None and index is not None:
        raise ValueError('Please specify either a name or an index')

    traj = Trajectory(name=new_name, add_time=add_time, dynamic_imports=dynamic_imports,
                      wildcard_functions=wildcard_functions)
    traj.f_load(name=name, index=index, as_new=as_new, load_parameters=load_parameters,
                load_derived_parameters=load_derived_parameters, load_results=load_results,
                load_other_data=load_other_data, recursive=recursive, load_data=load_data,
                max_depth=max_depth, force=force, with_run_information=with_run_information,
                storage_service=storage_service, **kwargs)
    return traj

Helper function that creates a novel trajectory and loads it from disk .


def make_set_name(idx):
    """Creates a run set name based on ``idx``"""
    GROUPSIZE = 1000
    set_idx = idx // GROUPSIZE
    if set_idx >= 0:
        return pypetconstants.FORMATTED_SET_NAME % set_idx
    else:
        return pypetconstants.SET_NAME_DUMMY

def make_set_name ( idx ) : GROUPSIZE = 1000 set_idx = idx // GROUPSIZE if set_idx >= 0 : return pypetconstants . FORMATTED_SET_NAME % set_idx else : return pypetconstants . SET_NAME_DUMMY


def f_set_properties(self, **kwargs):
        """Sets properties like ``v_fast_access``.

        For example: ``traj.f_set_properties(v_fast_access=True, v_auto_load=False)``

        """
        for name in kwargs:
            val = kwargs[name]
            if not name.startswith('v_'):
                name = 'v_' + name
            if not name in self._nn_interface._not_admissible_names:
                raise AttributeError('Cannot set property `%s` does not exist.' % name)
            else:
                setattr(self, name, val)


def f_add_to_dynamic_imports(self, dynamic_imports):
        """Adds classes or paths to classes to the trajectory to create custom parameters.

        :param dynamic_imports:

            If you've written custom parameter that needs to be loaded dynamically during runtime,
            this needs to be specified here as a list of classes or strings naming classes
            and there module paths. For example:
            `dynamic_imports = ['pypet.parameter.PickleParameter',MyCustomParameter]`

            If you only have a single class to import, you do not need the list brackets:
            `dynamic_imports = 'pypet.parameter.PickleParameter'`

        """

        if not isinstance(dynamic_imports, (list, tuple)):
            dynamic_imports = [dynamic_imports]

        for item in dynamic_imports:
            if not (isinstance(item, str) or inspect.isclass(item)):
                raise TypeError('Your dynamic import `%s` is neither a class nor a string.' %
                                str(item))

        self._dynamic_imports.extend(dynamic_imports)

Adds classes or paths to classes to the trajectory to create custom parameters .


def f_set_crun(self, name_or_idx):
        """Can make the trajectory behave as during a particular single run.

        It allows easier data analysis.

         Has the following effects:

        *
            `v_idx` and `v_crun` are set to the appropriate index and run name

        *
            All explored parameters are set to the corresponding value in the exploration
            ranges, i.e. when you call :func:`~pypet.parameter.Parameter.f_get` (or fast access)
            on them you will get in return the value at the corresponding `v_idx` position
            in the exploration range.

        *
            If you perform a search in the trajectory tree, the trajectory will
            only search the run subtree under *results* and *derived_parameters* with the
            corresponding index.
            For instance, if you use `f_set_crun('run_00000007')` or `f_set_crun(7)`
            and search for `traj.results.z` this will search for `z` only in the subtree
            `traj.results.run_00000007`. Yet, you can still explicitly name other subtrees,
            i.e. `traj.results.run_00000004.z` will still work.

        """
        if (name_or_idx is None or name_or_idx == self.f_wildcard('$', -1) or
                    name_or_idx == -1):
            self.f_restore_default()
        else:
            if isinstance(name_or_idx, str):
                self._idx = self.f_idx_to_run(name_or_idx)
                self._crun = name_or_idx
            else:
                self._crun = self.f_idx_to_run(name_or_idx)
                self._idx = name_or_idx

            self._set_explored_parameters_to_idx(self.v_idx)

def f_set_crun ( self , name_or_idx ) : if ( name_or_idx is None or name_or_idx == self . f_wildcard ( '$' , - 1 ) or name_or_idx == - 1 ) : self . f_restore_default ( ) else : if isinstance ( name_or_idx , str ) : self . _idx = self . f_idx_to_run ( name_or_idx ) self . _crun = name_or_idx else : self . _crun = self . f_idx_to_run ( name_or_idx ) self . _idx = name_or_idx self . _set_explored_parameters_to_idx ( self . v_idx )

Can make the trajectory behave as during a particular single run .


def f_iter_runs(self, start=0, stop=None, step=1, yields='name'):
        """Makes the trajectory iterate over all runs.

        :param start: Start index of run

        :param stop: Stop index, leave ``None`` for length of trajectory

        :param step: Stepsize

        :param yields:

            What should be yielded: ``'name'`` of run, ``idx`` of run
            or ``'self'`` to simply return the trajectory container.

            You can also pick ``'copy'`` to get **shallow** copies (ie the tree is copied but
            no leave nodes except explored ones.) of your trajectory,
            might lead to some of overhead.

        Note that after a full iteration, the trajectory is set back to normal.

        Thus, the following code snippet

        ::

            for run_name in traj.f_iter_runs():

                 # Do some stuff here...

        is equivalent to

        ::

            for run_name in traj.f_get_run_names(sort=True):
                traj.f_set_crun(run_name)

                # Do some stuff here...

            traj.f_set_crun(None)

        :return:

            Iterator over runs. The iterator itself will return the run names but modify
            the trajectory in each iteration and set it back do normal in the end.

        """
        if stop is None:
            stop =len(self)
        elif stop > len(self):
            raise ValueError('Stop cannot be larger than the trajectory lenght.')
        yields = yields.lower()
        if yields == 'name':
            yield_func = lambda x: self.f_idx_to_run(x)
        elif yields == 'idx':
            yield_func = lambda x: x
        elif yields == 'self':
            yield_func = lambda x: self
        elif yields == 'copy':
            yield_func = lambda x: self.__copy__()
        else:
            raise ValueError('Please choose yields among: `name`, `idx`, or `self`.')
        for idx in range(start, stop, step):
            self.f_set_crun(idx)
            yield yield_func(idx)

        self.f_set_crun(None)

Makes the trajectory iterate over all runs .