Skip to content
This repository was archived by the owner on Nov 6, 2025. It is now read-only.

Provide origin info at the channel level in h5 files #33

Merged
seb5g merged 4 commits into from
Mar 7, 2025
Merged

Provide origin info at the channel level in h5 files #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9c694db
Provide origin info at the CHXX level in h5 files
malik-irain Mar 6, 2025
7cfb7a3
Update docstrings
malik-irain Mar 6, 2025
55f1ecb
update metadata with kwargs in DataSaverLoader.add_data instead of pa…
malik-irain Mar 7, 2025
bc93cb3
kwargs is a dict
malik-irain Mar 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion src/pymodaq_data/h5modules/backends.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,7 @@
compression = 'gzip'
self.compression = dict(compression=compression, compression_opts=compression_opts)

def get_set_group(self, where, name, title=''):
def get_set_group(self, where, name, title='', **kwargs):
"""Retrieve or create (if absent) a node group
Get attributed to the class attribute ``current_group``

Expand All @@ -670,20 +670,29 @@
title: str
node title

Keyword Arguments:
any other metadata related to this node (for example: origin)
Returns
-------
group: group node
"""

if isinstance(where, Node):
where = where.node

if name not in list(self.get_children(where)):
if self.backend == 'tables':
group = self._h5file.create_group(where, name, title)
for key, value in kwargs.items():
if not hasattr(group._v_attrs, key):
Copy link
Contributor

@seb5g seb5g Mar 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're using a protected attribute specific to one hdf5 backend (._v_attrs), you should use the attrs property instead I introduced in h5backend

Copy link
Contributor Author

@malik-irain malik-irain Mar 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if the backend is tables the group is of type tables.group.Group (defined in pytables) and doesn't have a attrs property. Even if I add one, it will not be saved. Also, It is backed by the documentation:

Setting and getting user attributes

PyTables provides an easy and concise way to complement the meaning of your node objects on the tree by using the AttributeSet class (see The AttributeSet class). You can access this object through the standard attribute attrs in Leaf nodes and _v_attrs in Group nodes.

group._v_attrs[key] = value
else:
group = self.get_node(where).node.create_group(name)
group.attrs['TITLE'] = title
group.attrs['CLASS'] = 'GROUP'
for key, value in kwargs.items():
if not hasattr(group.attrs, key):
group.attrs[key] = value

Check warning on line 695 in src/pymodaq_data/h5modules/backends.py

View check run for this annotation

Codecov / codecov/patch

src/pymodaq_data/h5modules/backends.py#L694-L695 

Added lines #L694 - L695 were not covered by tests

else:
group = self.get_node(where, name)
Expand Down
30 changes: 14 additions & 16 deletions src/pymodaq_data/h5modules/data_saving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ def add_data(self, where: Union[Node, str], data: DataWithAxes, save_axes=True,
units=data.units,
nav_indexes=tuple(data.nav_indexes)
if data.nav_indexes is not None else None,)
metadata.update(kwargs)
for name in data.extra_attributes:
metadata[name] = getattr(data, name)
self._h5saver.add_array(where, self._get_next_node_name(where), self.data_type,
Expand Down Expand Up @@ -788,8 +789,7 @@ def channel_formatter(ind: int):
and formatted as below"""
return f'CH{ind:02d}'

def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml='',
metadata=None, **kwargs):
def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml='', **kwargs):
"""

Parameters
Expand All @@ -799,19 +799,18 @@ def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml=
data: DataToExport
settings_as_xml: str
The settings parameter as an XML string
metadata: dict
Keyword Arguments:
all extra metadata to be saved in the group node where data will be saved

"""
if metadata is None:
metadata = {}

dims = data.get_dim_presents()
for dim in dims:
dim_group = self._h5saver.get_set_group(where, dim)
for ind, dwa in enumerate(data.get_data_from_dim(dim)):
# dwa: DataWithAxes filtered by dim
dwa_group = self._h5saver.get_set_group(dim_group, self.channel_formatter(ind),
dwa.name)
dwa.name, origin=dwa.origin)
# dwa_group = self._h5saver.add_ch_group(dim_group, dwa.name)
self._data_saver.add_data(dwa_group, dwa, **kwargs)

Expand Down Expand Up @@ -881,7 +880,7 @@ def __init__(self, h5saver: H5SaverLowLevel,
def add_data(self, where: Union[Node, str], data: DataToExport,
axis_values: List[Union[float, np.ndarray]] = None,
axis_value: Union[float, np.ndarray] = None,
settings_as_xml='', metadata=None, **kwargs
settings_as_xml='', **kwargs
):
"""

Expand All @@ -897,14 +896,14 @@ def add_data(self, where: Union[Node, str], data: DataToExport,
The next value (or values) of the enlarged axis
settings_as_xml: str
The settings parameter as an XML string
metadata: dict
Keyword Arguments:
all extra metadata to be saved in the group node where data will be saved
"""

if axis_values is None and axis_value is not None:
axis_values = [axis_value]

super().add_data(where, data, settings_as_xml, metadata, **kwargs)
super().add_data(where, data, settings_as_xml, **kwargs)
# a parent navigation group (same for all data nodes)

where = self._get_node(where)
Expand Down Expand Up @@ -937,10 +936,8 @@ class DataToExportTimedSaver(DataToExportEnlargeableSaver):
def __init__(self, h5saver: H5SaverLowLevel):
super().__init__(h5saver, enl_axis_names=('time',), enl_axis_units=('s',))

def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml='',
metadata=None, **kwargs):
super().add_data(where, data, axis_values=[data.timestamp], settings_as_xml=settings_as_xml,
metadata=metadata)
def add_data(self, where: Union[Node, str], data: DataToExport, settings_as_xml='', **kwargs):
super().add_data(where, data, axis_values=[data.timestamp], settings_as_xml=settings_as_xml, **kwargs)


class DataToExportExtendedSaver(DataToExportSaver):
Expand Down Expand Up @@ -975,7 +972,8 @@ def add_nav_axes(self, where: Union[Node, str], axes: List[Axis]):

def add_data(self, where: Union[Node, str], data: DataToExport, indexes: Iterable[int],
distribution=DataDistribution['uniform'],
settings_as_xml='', metadata={}):
settings_as_xml='', **kwargs):

"""

Parameters
Expand All @@ -988,7 +986,7 @@ def add_data(self, where: Union[Node, str], data: DataToExport, indexes: Iterabl
extended_shape and with values coherent with this shape
settings_as_xml: str
The settings parameter as an XML string
metadata: dict
Keyword Arguments:
all extra metadata to be saved in the group node where data will be saved

"""
Expand All @@ -998,7 +996,7 @@ def add_data(self, where: Union[Node, str], data: DataToExport, indexes: Iterabl
for ind, dwa in enumerate(data.get_data_from_dim(dim)):
# dwa: DataWithAxes filtered by dim
dwa_group = self._h5saver.get_set_group(dim_group,
self.channel_formatter(ind), dwa.name)
self.channel_formatter(ind), dwa.name, origin=dwa.origin)
self._data_saver.add_data(dwa_group, dwa, indexes=indexes,
distribution=distribution)

Expand Down
5 changes: 3 additions & 2 deletions src/pymodaq_data/h5modules/saving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,12 +253,13 @@ def add_array(self, where: Union[GROUP, str], name: str, data_type: DataType, ar
self.set_attr(array, metadat, metadata[metadat])
return array

def get_set_group(self, where, name, title=''):
def get_set_group(self, where, name, title='', **kwargs):
"""Get the group located at where if it exists otherwise creates it

This also set the _current_group property
"""
self._current_group = super().get_set_group(where, name, title)

self._current_group = super().get_set_group(where, name, title, **kwargs)
return self._current_group

def get_groups(self, where: Union[str, GROUP], group_type: GroupType):
Expand Down