Upgrade to Python >3.9, numpy >2.0, support SNIRF 1.2.0-development

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       max-parallel: 5
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
     defaults:
       run:
         shell: bash -el {0}

README.md

@@ -22,7 +22,7 @@ Developed and maintained by the [Boston University Neurophotonics Center](https:
 ## Installation
 `pip install snirf`
 
-pysnirf2 requires Python > 3.6.
+pysnirf2 requires Python > 3.9.
 
 # Features
 

docs/README.md

@@ -16,13 +16,14 @@
 - [`pysnirf2.IndexedGroup`](./pysnirf2.md#class-indexedgroup)
 - [`pysnirf2.MeasurementList`](./pysnirf2.md#class-measurementlist): Interface for indexed group `MeasurementList`.
 - [`pysnirf2.MeasurementListElement`](./pysnirf2.md#class-measurementlistelement): Wrapper for an element of indexed group `MeasurementList`.
+- [`pysnirf2.MeasurementLists`](./pysnirf2.md#class-measurementlists)
 - [`pysnirf2.MetaDataTags`](./pysnirf2.md#class-metadatatags)
 - [`pysnirf2.Nirs`](./pysnirf2.md#class-nirs): Interface for indexed group `Nirs`.
 - [`pysnirf2.NirsElement`](./pysnirf2.md#class-nirselement): Wrapper for an element of indexed group `Nirs`.
 - [`pysnirf2.Probe`](./pysnirf2.md#class-probe)
 - [`pysnirf2.Snirf`](./pysnirf2.md#class-snirf)
 - [`pysnirf2.SnirfConfig`](./pysnirf2.md#class-snirfconfig): Structure containing Snirf-wide data and settings.
-- [`pysnirf2.SnirfFormatError`](./pysnirf2.md#class-snirfformaterror): Raised when SNIRF-specific error prevents file from loading properly.
+- [`pysnirf2.SnirfFormatError`](./pysnirf2.md#class-snirfformaterror): Raised when SNIRF-specific error prevents file from loading or saving properly.
 - [`pysnirf2.Stim`](./pysnirf2.md#class-stim)
 - [`pysnirf2.StimElement`](./pysnirf2.md#class-stimelement)
 - [`pysnirf2.ValidationIssue`](./pysnirf2.md#class-validationissue): Information about the validity of a given SNIRF file location.

gen/README.md

@@ -14,5 +14,5 @@ This ensures easy maintenance of the project as the specification develops.
 
 1. Ensure that [data.py](https://github.com/BUNPC/pysnirf2/blob/main/gen/data.py) contains correct data to parse the latest spec. Make sure `SPEC_SRC` and `VERSION` are up to date.
 2. IMPORTANT! Back up or commit local changes to the code via git. The generation process may delete your changes.
-3. Using a Python > 3.6 environment equipped with [gen/requirements.txt](https://github.com/BUNPC/pysnirf2/blob/main/gen/requirements.txt), run [gen.py](https://github.com/BUNPC/pysnirf2/blob/main/gen/gen.py) from the project root
+3. Using a Python > 3.9 environment equipped with [gen/requirements.txt](https://github.com/BUNPC/pysnirf2/blob/main/gen/requirements.txt), run [gen.py](https://github.com/BUNPC/pysnirf2/blob/main/gen/gen.py) from the project root
 4. Test the resulting library

gen/data.py

@@ -1,5 +1,5 @@
-SPEC_SRC = 'https://raw.githubusercontent.com/fNIRS/snirf/v1.1/snirf_specification.md'
-SPEC_VERSION = 'v1.1'  # Version of the spec linked above
+SPEC_SRC = 'https://raw.githubusercontent.com/sstucker/snirf/refs/heads/master/snirf_specification.md'
+SPEC_VERSION = '1.2-development'  # Version of the spec linked above
 
 """
 These types are fragments of the string codes used to describe the types of
@@ -38,6 +38,15 @@
 DEFINITIONS_DELIM_START = '### SNIRF data container definitions'
 DEFINITIONS_DELIM_END = '## Appendix'
 
+DATA_TYPE_DELIM_START = '### Supported `measurementList(k).dataType` values in `dataTimeSeries`'
+DATA_TYPE_DELIM_END = '### Supported `measurementList(k).dataTypeLabel` values in `dataTimeSeries`'
+
+DATA_TYPE_LABEL_TABLE_START = '### Supported `measurementList(k).dataTypeLabel` values in `dataTimeSeries`'
+DATA_TYPE_LABEL_TABLE_END = '### Supported `/nirs(i)/aux(j)/name` values'
+
+AUX_NAME_TABLE_START = '### Supported `/nirs(i)/aux(j)/name` values'
+AUX_NAME_TABLE_END = '### Examples of stimulus waveforms'
+
 # -- BIDS Probe name identifiers ---------------------------------------------
 
 BIDS_PROBE_NAMES = ['ICBM452AirSpace',

gen/gen.py

@@ -7,15 +7,15 @@
 import getpass
 import os
 import sys
-import warnings
+import re
 from pylint import lint
 
 """
 Generates SNIRF interface and validator from the summary table of the specification
 hosted at SPEC_SRC.
 """
 
-LIB_VERSION = '0.8.2'  # Version for this script
+LIB_VERSION = '0.9.2'  # Version for this script
 
 if __name__ == '__main__':
 
@@ -38,7 +38,7 @@
 
     local_spec = SPEC_SRC.split('/')[-1].split('.')[0] + '_retrieved_' + datetime.now().strftime('%d_%m_%y') + '.txt'
 
-    if os.path.exists(local_spec):
+    if os.path.exists(local_spec) and input('Use local specification document ' + local_spec + '? y/n\n') == 'y':
         print('Loading specification from local document', local_spec, '...')
         with open(local_spec, 'r') as f:
             text = f.read()
@@ -87,7 +87,7 @@
             # Get name: format pairs for each name
             if len(name) > 1:  # Skip the empty row
                 type_code = delim[-2].replace(' ', '').replace('`', '')
-                type_codes.append(type_code)
+                type_codes.append((type_code, name))
 
     print('Found', len(type_codes), 'types in the table...')
 
@@ -128,8 +128,16 @@
             f.write(location.replace('(i)', '').replace('(j)', '').replace('(k)', '') + '\n')
     print('Wrote to locations.txt')
 
+    errf = False
+    for (type_code, location) in zip(type_codes, locations):
+        if type_code[1] not in location:
+            errf = True
+            print('Specification format issue: location {} aligned to name/type {} from schema table'.format(location, type_code))
+    if errf:
+        sys.exit('pysnirf2 generation aborted.')
+
     if len(locations) != len(type_codes) or len(locations) != len(descriptions):
-        sys.exit('Parsed ' + str(len(type_codes)) + ' type codes from the summary table but '
+        sys.exit('Parsed ' + str(len(type_codes[0])) + ' type codes from the summary table but '
                  + str(len(locations)) + ' names from the definitions and ' + str(len(descriptions))
                  + ' descriptions: the specification hosted at ' + SPEC_SRC +' was parsed incorrectly. Try adjusting the delimiters and then debug the parsing code (gen.py).')
 
@@ -144,7 +152,7 @@
                 })
 
     for i, (location, description) in enumerate(zip(locations, descriptions)):
-            type_code = type_codes[i]
+            type_code = type_codes[i][0]
             name = location.split('/')[-1].split('(')[0]  # Remove (i), (j)
             parent = location.split('/')[-2].split('(')[0]  # Remove (i), (j)
             print('Found', location, 'with type', type_code)
@@ -163,18 +171,38 @@
                             'required': required
                             })
 
-    ans = input('Proceed? y/n\n')
-    if ans not in ['y', 'Y']:
+    if input('Proceed? y/n\n') not in ['y', 'Y']:
         sys.exit('pysnirf2 generation aborted.')
 
     print('Loading BIDS-specified Probe names from gen/data.py...')
     for name in BIDS_PROBE_NAMES:
         print('Found', name)
-
-    ans = input('Proceed? y/n\n')
-    if ans not in ['y', 'Y']:
+
+    print('\nParsing specification for supported data type integer values...')
+    data_type_table = unidecode(text).split(DATA_TYPE_DELIM_START)[1].split(DATA_TYPE_DELIM_END)[0]
+    data_types = re.findall(r'(?<=\s)-\s(\d+)\s-', data_type_table)
+    data_types = [int(i) for i in data_types]
+    for i in data_types:
+        print('Found', i)
+    if input('Proceed? y/n\n') not in ['y', 'Y']:
         sys.exit('pysnirf2 generation aborted.')
-
+
+    print('\nParsing specification for supported aux names...')
+    aux_name_table = unidecode(text).split(AUX_NAME_TABLE_START)[1].split(AUX_NAME_TABLE_END)[0]
+    aux_names = re.findall(r'"(.*?)"', aux_name_table)
+    for name in aux_names:
+        print('Found', name)
+    if input('Proceed? y/n\n') not in ['y', 'Y']:
+        sys.exit('pysnirf2 generation aborted.')
+
+    print('\nParsing specification for supported data type labels...')
+    data_type_label_table = unidecode(text).split(DATA_TYPE_LABEL_TABLE_START)[1].split(DATA_TYPE_LABEL_TABLE_END)[0]
+    data_type_labels = re.findall(r'"(.*?)"', data_type_label_table)
+    for name in data_type_labels:
+        print('Found', name)
+    if input('Proceed? y/n\n') not in ['y', 'Y']:
+        sys.exit('pysnirf2 generation aborted.')
+
     #  Generate data for template
     SNIRF = {
             'VERSION': SPEC_VERSION,
@@ -188,7 +216,10 @@
             'INDEXED_GROUPS': [], 
             'GROUPS': [], 
             'UNSPECIFIED_DATASETS_OK': UNSPECIFIED_DATASETS_OK,
-            'BIDS_COORDINATE_SYSTEM_NAMES': BIDS_PROBE_NAMES
+            'BIDS_COORDINATE_SYSTEM_NAMES': BIDS_PROBE_NAMES,
+            'AUX_NAMES': aux_names,
+            'DATA_TYPES': data_types,
+            'DATA_TYPE_LABELS': data_type_labels
             }
 
     #  Build list of groups and indexed groups
@@ -223,8 +254,7 @@
         SNIRF['FOOTER'] = TEMPLATE_INSERT_END_STR +  b.split(TEMPLATE_INSERT_END_STR, 1)[1]
         print('Loaded footer code, {} lines'.format(len(SNIRF['FOOTER'].split('\n'))))
 
-    ans = input('Proceed? LOCAL CHANGES MAY BE OVERWRITTEN OR LOST! y/n\n')
-    if ans not in ['y', 'Y']:
+    if input('Proceed? LOCAL CHANGES MAY BE OVERWRITTEN OR LOST! y/n\n') not in ['y', 'Y']:
         sys.exit('pysnirf2 generation aborted.')
     try:
         os.remove(library_path)
@@ -248,12 +278,10 @@
         if errors == 0:
             print('pysnirf2.py generated with', errors, 'errors.')
 
-    ans = input('Format the generated code? y/n\n')
-    if ans in ['y', 'Y']:
+    if input('Format the generated code? y/n\n') in ['y', 'Y']:
         FormatFile(library_path, in_place=True)[:2]
-
-    ans = input('Lint the generated code? y/n\n')
-    if ans in ['y', 'Y']:
+
+    if input('Lint the generated code? y/n\n') in ['y', 'Y']:
         lint.Run(['--errors-only', library_path])
 
     print('\npysnirf2 generation complete.')

gen/pysnirf2.jinja

@@ -76,11 +76,11 @@
     {% if TYPES.INDEXED_GROUP in CHILD.type %}
         return self._{{ CHILD.name }}
     {% elif TYPES.GROUP in CHILD.type %}
-        if type(self._{{ CHILD.name }}) is type(_AbsentGroup):
+        if self._{{ CHILD.name }} is _AbsentGroup:
             return None
         return self._{{ CHILD.name }}
     {% else %}
-        if type(self._{{ CHILD.name }}) is type(_AbsentDataset):
+        if self._{{ CHILD.name }} is _AbsentDataset:
             return None
         if type(self._{{ CHILD.name }}) is type(_PresentDataset):
         {% if (TYPES.ARRAY_1D in CHILD.type) or (TYPES.ARRAY_2D in CHILD.type) %}
@@ -117,6 +117,9 @@
             self._{{ CHILD.name }} = _recursive_hdf5_copy(self._{{ CHILD.name }}, value)
         else:
             raise ValueError("Only a Group of type {{ sentencecase(CHILD.name) }} can be assigned to {{ CHILD.name }}.")
+        {% elif TYPES.ARRAY_1D in CHILD.type or TYPES.ARRAY_2D in CHILD.type %}
+        if value is not None and any([v is not None for v in value]):
+            self._{{ CHILD.name }} = np.array(value)
         {% else %}
         self._{{ CHILD.name }} = value
         {% endif %}
@@ -152,18 +155,18 @@
             else:
                 raise ValueError('Cannot save an anonymous ' + self.__class__.__name__ + ' instance without a filename')
 {% for CHILD in NODE.children %}
+        name = self.location + '/{{ CHILD.name }}'
     {% if TYPES.INDEXED_GROUP in CHILD.type %}
         self.{{ CHILD.name }}._save(*args)
     {% elif TYPES.GROUP in CHILD.type %}
-        if type(self._{{ CHILD.name }}) is type(_AbsentGroup) or self._{{ CHILD.name }}.is_empty():
-            if '{{ CHILD.name }}' in file:
-                del file['{{ CHILD.name }}']
+        if self._{{ CHILD.name }} is _AbsentGroup or self._{{ CHILD.name }}.is_empty():
+            if name in file:
+                del file[name]
                 self._cfg.logger.info('Deleted Group %s/{{ CHILD.name }} from %s', self.location, file)
         else:
             self.{{ CHILD.name }}._save(*args)
     {% else %}
-        name = self.location + '/{{ CHILD.name }}'
-        if type(self._{{ CHILD.name }}) not in [type(_AbsentDataset), type(None)]:
+        if not self._{{ CHILD.name }} is _AbsentDataset:
             data = self.{{ CHILD.name }}  # Use loader function via getter
             if name in file:
                 del file[name]
@@ -222,7 +225,7 @@
 {% macro gen_validator(NODE) %}
     def _validate(self, result: ValidationResult):
         # Validate unwritten datasets after writing them to this tempfile
-        with h5py.File(TemporaryFile(), 'w') as tmp:
+        with h5py.File(str(uuid.uuid4()), 'w', driver='core', backing_store=False) as tmp:
         {% for CHILD in NODE.children %}
             name = self.location + '/{{ CHILD.name }}'
         {% if TYPES.INDEXED_GROUP in CHILD.type %}
@@ -237,7 +240,7 @@
                 self.{{ CHILD.name }}._validate(result)
         {% elif TYPES.GROUP in CHILD.type %}
             # If Group is not present in file and empty in the wrapper, it is missing
-            if type(self._{{ CHILD.name }}) in [type(_AbsentGroup), type(None)] or ('{{ CHILD.name }}' not in self._h and self._{{ CHILD.name }}.is_empty()):
+            if self._{{ CHILD.name }} is _AbsentGroup or ('{{ CHILD.name }}' not in self._h and self._{{ CHILD.name }}.is_empty()):
             {% if TYPES.REQUIRED in CHILD.type %}
                 result._add(name, 'REQUIRED_GROUP_MISSING')
             {% else %}
@@ -246,7 +249,7 @@
             else:
                 self._{{ CHILD.name }}._validate(result)
         {% else %}
-            if type(self._{{ CHILD.name }}) in [type(_AbsentDataset), type(None)]:
+            if self._{{ CHILD.name }} is _AbsentDataset:
             {% if TYPES.REQUIRED in CHILD.type %}
                 result._add(name, 'REQUIRED_DATASET_MISSING')
             {% else %}
@@ -379,6 +382,7 @@ class Snirf(Group):
         self._cfg = SnirfConfig()
         self._cfg.dynamic_loading = dynamic_loading
         self._cfg.fmode = ''
+        self._f = None  # handle for filelikes and temporary files
         if len(args) > 0:
             path = args[0]
             if enable_logging:
@@ -416,7 +420,8 @@ class Snirf(Group):
                 self._cfg.logger.info('Loading from filelike object')
                 if self._cfg.fmode == '':
                     self._cfg.fmode = 'r'
-                self._h = h5py.File(path, 'r')
+                self._f = args[0]
+                self._h = h5py.File(self._f, 'r', backing_store=False)
             else:
                 raise TypeError(str(path) + ' is not a valid filename')
         else:
@@ -427,7 +432,7 @@ class Snirf(Group):
             else:
                 self._cfg.logger = _create_logger('', None)  # Do not log to file
             self._cfg.fmode = 'w'
-            self._h = h5py.File(TemporaryFile(), 'w')
+            self._h = h5py.File(str(uuid.uuid4()), 'w', driver='core', backing_store=False)
     {{ declare_members(ROOT) | indent }}
     {{ init_members(ROOT) | indent }}
 {{ gen_properties(ROOT) }}
@@ -436,4 +441,10 @@ class Snirf(Group):
 
 _RECOGNIZED_COORDINATE_SYSTEM_NAMES = [{% for NAME in BIDS_COORDINATE_SYSTEM_NAMES %}'{{ NAME }}', {% endfor -%}]
 
+_RECOGNIZED_AUX_NAMES = [{% for NAME in AUX_NAMES %}'{{ NAME }}', {% endfor -%}]
+
+_RECOGNIZED_DATA_TYPES = [{% for VALUE in DATA_TYPES %}{{ VALUE }}, {% endfor -%}]
+
+_RECOGNIZED_DATA_TYPE_LABELS = [{% for NAME in DATA_TYPE_LABELS %}'{{ NAME }}', {% endfor -%}]
+
 {{ FOOTER }}

requirements.txt

@@ -1,5 +1,5 @@
 h5py
-numpy
+numpy>=2.0.0
 setuptools
 pip
 termcolor

setup.py

@@ -75,10 +75,10 @@ def run(self):
     long_description=long_description,
     long_description_content_type='text/markdown',
     author_email='sstucker@bu.edu',
-    python_requires='>=3.6.0',
+    python_requires='>=3.9.0',
     install_requires=[
         'h5py>=3.1.0',
-        'numpy',
+        'numpy>2.0.0',
         'setuptools',
         'pip',
         'termcolor',