Fixtures depending on comm fixtures: fix bug in sequential scheduler + added test

Author: BerengerBerthoul

pytest_parallel/mpi_reporter.py

@@ -72,7 +72,7 @@ def add_sub_comm(items, global_comm, test_comm_creation, mpi_comm_creation_funct
 
 class SequentialScheduler:
     def __init__(self, global_comm):
-        self.global_comm = global_comm.Dup()  # ensure that all communications within the framework are private to the framework
+        self.global_comm = global_comm.Dup() # ensure that all communications within the framework are private to the framework
 
         # These parameters are not accessible through the API, but are left here for tweaking and experimenting
         self.test_comm_creation = 'by_rank' # possible values : 'by_rank' | 'by_test'
@@ -83,6 +83,17 @@ def __init__(self, global_comm):
             self.mpi_comm_creation_function = 'MPI_Comm_split' # because 'MPI_Comm_create' uses `Create_group`,
                                                                # that is not implemented in mpi4py for Windows
 
+    # Note: We don't need to do this for the static and dynamic scheduler
+    #       So we should investigate on what is the difference here
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_fixture_setup(self, fixturedef, request):
+        if hasattr(request.node, 'sub_comm'): # for some fixtures (notably pytest built-in ones), the communicator is not set yet
+            comm = request.node.sub_comm
+            if comm == MPI.COMM_NULL: # our process does not participate in the test: do not execute fixtures
+                # pytest needs `cached_result` to be non-None, so put something there, hoping it will never be used as a valid value
+                fixturedef.cached_result = (None, 'this_key_should_not_be_matched', RuntimeError('Pytest internal error from pytest_parallel::pytest_fixture_setup'))
+                return True
+
     @pytest.hookimpl(trylast=True)
     def pytest_collection_modifyitems(self, config, items):
         add_sub_comm(items, self.global_comm, self.test_comm_creation, self.mpi_comm_creation_function)
@@ -97,7 +108,6 @@ def pytest_runtest_protocol(self, item, nextitem):
 
     @pytest.hookimpl(tryfirst=True)
     def pytest_pyfunc_call(self, pyfuncitem):
-        #print(f'pytest_pyfunc_call {MPI.COMM_WORLD.rank=}')
         # This is where the test is normally run.
         # Only run the test for the ranks that do participate in the test
         if pyfuncitem.sub_comm == MPI.COMM_NULL:

pytest_parallel/plugin.py

@@ -245,15 +245,15 @@ def __init__(self, comm):
 
     def __enter__(self):
         from mpi4py import MPI
-        if self.comm != MPI.COMM_NULL: # TODO DEL once non-participating rank do not participate in fixtures either
+        if self.comm != MPI.COMM_NULL: # TODO 2025-10: should not be needed anymore > try deletion
             rank = self.comm.rank
             self.tmp_dir = tempfile.TemporaryDirectory() if rank == 0 else None
             self.tmp_path = Path(self.tmp_dir.name) if rank == 0 else None
             return self.comm.bcast(self.tmp_path, root=0)
 
     def __exit__(self, ex_type, ex_value, traceback):
         from mpi4py import MPI
-        if self.comm != MPI.COMM_NULL: # TODO DEL once non-participating rank do not participate in fixtures either
+        if self.comm != MPI.COMM_NULL: # TODO 2025-10: should not be needed anymore > try deletion
             self.comm.barrier()
             if self.comm.rank == 0:
                 self.tmp_dir.cleanup()

test/pytest_parallel_refs/terminal_fixture

@@ -0,0 +1,28 @@
+[=]+ test session starts [=]+
+platform [^\n]*
+cachedir: [^\n]*
+?(?:metadata: [^\n]*)?
+rootdir: [^\n]*
+?(?:configfile: [^\n]*)?
+?(?:plugins: [^\n]*)?
+collecting ... [\s]*collected 1 item[\s]*
+?(?:Submitting tests to SLURM...)?
+?(?:SLURM job [^\n]* has been submitted)?
+
+[^\n]*test_fixture.py::test_fixture\[2\] FAILED
+
+[=]+ FAILURES [=]+
+[_]+ test_fixture\[2\] [_]+
+
+[-]+ On rank 1 of 2 [-]+
+my_fixture = 1
+
+    @pytest_parallel.mark.parallel\(2\)
+    def test_fixture\(my_fixture\):
+>       assert my_fixture == 0 # should fail on proc 1
+E       assert 1 == 0
+
+[^\n]*test_fixture.py:12: AssertionError
+[=]+ short test summary info [=]+
+FAILED [^\n]*test_fixture.py::test_fixture\[2\][^\n]*
+[=]+ 1 failed in [^\n]*s [=]+

test/pytest_parallel_tests/test_fixture.py

@@ -0,0 +1,12 @@
+import pytest
+import pytest_parallel
+
+
+@pytest.fixture
+def my_fixture(comm):
+    return comm.rank
+
+
+@pytest_parallel.mark.parallel(2)
+def test_fixture(my_fixture):
+    assert my_fixture == 0 # should fail on proc 1

test/test_pytest_parallel.py

@@ -99,10 +99,13 @@ def test_16(self, scheduler, capfd): run_pytest_parallel_test('two_success_fail_
 
     def test_17(self, scheduler, capfd): run_pytest_parallel_test('fixture_error'                   , 1, scheduler, capfd) # check that fixture errors are correctly reported
 
-    def test_18(self, scheduler, capfd): run_pytest_parallel_test('parametrize'                     , 2, scheduler, capfd) # check the parametrize API
+    def test_18(self, scheduler, capfd): run_pytest_parallel_test('fixture'                         , 2, scheduler, capfd) # check that fixtures can depend on comm
+    def test_19(self, scheduler, capfd): run_pytest_parallel_test('fixture'                         , 4, scheduler, capfd) # check that fixtures can depend on comm, more procs
 
-    def test_19(self, scheduler, capfd): run_pytest_parallel_test('scheduling'                      , 4, scheduler, capfd) # check 'real' case
-    def test_20(self, scheduler, capfd): run_pytest_parallel_test('fail_complex_assert_two_procs'   , 2, scheduler, capfd) # check 'complex' error message
+    def test_20(self, scheduler, capfd): run_pytest_parallel_test('parametrize'                     , 2, scheduler, capfd) # check the parametrize API
+
+    def test_21(self, scheduler, capfd): run_pytest_parallel_test('scheduling'                      , 4, scheduler, capfd) # check 'real' case
+    def test_22(self, scheduler, capfd): run_pytest_parallel_test('fail_complex_assert_two_procs'   , 2, scheduler, capfd) # check 'complex' error message
 # fmt: on
 
 ## If one test fail, it may be useful to debug regex matching along the following lines