cleanup before 2.0 release

looper/command_models/arguments.py

@@ -184,6 +184,12 @@ class ArgumentEnum(enum.Enum):
         default=(str, None),
         description="Output directory",
     )
+    REPORT_OUTPUT_DIR = Argument(
+        name="report_dir",
+        alias="-r",
+        default=(str, None),
+        description="Set location for looper report and looper table outputs",
+    )
 
     GENERIC = Argument(
         name="generic",

looper/command_models/commands.py

@@ -124,7 +124,9 @@ def create_model(self) -> Type[pydantic.BaseModel]:
 TableParser = Command(
     "table",
     MESSAGE_BY_SUBCOMMAND["table"],
-    [],
+    [
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
+    ],
 )
 
 
@@ -134,6 +136,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
     MESSAGE_BY_SUBCOMMAND["report"],
     [
         ArgumentEnum.PORTABLE.value,
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
     ],
 )
 

looper/divvy.py

@@ -158,7 +158,7 @@ def activate_package(self, package_name):
             # but now, it makes more sense to do it here so we can piggyback on
             # the default update() method and not even have to do that.
             if not os.path.isabs(self.compute["submission_template"]):
-                
+
                 try:
                     if self.filepath:
                         self.compute["submission_template"] = os.path.join(

looper/exceptions.py

@@ -15,6 +15,7 @@
     "PipelineInterfaceConfigError",
     "PipelineInterfaceRequirementsError",
     "MisconfigurationException",
+    "LooperReportError",
 ]
 
 
@@ -109,3 +110,10 @@ def __init__(self, typename_by_requirement):
             )
         )
         self.error_specs = typename_by_requirement
+
+
+class LooperReportError(LooperError):
+    """Looper reporting errors"""
+
+    def __init__(self, reason):
+        super(LooperReportError, self).__init__(reason)

looper/looper.py

@@ -46,6 +46,7 @@
     sample_folder,
 )
 from pipestat.reports import get_file_for_table
+from pipestat.exceptions import PipestatSummarizeError
 
 _PKGNAME = "looper"
 _LOGGER = logging.getLogger(_PKGNAME)
@@ -94,11 +95,19 @@ def __call__(self, args):
 
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    s = piface.psm.get_status() or "unknown"
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                all_project_level_records = psm.select_records()
+                for record in all_project_level_records["records"]:
+                    s = piface.psm.get_status(
+                        record_identifier=record["record_identifier"]
+                    )
                     status.setdefault(piface.psm.pipeline_name, {})
-                    status[piface.psm.pipeline_name][self.prj.name] = s
-                    _LOGGER.debug(f"{self.prj.name} ({piface.psm.pipeline_name}): {s}")
+                    status[piface.psm.pipeline_name][record["record_identifier"]] = s
+                    _LOGGER.debug(
+                        f"{self.prj.name} ({record['record_identifier']}): {s}"
+                    )
 
         else:
             for sample in self.prj.samples:
@@ -559,28 +568,48 @@ def __call__(self, args):
 
         portable = args.portable
 
+        report_dir = getattr(args, "report_dir", None)
+
         psms = {}
 
         if project_level:
 
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    report_directory = piface.psm.summarize(
-                        looper_samples=self.prj.samples, portable=portable
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                try:
+                    report_directory = psm.summarize(
+                        looper_samples=self.prj.samples,
+                        portable=portable,
+                        output_dir=report_dir,
+                    )
+                except PipestatSummarizeError as e:
+                    raise LooperReportError(
+                        f"Looper report error due to the following exception: {e}"
                     )
                 print(f"Report directory: {report_directory}")
                 self.debug["report_directory"] = report_directory
             return self.debug
         else:
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    report_directory = piface.psm.summarize(
-                        looper_samples=self.prj.samples, portable=portable
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                try:
+                    report_directory = psm.summarize(
+                        looper_samples=self.prj.samples,
+                        portable=portable,
+                        output_dir=report_dir,
+                    )
+                except PipestatSummarizeError as e:
+                    raise LooperReportError(
+                        f"Looper report error due to the following exception: {e}"
                     )
-                    print(f"Report directory: {report_directory}")
-                    self.debug["report_directory"] = report_directory
+                print(f"Report directory: {report_directory}")
+                self.debug["report_directory"] = report_directory
             return self.debug
 
 
@@ -618,18 +647,23 @@ class Tabulator(Executor):
     def __call__(self, args):
         # p = self.prj
         project_level = getattr(args, "project", None)
+        report_dir = getattr(args, "report_dir", None)
         results = []
         psms = {}
         if project_level:
             for piface in self.prj.project_pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    results = piface.psm.table()
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                results = psm.table(output_dir=report_dir)
         else:
             for piface in self.prj.pipeline_interfaces:
                 if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
-                    psms[piface.psm.pipeline_name] = piface.psm
-                    results = piface.psm.table()
+                    if piface.psm.pipeline_name not in psms:
+                        psms[piface.psm.pipeline_name] = piface.psm
+            for pl_name, psm in psms.items():
+                results = psm.table(output_dir=report_dir)
         # Results contains paths to stats and object summaries.
         return results
 

requirements/requirements-all.txt

@@ -4,7 +4,7 @@ jinja2
 logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.4.0
-pipestat>=0.10.2
+pipestat>=0.12.0a1
 peppy>=0.40.6
 pyyaml>=3.12
 rich>=9.10.0

tests/smoketests/test_other.py

@@ -3,7 +3,11 @@
 import pytest
 from peppy import Project
 
-from looper.exceptions import PipestatConfigurationException, MisconfigurationException
+from looper.exceptions import (
+    PipestatConfigurationException,
+    MisconfigurationException,
+    LooperReportError,
+)
 from tests.conftest import *
 from looper.cli_pydantic import main
 import pandas as pd
@@ -78,12 +82,18 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd):
             # Not every command supports dry run
             x = [cmd, "--config", tp]
 
-        try:
-            result = main(test_args=x)
-            if cmd == "run":
-                assert result["Pipestat compatible"] is True
-        except Exception:
-            raise pytest.fail("DID RAISE {0}".format(Exception))
+        if cmd not in ["report"]:
+            try:
+                result = main(test_args=x)
+                if cmd == "run":
+                    assert result["Pipestat compatible"] is True
+            except Exception:
+                raise pytest.fail("DID RAISE {0}".format(Exception))
+        else:
+            with pytest.raises(
+                expected_exception=LooperReportError
+            ):  # Looper report will and should raise exception if there are no results reported.
+                result = main(test_args=x)
 
 
 class TestLooperRerun:

tests/test_comprehensive.py

@@ -123,7 +123,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat):
 
     try:
         result = main(test_args=x)
-        assert result == {"example_pipestat_pipeline": {"project": "unknown"}}
+        assert result == {}
 
     except Exception:
         raise pytest.fail("DID RAISE {0}".format(Exception))