diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py index 2d6b2fbc..68c32977 100644 --- a/looper/command_models/arguments.py +++ b/looper/command_models/arguments.py @@ -184,6 +184,12 @@ class ArgumentEnum(enum.Enum): default=(str, None), description="Output directory", ) + REPORT_OUTPUT_DIR = Argument( + name="report_dir", + alias="-r", + default=(str, None), + description="Set location for looper report and looper table outputs", + ) GENERIC = Argument( name="generic", diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py index 1df4662c..69312f0d 100644 --- a/looper/command_models/commands.py +++ b/looper/command_models/commands.py @@ -124,7 +124,9 @@ def create_model(self) -> Type[pydantic.BaseModel]: TableParser = Command( "table", MESSAGE_BY_SUBCOMMAND["table"], - [], + [ + ArgumentEnum.REPORT_OUTPUT_DIR.value, + ], ) @@ -134,6 +136,7 @@ def create_model(self) -> Type[pydantic.BaseModel]: MESSAGE_BY_SUBCOMMAND["report"], [ ArgumentEnum.PORTABLE.value, + ArgumentEnum.REPORT_OUTPUT_DIR.value, ], ) diff --git a/looper/divvy.py b/looper/divvy.py index c9f97258..e458bad6 100644 --- a/looper/divvy.py +++ b/looper/divvy.py @@ -158,7 +158,7 @@ def activate_package(self, package_name): # but now, it makes more sense to do it here so we can piggyback on # the default update() method and not even have to do that. if not os.path.isabs(self.compute["submission_template"]): - + try: if self.filepath: self.compute["submission_template"] = os.path.join( diff --git a/looper/exceptions.py b/looper/exceptions.py index 469f68af..7d478feb 100644 --- a/looper/exceptions.py +++ b/looper/exceptions.py @@ -15,6 +15,7 @@ "PipelineInterfaceConfigError", "PipelineInterfaceRequirementsError", "MisconfigurationException", + "LooperReportError", ] @@ -109,3 +110,10 @@ def __init__(self, typename_by_requirement): ) ) self.error_specs = typename_by_requirement + + +class LooperReportError(LooperError): + """Looper reporting errors""" + + def __init__(self, reason): + super(LooperReportError, self).__init__(reason) diff --git a/looper/looper.py b/looper/looper.py index 18f0d9ed..cb3cb301 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -46,6 +46,7 @@ sample_folder, ) from pipestat.reports import get_file_for_table +from pipestat.exceptions import PipestatSummarizeError _PKGNAME = "looper" _LOGGER = logging.getLogger(_PKGNAME) @@ -94,11 +95,19 @@ def __call__(self, args): for piface in self.prj.project_pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.PROJECT.value: - psms[piface.psm.pipeline_name] = piface.psm - s = piface.psm.get_status() or "unknown" + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + all_project_level_records = psm.select_records() + for record in all_project_level_records["records"]: + s = piface.psm.get_status( + record_identifier=record["record_identifier"] + ) status.setdefault(piface.psm.pipeline_name, {}) - status[piface.psm.pipeline_name][self.prj.name] = s - _LOGGER.debug(f"{self.prj.name} ({piface.psm.pipeline_name}): {s}") + status[piface.psm.pipeline_name][record["record_identifier"]] = s + _LOGGER.debug( + f"{self.prj.name} ({record['record_identifier']}): {s}" + ) else: for sample in self.prj.samples: @@ -559,15 +568,26 @@ def __call__(self, args): portable = args.portable + report_dir = getattr(args, "report_dir", None) + psms = {} if project_level: for piface in self.prj.project_pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.PROJECT.value: - psms[piface.psm.pipeline_name] = piface.psm - report_directory = piface.psm.summarize( - looper_samples=self.prj.samples, portable=portable + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + try: + report_directory = psm.summarize( + looper_samples=self.prj.samples, + portable=portable, + output_dir=report_dir, + ) + except PipestatSummarizeError as e: + raise LooperReportError( + f"Looper report error due to the following exception: {e}" ) print(f"Report directory: {report_directory}") self.debug["report_directory"] = report_directory @@ -575,12 +595,21 @@ def __call__(self, args): else: for piface in self.prj.pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value: - psms[piface.psm.pipeline_name] = piface.psm - report_directory = piface.psm.summarize( - looper_samples=self.prj.samples, portable=portable + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + try: + report_directory = psm.summarize( + looper_samples=self.prj.samples, + portable=portable, + output_dir=report_dir, + ) + except PipestatSummarizeError as e: + raise LooperReportError( + f"Looper report error due to the following exception: {e}" ) - print(f"Report directory: {report_directory}") - self.debug["report_directory"] = report_directory + print(f"Report directory: {report_directory}") + self.debug["report_directory"] = report_directory return self.debug @@ -618,18 +647,23 @@ class Tabulator(Executor): def __call__(self, args): # p = self.prj project_level = getattr(args, "project", None) + report_dir = getattr(args, "report_dir", None) results = [] psms = {} if project_level: for piface in self.prj.project_pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.PROJECT.value: - psms[piface.psm.pipeline_name] = piface.psm - results = piface.psm.table() + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + results = psm.table(output_dir=report_dir) else: for piface in self.prj.pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value: - psms[piface.psm.pipeline_name] = piface.psm - results = piface.psm.table() + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + results = psm.table(output_dir=report_dir) # Results contains paths to stats and object summaries. return results diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 5005921c..5be6ead7 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -4,7 +4,7 @@ jinja2 logmuse>=0.2.0 pandas>=2.0.2 pephubclient>=0.4.0 -pipestat>=0.10.2 +pipestat>=0.12.0a1 peppy>=0.40.6 pyyaml>=3.12 rich>=9.10.0 diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py index 9713d16a..bc23bfb6 100644 --- a/tests/smoketests/test_other.py +++ b/tests/smoketests/test_other.py @@ -3,7 +3,11 @@ import pytest from peppy import Project -from looper.exceptions import PipestatConfigurationException, MisconfigurationException +from looper.exceptions import ( + PipestatConfigurationException, + MisconfigurationException, + LooperReportError, +) from tests.conftest import * from looper.cli_pydantic import main import pandas as pd @@ -78,12 +82,18 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd): # Not every command supports dry run x = [cmd, "--config", tp] - try: - result = main(test_args=x) - if cmd == "run": - assert result["Pipestat compatible"] is True - except Exception: - raise pytest.fail("DID RAISE {0}".format(Exception)) + if cmd not in ["report"]: + try: + result = main(test_args=x) + if cmd == "run": + assert result["Pipestat compatible"] is True + except Exception: + raise pytest.fail("DID RAISE {0}".format(Exception)) + else: + with pytest.raises( + expected_exception=LooperReportError + ): # Looper report will and should raise exception if there are no results reported. + result = main(test_args=x) class TestLooperRerun: diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py index cb79b356..41c73ea0 100644 --- a/tests/test_comprehensive.py +++ b/tests/test_comprehensive.py @@ -123,7 +123,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat): try: result = main(test_args=x) - assert result == {"example_pipestat_pipeline": {"project": "unknown"}} + assert result == {} except Exception: raise pytest.fail("DID RAISE {0}".format(Exception))