diff --git a/docs/changelog.md b/docs/changelog.md index 133020ce4..986d85502 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,34 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [2.0.0] -- 2025-01-16 + +This release breaks backwards compatibility for Looper versions < 2.0.0 + +### Fixed +- divvy init [#520](https://github.com/pepkit/looper/issues/520) +- replaced deprecated PEPHubClient function, `_load_raw_pep` with `.load_raw_pep` +- looper cli parameters now take priority as originally intended [#518](https://github.com/pepkit/looper/issues/518) +- fix divvy inspect +- remove printed dictionary at looper finish [#511](https://github.com/pepkit/looper/issues/511) +- fix [#536](https://github.com/pepkit/looper/issues/536) +- fix [#522](https://github.com/pepkit/looper/issues/522) +- fix [#537](https://github.com/pepkit/looper/issues/537) +- fix [#534](https://github.com/pepkit/looper/issues/534) + +### Changed +- `--looper-config` is now `--config`, `-c`. [#455](https://github.com/pepkit/looper/issues/455) +- A pipeline interface now consolidates a `sample_interface` and a `project_interface` [#493](https://github.com/pepkit/looper/issues/493) +- Updated documentation for Looper 2.0.0, removing previous versions [pepspec PR #34](https://github.com/pepkit/pepspec/pull/34) +- remove position based argument for divvy config, must use --config or run as default config + +### Added +- `looper init` tutorial [#466](https://github.com/pepkit/looper/issues/466) +- looper config allows for `pephub_path` in pipestat config section of `.looper.yaml` [#519](https://github.com/pepkit/looper/issues/519) +- improve error messaging for bad/malformed looper configurations [#515](https://github.com/pepkit/looper/issues/515) +- add shortform argument for --package (alias is now -p) + + ## [1.9.1] -- 2024-07-18 ### Changed diff --git a/looper/_version.py b/looper/_version.py index 4e14cfd7f..0ac2c6757 100644 --- a/looper/_version.py +++ b/looper/_version.py @@ -1,2 +1,2 @@ -__version__ = "1.9.1" -# You must change the version in parser = pydantic2_argparse.ArgumentParser in cli_pydantic.py!!! +__version__ = "2.0.0" +# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!! diff --git a/looper/cli_divvy.py b/looper/cli_divvy.py index 0c152e252..1fa98b69e 100644 --- a/looper/cli_divvy.py +++ b/looper/cli_divvy.py @@ -53,10 +53,10 @@ def add_subparser(cmd, description): for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]: sp.add_argument( - "config", nargs="?", default=None, help="Divvy configuration file." + "--config", nargs="?", default=None, help="Divvy configuration file." ) - sps["init"].add_argument("config", default=None, help="Divvy configuration file.") + sps["init"].add_argument("--config", default=None, help="Divvy configuration file.") for sp in [sps["inspect"]]: sp.add_argument( @@ -124,9 +124,11 @@ def main(): sys.exit(0) _LOGGER.debug("Divvy config: {}".format(args.config)) + divcfg = select_divvy_config(args.config) + _LOGGER.info("Using divvy config: {}".format(divcfg)) - dcc = ComputingConfiguration(filepath=divcfg) + dcc = ComputingConfiguration.from_yaml_file(filepath=divcfg) if args.command == "list": # Output header via logger and content via print so the user can @@ -142,11 +144,13 @@ def main(): for pkg_name, pkg in dcc.compute_packages.items(): if pkg_name == args.package: found = True - with open(pkg.submission_template, "r") as f: + with open(pkg["submission_template"], "r") as f: print(f.read()) - _LOGGER.info("Submission command is: " + pkg.submission_command + "\n") + _LOGGER.info( + "Submission command is: " + pkg["submission_command"] + "\n" + ) if pkg_name == "docker": - print("Docker args are: " + pkg.docker_args) + print("Docker args are: " + pkg["docker_args"]) if not found: _LOGGER.info("Package not found. Use 'divvy list' to see list of packages.") diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py index 4efec2188..1bef39f30 100644 --- a/looper/cli_pydantic.py +++ b/looper/cli_pydantic.py @@ -26,8 +26,6 @@ from pephubclient import PEPHubClient from pydantic_argparse.argparse.parser import ArgumentParser -from divvy import select_divvy_config - from . import __version__ from .command_models.arguments import ArgumentEnum @@ -54,9 +52,11 @@ read_yaml_file, inspect_looper_config_file, is_PEP_file_type, + looper_config_tutorial, ) from typing import List, Tuple +from rich.console import Console def opt_attr_pair(name: str) -> Tuple[str, str]: @@ -122,52 +122,56 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): sys.exit(1) if subcommand_name == "init": - return int( - not initiate_looper_config( - dotfile_path(), - subcommand_args.pep_config, - subcommand_args.output_dir, - subcommand_args.sample_pipeline_interfaces, - subcommand_args.project_pipeline_interfaces, - subcommand_args.force_yes, + + console = Console() + console.clear() + console.rule(f"\n[magenta]Looper initialization[/magenta]") + selection = subcommand_args.generic + if selection is True: + console.clear() + return int( + not initiate_looper_config( + dotfile_path(), + subcommand_args.pep_config, + subcommand_args.output_dir, + subcommand_args.sample_pipeline_interfaces, + subcommand_args.project_pipeline_interfaces, + subcommand_args.force_yes, + ) ) - ) + else: + console.clear() + return int(looper_config_tutorial()) if subcommand_name == "init_piface": sys.exit(int(not init_generic_pipeline())) _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name)) - if subcommand_args.config_file is None: - looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir) - try: - if subcommand_args.looper_config: - looper_config_dict = read_looper_config_file( - subcommand_args.looper_config - ) + looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir) + try: + if subcommand_args.config: + looper_config_dict = read_looper_config_file(subcommand_args.config) + else: + looper_config_dict = read_looper_dotfile() + _LOGGER.info(f"Using looper config ({looper_cfg_path}).") + + cli_modifiers_dict = None + for looper_config_key, looper_config_item in looper_config_dict.items(): + if looper_config_key == CLI_KEY: + cli_modifiers_dict = looper_config_item else: - looper_config_dict = read_looper_dotfile() - _LOGGER.info(f"Using looper config ({looper_cfg_path}).") - - cli_modifiers_dict = None - for looper_config_key, looper_config_item in looper_config_dict.items(): - if looper_config_key == CLI_KEY: - cli_modifiers_dict = looper_config_item - else: - setattr(subcommand_args, looper_config_key, looper_config_item) - - except OSError: - parser.print_help(sys.stderr) + setattr(subcommand_args, looper_config_key, looper_config_item) + + except OSError as e: + if subcommand_args.config: _LOGGER.warning( - f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}." + f"\nLooper config file does not exist at given path {subcommand_args.config}. Use looper init to create one at {looper_cfg_path}." ) - sys.exit(1) - else: - _LOGGER.warning( - "This PEP configures looper through the project config. This approach is deprecated and will " - "be removed in future versions. Please use a looper config file. For more information see " - "looper.databio.org/en/latest/looper-config" - ) + else: + _LOGGER.warning(e) + + sys.exit(1) subcommand_args = enrich_args_via_cfg( subcommand_name, @@ -191,12 +195,12 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): subcommand_args.ignore_flags = True # Initialize project - if is_PEP_file_type(subcommand_args.config_file) and os.path.exists( - subcommand_args.config_file + if is_PEP_file_type(subcommand_args.pep_config) and os.path.exists( + subcommand_args.pep_config ): try: p = Project( - cfg=subcommand_args.config_file, + cfg=subcommand_args.pep_config, amendments=subcommand_args.amend, divcfg_path=divcfg, runp=subcommand_name == "runp", @@ -209,14 +213,14 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): except yaml.parser.ParserError as e: _LOGGER.error(f"Project config parse failed -- {e}") sys.exit(1) - elif is_pephub_registry_path(subcommand_args.config_file): + elif is_pephub_registry_path(subcommand_args.pep_config): if vars(subcommand_args)[SAMPLE_PL_ARG]: p = Project( amendments=subcommand_args.amend, divcfg_path=divcfg, runp=subcommand_name == "runp", - project_dict=PEPHubClient()._load_raw_pep( - registry_path=subcommand_args.config_file + project_dict=PEPHubClient().load_raw_pep( + registry_path=subcommand_args.pep_config ), **{ attr: getattr(subcommand_args, attr) @@ -252,7 +256,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): # Check at the beginning if user wants to use pipestat and pipestat is configurable is_pipestat_configured = ( prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value) - if getattr(subcommand_args, "project", None) + if getattr(subcommand_args, "project", None) or subcommand_name == "runp" else prj._check_if_pipestat_configured() ) @@ -330,13 +334,13 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None): _LOGGER.warning("No looper configuration was supplied.") -def main(test_args=None) -> None: +def main(test_args=None) -> dict: parser = pydantic_argparse.ArgumentParser( model=TopLevelParser, prog="looper", description="Looper: A job submitter for Portable Encapsulated Projects", add_help=True, - version="1.9.1", + version="2.0.0", ) parser = add_short_arguments(parser, ArgumentEnum) @@ -349,6 +353,10 @@ def main(test_args=None) -> None: return run_looper(args, parser, test_args=test_args) +def main_cli() -> None: + main() + + def _proc_resources_spec(args): """ Process CLI-sources compute setting specification. There are two sources @@ -375,20 +383,29 @@ def _proc_resources_spec(args): settings_data = {} if not spec: return settings_data - pairs = [(kv, kv.split("=")) for kv in spec] - bads = [] - for orig, pair in pairs: - try: - k, v = pair - except ValueError: - bads.append(orig) - else: - settings_data[k] = v - if bads: - raise ValueError( - "Could not correctly parse itemized compute specification. " - "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT - ) + if isinstance( + spec, str + ): # compute: "partition=standard time='01-00:00:00' cores='32' mem='32000'" + spec = spec.split(sep=" ") + if isinstance(spec, list): + pairs = [(kv, kv.split("=")) for kv in spec] + bads = [] + for orig, pair in pairs: + try: + k, v = pair + except ValueError: + bads.append(orig) + else: + settings_data[k] = v + if bads: + raise ValueError( + "Could not correctly parse itemized compute specification. " + "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT + ) + elif isinstance(spec, dict): + for key, value in spec.items(): + settings_data[key] = value + return settings_data diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py index 8c484d33d..68c329772 100644 --- a/looper/command_models/arguments.py +++ b/looper/command_models/arguments.py @@ -162,13 +162,9 @@ class ArgumentEnum(enum.Enum): default=(int, None), description="Skip samples by numerical index", ) - CONFIG_FILE = Argument( - name="config_file", - default=(str, None), - description="Project configuration file", - ) - LOOPER_CONFIG = Argument( - name="looper_config", + CONFIG = Argument( + name="config", + alias="-c", default=(str, None), description="Looper configuration file (YAML)", ) @@ -188,6 +184,20 @@ class ArgumentEnum(enum.Enum): default=(str, None), description="Output directory", ) + REPORT_OUTPUT_DIR = Argument( + name="report_dir", + alias="-r", + default=(str, None), + description="Set location for looper report and looper table outputs", + ) + + GENERIC = Argument( + name="generic", + alias="-g", + default=(bool, False), + description="Use generic looper config?", + ) + SAMPLE_PIPELINE_INTERFACES = Argument( name="sample_pipeline_interfaces", alias="-S", @@ -232,12 +242,12 @@ class ArgumentEnum(enum.Enum): ) PACKAGE = Argument( name="package", + alias="-p", default=(str, None), description="Name of computing resource package to use", ) COMPUTE = Argument( name="compute", - alias="-c", default=(List, []), description="List of key-value pairs (k1=v1)", ) diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py index e764c99db..69312f0d6 100644 --- a/looper/command_models/commands.py +++ b/looper/command_models/commands.py @@ -53,8 +53,7 @@ def create_model(self) -> Type[pydantic.BaseModel]: ArgumentEnum.SKIP.value, ArgumentEnum.PEP_CONFIG.value, ArgumentEnum.OUTPUT_DIR.value, - ArgumentEnum.CONFIG_FILE.value, - ArgumentEnum.LOOPER_CONFIG.value, + ArgumentEnum.CONFIG.value, ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value, ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value, ArgumentEnum.PIPESTAT.value, @@ -125,7 +124,9 @@ def create_model(self) -> Type[pydantic.BaseModel]: TableParser = Command( "table", MESSAGE_BY_SUBCOMMAND["table"], - [], + [ + ArgumentEnum.REPORT_OUTPUT_DIR.value, + ], ) @@ -135,6 +136,7 @@ def create_model(self) -> Type[pydantic.BaseModel]: MESSAGE_BY_SUBCOMMAND["report"], [ ArgumentEnum.PORTABLE.value, + ArgumentEnum.REPORT_OUTPUT_DIR.value, ], ) @@ -188,6 +190,7 @@ def create_model(self) -> Type[pydantic.BaseModel]: ArgumentEnum.PEP_CONFIG.value, ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value, ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value, + ArgumentEnum.GENERIC.value, ], ) diff --git a/looper/conductor.py b/looper/conductor.py index 93598431e..231a2366a 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -198,6 +198,9 @@ def __init__( self.collate = collate self.section_key = PROJECT_PL_KEY if self.collate else SAMPLE_PL_KEY + self.pipeline_interface_type = ( + "project_interface" if self.collate else "sample_interface" + ) self.pl_iface = pipeline_interface self.pl_name = self.pl_iface.pipeline_name self.prj = prj @@ -658,6 +661,7 @@ def _set_pipestat_namespace( "record_identifier": psm.record_identifier, "config_file": psm.config_path, "output_schema": psm.cfg["_schema_path"], + "pephub_path": psm.cfg["pephub_path"], } filtered_namespace = {k: v for k, v in full_namespace.items() if v} return YAMLConfigManager(filtered_namespace) @@ -681,7 +685,11 @@ def write_script(self, pool, size): pipeline=self.pl_iface, compute=self.prj.dcc.compute, ) - templ = self.pl_iface["command_template"] + + if self.pipeline_interface_type is None: + templ = self.pl_iface["command_template"] + else: + templ = self.pl_iface[self.pipeline_interface_type]["command_template"] if not self.override_extra: extras_template = ( EXTRA_PROJECT_CMD_TEMPLATE diff --git a/looper/divvy.py b/looper/divvy.py index 38dd8fe21..e458bad6b 100644 --- a/looper/divvy.py +++ b/looper/divvy.py @@ -111,9 +111,12 @@ def templates_folder(self): :return str: path to folder with default submission templates """ - return os.path.join( - os.path.dirname(__file__), "default_config", "divvy_templates" - ) + if self.filepath: + return os.path.join(os.path.dirname(self.filepath), "divvy_templates") + else: + return os.path.join( + os.path.dirname(__file__), "default_config", "divvy_templates" + ) def activate_package(self, package_name): """ @@ -155,11 +158,18 @@ def activate_package(self, package_name): # but now, it makes more sense to do it here so we can piggyback on # the default update() method and not even have to do that. if not os.path.isabs(self.compute["submission_template"]): + try: - self.compute["submission_template"] = os.path.join( - os.path.dirname(self.default_config_file), - self.compute["submission_template"], - ) + if self.filepath: + self.compute["submission_template"] = os.path.join( + os.path.dirname(self.filepath), + self.compute["submission_template"], + ) + else: + self.compute["submission_template"] = os.path.join( + os.path.dirname(self.default_config_file), + self.compute["submission_template"], + ) except AttributeError as e: # Environment and environment compute should at least have been # set as null-valued attributes, so execution here is an error. @@ -200,6 +210,11 @@ def get_active_package(self) -> YAMLConfigManager: """ return self.compute + @property + def compute_packages(self): + + return self["compute_packages"] + def list_compute_packages(self): """ Returns a list of available compute packages. @@ -396,11 +411,13 @@ def divvy_init(config_path, template_config_path): _LOGGER.error("You must specify a template config file path.") return + if not os.path.isabs(config_path): + config_path = os.path.abspath(config_path) + if config_path and not os.path.exists(config_path): - # dcc.write(config_path) # Init should *also* write the templates. dest_folder = os.path.dirname(config_path) - copytree(os.path.dirname(template_config_path), dest_folder) + copytree(os.path.dirname(template_config_path), dest_folder, dirs_exist_ok=True) template_subfolder = os.path.join(dest_folder, "divvy_templates") _LOGGER.info("Wrote divvy templates to folder: {}".format(template_subfolder)) new_template = os.path.join( diff --git a/looper/exceptions.py b/looper/exceptions.py index f9cb9e0c7..7d478feb7 100644 --- a/looper/exceptions.py +++ b/looper/exceptions.py @@ -15,6 +15,7 @@ "PipelineInterfaceConfigError", "PipelineInterfaceRequirementsError", "MisconfigurationException", + "LooperReportError", ] @@ -31,7 +32,7 @@ class SampleFailedException(LooperError): class MisconfigurationException(LooperError): - """Duplication of pipeline identifier precludes unique pipeline ref.""" + """Looper not properly configured""" def __init__(self, key): super(MisconfigurationException, self).__init__(key) @@ -109,3 +110,10 @@ def __init__(self, typename_by_requirement): ) ) self.error_specs = typename_by_requirement + + +class LooperReportError(LooperError): + """Looper reporting errors""" + + def __init__(self, reason): + super(LooperReportError, self).__init__(reason) diff --git a/looper/looper.py b/looper/looper.py index 18f0d9ed8..cb3cb3014 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -46,6 +46,7 @@ sample_folder, ) from pipestat.reports import get_file_for_table +from pipestat.exceptions import PipestatSummarizeError _PKGNAME = "looper" _LOGGER = logging.getLogger(_PKGNAME) @@ -94,11 +95,19 @@ def __call__(self, args): for piface in self.prj.project_pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.PROJECT.value: - psms[piface.psm.pipeline_name] = piface.psm - s = piface.psm.get_status() or "unknown" + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + all_project_level_records = psm.select_records() + for record in all_project_level_records["records"]: + s = piface.psm.get_status( + record_identifier=record["record_identifier"] + ) status.setdefault(piface.psm.pipeline_name, {}) - status[piface.psm.pipeline_name][self.prj.name] = s - _LOGGER.debug(f"{self.prj.name} ({piface.psm.pipeline_name}): {s}") + status[piface.psm.pipeline_name][record["record_identifier"]] = s + _LOGGER.debug( + f"{self.prj.name} ({record['record_identifier']}): {s}" + ) else: for sample in self.prj.samples: @@ -559,15 +568,26 @@ def __call__(self, args): portable = args.portable + report_dir = getattr(args, "report_dir", None) + psms = {} if project_level: for piface in self.prj.project_pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.PROJECT.value: - psms[piface.psm.pipeline_name] = piface.psm - report_directory = piface.psm.summarize( - looper_samples=self.prj.samples, portable=portable + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + try: + report_directory = psm.summarize( + looper_samples=self.prj.samples, + portable=portable, + output_dir=report_dir, + ) + except PipestatSummarizeError as e: + raise LooperReportError( + f"Looper report error due to the following exception: {e}" ) print(f"Report directory: {report_directory}") self.debug["report_directory"] = report_directory @@ -575,12 +595,21 @@ def __call__(self, args): else: for piface in self.prj.pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value: - psms[piface.psm.pipeline_name] = piface.psm - report_directory = piface.psm.summarize( - looper_samples=self.prj.samples, portable=portable + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + try: + report_directory = psm.summarize( + looper_samples=self.prj.samples, + portable=portable, + output_dir=report_dir, + ) + except PipestatSummarizeError as e: + raise LooperReportError( + f"Looper report error due to the following exception: {e}" ) - print(f"Report directory: {report_directory}") - self.debug["report_directory"] = report_directory + print(f"Report directory: {report_directory}") + self.debug["report_directory"] = report_directory return self.debug @@ -618,18 +647,23 @@ class Tabulator(Executor): def __call__(self, args): # p = self.prj project_level = getattr(args, "project", None) + report_dir = getattr(args, "report_dir", None) results = [] psms = {} if project_level: for piface in self.prj.project_pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.PROJECT.value: - psms[piface.psm.pipeline_name] = piface.psm - results = piface.psm.table() + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + results = psm.table(output_dir=report_dir) else: for piface in self.prj.pipeline_interfaces: if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value: - psms[piface.psm.pipeline_name] = piface.psm - results = piface.psm.table() + if piface.psm.pipeline_name not in psms: + psms[piface.psm.pipeline_name] = piface.psm + for pl_name, psm in psms.items(): + results = psm.table(output_dir=report_dir) # Results contains paths to stats and object summaries. return results diff --git a/looper/pipeline_interface.py b/looper/pipeline_interface.py index f796354a4..387c6d49e 100644 --- a/looper/pipeline_interface.py +++ b/looper/pipeline_interface.py @@ -56,15 +56,6 @@ def __init__(self, config, pipeline_type=None): ) self.update(config) self._validate(schema_src=PIFACE_SCHEMA_SRC) - if "path" in self: - warn( - message="'path' specification as a top-level pipeline " - "interface key is deprecated and will be removed with " - "the next release. Please use 'paths' section " - "from now on.", - category=DeprecationWarning, - ) - self._expand_paths(["path"]) self._expand_paths(["compute", "dynamic_variables_script_path"]) @property diff --git a/looper/project.py b/looper/project.py index 7a652db7c..d77dc3681 100644 --- a/looper/project.py +++ b/looper/project.py @@ -413,10 +413,12 @@ def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value): pipestat_config_path = self._check_for_existing_pipestat_config(piface) if not pipestat_config_path: - self._create_pipestat_config(piface) + self._create_pipestat_config(piface, pipeline_type) else: piface.psm = PipestatManager( - config_file=pipestat_config_path, multi_pipelines=True + config_file=pipestat_config_path, + multi_pipelines=True, + pipeline_type="sample", ) elif pipeline_type == PipelineLevel.PROJECT.value: @@ -426,10 +428,12 @@ def _get_pipestat_configuration(self, pipeline_type=PipelineLevel.SAMPLE.value): ) if not pipestat_config_path: - self._create_pipestat_config(prj_piface) + self._create_pipestat_config(prj_piface, pipeline_type) else: prj_piface.psm = PipestatManager( - config_file=pipestat_config_path, multi_pipelines=True + config_file=pipestat_config_path, + multi_pipelines=True, + pipeline_type="project", ) else: _LOGGER.error( @@ -469,7 +473,7 @@ def _check_for_existing_pipestat_config(self, piface): else: return None - def _create_pipestat_config(self, piface): + def _create_pipestat_config(self, piface, pipeline_type): """ Each piface needs its own config file and associated psm """ @@ -512,8 +516,6 @@ def _create_pipestat_config(self, piface): pipestat_config_dict.update({"pipeline_name": piface.data["pipeline_name"]}) else: pipeline_name = None - if "pipeline_type" in piface.data: - pipestat_config_dict.update({"pipeline_type": piface.data["pipeline_type"]}) # Warn user if there is a mismatch in pipeline_names from sources!!! if pipeline_name != output_schema_pipeline_name: diff --git a/looper/schemas/pipeline_interface_schema_generic.yaml b/looper/schemas/pipeline_interface_schema_generic.yaml index 3d1b4ea18..8528b6138 100644 --- a/looper/schemas/pipeline_interface_schema_generic.yaml +++ b/looper/schemas/pipeline_interface_schema_generic.yaml @@ -9,12 +9,20 @@ properties: type: string enum: ["project", "sample"] description: "type of the pipeline, either 'project' or 'sample'" - command_template: - type: string - description: "Jinja2-like template to construct the command to run" - path: - type: string - description: "path to the pipeline program. Relative to pipeline interface file or absolute." + sample_interface: + type: object + description: "Section that defines compute environment settings" + properties: + command_template: + type: string + description: "Jinja2-like template to construct the command to run" + project_interface: + type: object + description: "Section that defines compute environment settings" + properties: + command_template: + type: string + description: "Jinja2-like template to construct the command to run" compute: type: object description: "Section that defines compute environment settings" diff --git a/looper/utils.py b/looper/utils.py index 4eb8ca4b8..5a8279bd4 100644 --- a/looper/utils.py +++ b/looper/utils.py @@ -17,10 +17,13 @@ from pephubclient.constants import RegistryPath from pydantic import ValidationError from yacman import load_yaml +from yaml.parser import ParserError from .const import * from .command_models.commands import SUPPORTED_COMMANDS -from .exceptions import MisconfigurationException +from .exceptions import MisconfigurationException, PipelineInterfaceConfigError +from rich.console import Console +from rich.pretty import pprint _LOGGER = getLogger(__name__) @@ -261,31 +264,29 @@ def enrich_args_via_cfg( cli_modifiers=None, ): """ - Read in a looper dotfile and set arguments. + Read in a looper dotfile, pep config and set arguments. - Priority order: CLI > dotfile/config > parser default + Priority order: CLI > dotfile/config > pep_config > parser default :param subcommand name: the name of the command used :param argparse.Namespace parser_args: parsed args by the original parser - :param argparse.Namespace aux_parser: parsed args by the a parser + :param argparse.Namespace aux_parser: parsed args by the argument parser with defaults suppressed + :param dict test_args: dict of args used for pytesting + :param dict cli_modifiers: dict of args existing if user supplied cli args in looper config file :return argparse.Namespace: selected argument values """ + + # Did the user provide arguments in the PEP config? cfg_args_all = ( _get_subcommand_args(subcommand_name, parser_args) - if os.path.exists(parser_args.config_file) + if os.path.exists(parser_args.pep_config) else dict() ) - - # If user provided project-level modifiers in the looper config, they are prioritized - if cfg_args_all: - for key, value in cfg_args_all.items(): - if getattr(parser_args, key, None): - new_value = getattr(parser_args, key) - cfg_args_all[key] = new_value - else: + if not cfg_args_all: cfg_args_all = {} + # Did the user provide arguments/modifiers in the looper config file? looper_config_cli_modifiers = None if cli_modifiers: if str(subcommand_name) in cli_modifiers: @@ -310,6 +311,13 @@ def enrich_args_via_cfg( else: cli_args, _ = aux_parser.parse_known_args() + # If any CLI args were provided, make sure they take priority + if cli_args: + r = getattr(cli_args, subcommand_name) + for k, v in cfg_args_all.items(): + if k in r: + cfg_args_all[k] = getattr(r, k) + def set_single_arg(argname, default_source_namespace, result_namespace): if argname not in POSITIONAL or not hasattr(result, argname): if argname in cli_args: @@ -322,6 +330,8 @@ def set_single_arg(argname, default_source_namespace, result_namespace): elif cfg_args_all is not None and argname in cfg_args_all: if isinstance(cfg_args_all[argname], list): r = [convert_value(i) for i in cfg_args_all[argname]] + elif isinstance(cfg_args_all[argname], dict): + r = cfg_args_all[argname] else: r = convert_value(cfg_args_all[argname]) else: @@ -360,7 +370,7 @@ def _get_subcommand_args(subcommand_name, parser_args): """ args = dict() cfg = peppyProject( - parser_args.config_file, + parser_args.pep_config, defer_samples_creation=True, amendments=parser_args.amend, ) @@ -402,40 +412,65 @@ def _get_subcommand_args(subcommand_name, parser_args): return args -def init_generic_pipeline(): +def init_generic_pipeline(pipelinepath: Optional[str] = None): """ Create generic pipeline interface """ - try: - os.makedirs("pipeline") - except FileExistsError: - pass + console = Console() # Destination one level down from CWD in pipeline folder - dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE) + if not pipelinepath: + try: + os.makedirs("pipeline") + except FileExistsError: + pass + + dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE) + else: + if os.path.isabs(pipelinepath): + dest_file = pipelinepath + else: + dest_file = os.path.join(os.getcwd(), os.path.relpath(pipelinepath)) + try: + os.makedirs(os.path.dirname(dest_file)) + except FileExistsError: + pass # Create Generic Pipeline Interface generic_pipeline_dict = { "pipeline_name": "default_pipeline_name", - "pipeline_type": "sample", "output_schema": "output_schema.yaml", - "var_templates": {"pipeline": "{looper.piface_dir}/pipeline.sh"}, - "command_template": "{pipeline.var_templates.pipeline} {sample.file} " - "--output-parent {looper.sample_output_folder}", + "sample_interface": { + "command_template": "{looper.piface_dir}/count_lines.sh {sample.file} " + "--output-parent {looper.sample_output_folder}" + }, } + console.rule(f"\n[magenta]Pipeline Interface[/magenta]") # Write file if not os.path.exists(dest_file): + pprint(generic_pipeline_dict, expand_all=True) + with open(dest_file, "w") as file: yaml.dump(generic_pipeline_dict, file) - print(f"Pipeline interface successfully created at: {dest_file}") + + console.print( + f"Pipeline interface successfully created at: [yellow]{dest_file}[/yellow]" + ) + else: - print( - f"Pipeline interface file already exists `{dest_file}`. Skipping creation.." + console.print( + f"Pipeline interface file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.." ) # Create Generic Output Schema - dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA) + if not pipelinepath: + dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA) + else: + dest_file = os.path.join( + os.path.dirname(dest_file), LOOPER_GENERIC_OUTPUT_SCHEMA + ) + generic_output_schema_dict = { "pipeline_name": "default_pipeline_name", "samples": { @@ -445,27 +480,45 @@ def init_generic_pipeline(): } }, } + + console.rule(f"\n[magenta]Output Schema[/magenta]") # Write file if not os.path.exists(dest_file): + pprint(generic_output_schema_dict, expand_all=True) with open(dest_file, "w") as file: yaml.dump(generic_output_schema_dict, file) - print(f"Output schema successfully created at: {dest_file}") + console.print( + f"Output schema successfully created at: [yellow]{dest_file}[/yellow]" + ) else: - print(f"Output schema file already exists `{dest_file}`. Skipping creation..") + console.print( + f"Output schema file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.." + ) + console.rule(f"\n[magenta]Example Pipeline Shell Script[/magenta]") # Create Generic countlines.sh - dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES) + + if not pipelinepath: + dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES) + else: + dest_file = os.path.join(os.path.dirname(dest_file), LOOPER_GENERIC_COUNT_LINES) + shell_code = """#!/bin/bash linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3 echo "Number of lines: $linecount" """ if not os.path.exists(dest_file): + console.print(shell_code) with open(dest_file, "w") as file: file.write(shell_code) - print(f"count_lines.sh successfully created at: {dest_file}") + console.print( + f"count_lines.sh successfully created at: [yellow]{dest_file}[/yellow]" + ) else: - print(f"count_lines.sh file already exists `{dest_file}`. Skipping creation..") + console.print( + f"count_lines.sh file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.." + ) return True @@ -500,8 +553,14 @@ def initiate_looper_config( :param bool force: whether the existing file should be overwritten :return bool: whether the file was initialized """ + console = Console() + console.clear() + console.rule(f"\n[magenta]Looper initialization[/magenta]") + if os.path.exists(looper_config_path) and not force: - print(f"Can't initialize, file exists: {looper_config_path}") + console.print( + f"[red]Can't initialize, file exists:[/red] [yellow]{looper_config_path}[/yellow]" + ) return False if pep_path: @@ -521,24 +580,154 @@ def initiate_looper_config( if not output_dir: output_dir = "." + if sample_pipeline_interfaces is None or sample_pipeline_interfaces == []: + sample_pipeline_interfaces = "pipeline_interface1.yaml" + + if project_pipeline_interfaces is None or project_pipeline_interfaces == []: + project_pipeline_interfaces = "pipeline_interface2.yaml" + looper_config_dict = { "pep_config": os.path.relpath(pep_path), "output_dir": output_dir, - "pipeline_interfaces": { - "sample": sample_pipeline_interfaces, - "project": project_pipeline_interfaces, - }, + "pipeline_interfaces": [ + sample_pipeline_interfaces, + project_pipeline_interfaces, + ], } + pprint(looper_config_dict, expand_all=True) + with open(looper_config_path, "w") as dotfile: yaml.dump(looper_config_dict, dotfile) - print(f"Initialized looper config file: {looper_config_path}") + console.print( + f"Initialized looper config file: [yellow]{looper_config_path}[/yellow]" + ) + + return True + + +def looper_config_tutorial(): + """ + Prompt a user through configuring a .looper.yaml file for a new project. + + :return bool: whether the file was initialized + """ + + console = Console() + console.clear() + console.rule(f"\n[magenta]Looper initialization[/magenta]") + + looper_cfg_path = ".looper.yaml" # not changeable + + if os.path.exists(looper_cfg_path): + console.print( + f"[bold red]File exists at '{looper_cfg_path}'. Delete it to re-initialize. \n[/bold red]" + ) + raise SystemExit + + cfg = {} + + console.print( + "This utility will walk you through creating a [yellow].looper.yaml[/yellow] file." + ) + console.print("See [yellow]`looper init --help`[/yellow] for details.") + console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.") + console.print("Press [yellow]^C[/yellow] at any time to quit.\n") + + DEFAULTS = { # What you get if you just press enter + "pep_config": "databio/example", + "output_dir": "results", + "piface_path": "pipeline/pipeline_interface.yaml", + "project_name": os.path.basename(os.getcwd()), + } + + cfg["project_name"] = ( + console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >") + or DEFAULTS["project_name"] + ) + + cfg["pep_config"] = ( + console.input( + f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >" + ) + or DEFAULTS["pep_config"] + ) + + if not os.path.exists(cfg["pep_config"]) and not is_pephub_registry_path( + cfg["pep_config"] + ): + console.print( + f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'" + ) + + cfg["output_dir"] = ( + console.input( + f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >" + ) + or DEFAULTS["output_dir"] + ) + + add_more_pifaces = True + piface_paths = [] + while add_more_pifaces: + piface_path = ( + console.input( + "Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >" + ) + or None + ) + if piface_path is None: + if piface_paths == []: + piface_paths.append(DEFAULTS["piface_path"]) + add_more_pifaces = False + else: + piface_paths.append(piface_path) + + console.print("\n") + + console.print( + f"""\ +[yellow]pep_config:[/yellow] {cfg['pep_config']} +[yellow]output_dir:[/yellow] {cfg['output_dir']} +[yellow]pipeline_interfaces:[/yellow] + - {piface_paths} +""" + ) + + for piface_path in piface_paths: + if not os.path.exists(piface_path): + console.print( + f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]" + ) + console.print( + "Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..." + ) + selection = None + while selection not in ["y", "n"]: + selection = console.input("\nSelection: ").lower().strip() + if selection == "n": + console.print( + "Use command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface." + ) + if selection == "y": + init_generic_pipeline(pipelinepath=piface_path) + + console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]") + + looper_config_dict = {} + looper_config_dict["pep_config"] = cfg["pep_config"] + looper_config_dict["output_dir"] = cfg["output_dir"] + looper_config_dict["pipeline_interfaces"] = piface_paths + + with open(looper_cfg_path, "w") as fp: + yaml.dump(looper_config_dict, fp) + return True def determine_pipeline_type(piface_path: str, looper_config_path: str): """ - Read pipeline interface from disk and determine if pipeline type is sample or project-level + Read pipeline interface from disk and determine if it contains "sample_interface", "project_interface" or both :param str piface_path: path to pipeline_interface @@ -548,7 +737,15 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str): if piface_path is None: return None, None - piface_path = expandpath(piface_path) + try: + piface_path = expandpath(piface_path) + except TypeError as e: + _LOGGER.warning( + f"Pipeline interface not found at given path: {piface_path}. Type Error: " + + str(e) + ) + return None, None + if not os.path.isabs(piface_path): piface_path = os.path.realpath( os.path.join(os.path.dirname(looper_config_path), piface_path) @@ -556,11 +753,21 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str): try: piface_dict = load_yaml(piface_path) except FileNotFoundError: + _LOGGER.warning(f"Pipeline interface not found at given path: {piface_path}") return None, None - pipeline_type = piface_dict.get("pipeline_type", None) + pipeline_types = [] + if piface_dict.get("sample_interface", None): + pipeline_types.append(PipelineLevel.SAMPLE.value) + if piface_dict.get("project_interface", None): + pipeline_types.append(PipelineLevel.PROJECT.value) + + if pipeline_types == []: + raise PipelineInterfaceConfigError( + f"sample_interface and/or project_interface must be defined in each pipeline interface." + ) - return pipeline_type, piface_path + return pipeline_types, piface_path def read_looper_config_file(looper_config_path: str) -> dict: @@ -575,19 +782,18 @@ def read_looper_config_file(looper_config_path: str) -> dict: :raise MisconfigurationException: incorrect configuration. """ return_dict = {} - with open(looper_config_path, "r") as dotfile: - dp_data = yaml.safe_load(dotfile) + + try: + with open(looper_config_path, "r") as dotfile: + dp_data = yaml.safe_load(dotfile) + except ParserError as e: + _LOGGER.warning( + "Could not load looper config file due to the following exception" + ) + raise ParserError(context=str(e)) if PEP_CONFIG_KEY in dp_data: - # Looper expects the config path to live at looper.config_file - # However, user may wish to access the pep at looper.pep_config - return_dict[PEP_CONFIG_FILE_KEY] = dp_data[PEP_CONFIG_KEY] return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY] - - # TODO: delete it in looper 2.0 - elif DOTFILE_CFG_PTH_KEY in dp_data: - return_dict[PEP_CONFIG_FILE_KEY] = dp_data[DOTFILE_CFG_PTH_KEY] - else: raise MisconfigurationException( f"Looper dotfile ({looper_config_path}) is missing '{PEP_CONFIG_KEY}' key" @@ -613,36 +819,25 @@ def read_looper_config_file(looper_config_path: str) -> dict: dp_data.setdefault(PIPELINE_INTERFACES_KEY, {}) - if isinstance(dp_data.get(PIPELINE_INTERFACES_KEY), dict) and ( - dp_data.get(PIPELINE_INTERFACES_KEY).get("sample") - or dp_data.get(PIPELINE_INTERFACES_KEY).get("project") - ): - # Support original nesting of pipeline interfaces under "sample" and "project" - return_dict[SAMPLE_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get( - "sample" - ) - return_dict[PROJECT_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get( - "project" + all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY) + + sample_pifaces = [] + project_pifaces = [] + if isinstance(all_pipeline_interfaces, str): + all_pipeline_interfaces = [all_pipeline_interfaces] + for piface in all_pipeline_interfaces: + pipeline_types, piface_path = determine_pipeline_type( + piface, looper_config_path ) - else: - # infer pipeline type based from interface instead of nested keys: https://github.com/pepkit/looper/issues/465 - all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY) - sample_pifaces = [] - project_pifaces = [] - if isinstance(all_pipeline_interfaces, str): - all_pipeline_interfaces = [all_pipeline_interfaces] - for piface in all_pipeline_interfaces: - pipeline_type, piface_path = determine_pipeline_type( - piface, looper_config_path - ) - if pipeline_type == PipelineLevel.SAMPLE.value: + if pipeline_types is not None: + if PipelineLevel.SAMPLE.value in pipeline_types: sample_pifaces.append(piface_path) - elif pipeline_type == PipelineLevel.PROJECT.value: + if PipelineLevel.PROJECT.value in pipeline_types: project_pifaces.append(piface_path) - if len(sample_pifaces) > 0: - return_dict[SAMPLE_PL_ARG] = sample_pifaces - if len(project_pifaces) > 0: - return_dict[PROJECT_PL_ARG] = project_pifaces + if len(sample_pifaces) > 0: + return_dict[SAMPLE_PL_ARG] = sample_pifaces + if len(project_pifaces) > 0: + return_dict[PROJECT_PL_ARG] = project_pifaces else: _LOGGER.warning( @@ -656,6 +851,7 @@ def read_looper_config_file(looper_config_path: str) -> dict: for k, v in return_dict.items(): if k == SAMPLE_PL_ARG or k == PROJECT_PL_ARG: # Pipeline interfaces are resolved at a later point. Do it there only to maintain consistency. #474 + pass if isinstance(v, str): v = expandpath(v) diff --git a/looper_init.py b/looper_init.py deleted file mode 100644 index 9d7a3c5f3..000000000 --- a/looper_init.py +++ /dev/null @@ -1,68 +0,0 @@ -# A simple utility, to be run in the root of a project, to prompt a user through -# configuring a .looper.yaml file for a new project. To be used as `looper init`. - -import os - -cfg = {} - -print("This utility will walk you through creating a .looper.yaml file.") -print("See `looper init --help` for details.") -print("Use `looper run` afterwards to run the pipeline.") -print("Press ^C at any time to quit.\n") - -looper_cfg_path = ".looper.yaml" # not changeable - -if os.path.exists(looper_cfg_path): - print(f"File exists at '{looper_cfg_path}'. Delete it to re-initialize.") - raise SystemExit - -DEFAULTS = { # What you get if you just press enter - "pep_config": "databio/example", - "output_dir": "results", - "piface_path": "pipeline_interface.yaml", - "project_name": os.path.basename(os.getcwd()), -} - - -cfg["project_name"] = ( - input(f"Project name: ({DEFAULTS['project_name']}) ") or DEFAULTS["project_name"] -) - -cfg["pep_config"] = ( - input(f"Registry path or file path to PEP: ({DEFAULTS['pep_config']}) ") - or DEFAULTS["pep_config"] -) - -if not os.path.exists(cfg["pep_config"]): - print(f"Warning: PEP file does not exist at '{cfg['pep_config']}'") - -cfg["output_dir"] = ( - input(f"Path to output directory: ({DEFAULTS['output_dir']}) ") - or DEFAULTS["output_dir"] -) - -# TODO: Right now this assumes you will have one pipeline interface, and a sample pipeline -# but this is not the only way you could configure things. - -piface_path = ( - input("Path to sample pipeline interface: (pipeline_interface.yaml) ") - or DEFAULTS["piface_path"] -) - -if not os.path.exists(piface_path): - print(f"Warning: file does not exist at {piface_path}") - -print(f"Writing config file to {looper_cfg_path}") -print(f"PEP path: {cfg['pep_config']}") -print(f"Pipeline interface path: {piface_path}") - - -with open(looper_cfg_path, "w") as fp: - fp.write( - f"""\ -pep_config: {cfg['pep_config']} -output_dir: {cfg['output_dir']} -pipeline_interfaces: - sample: {piface_path} -""" - ) diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index da94f546d..6533040a9 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -1,12 +1,11 @@ colorama>=0.3.9 -divvy>=0.5.0 -eido>=0.2.1 +eido>=0.2.4 jinja2 logmuse>=0.2.0 pandas>=2.0.2 pephubclient>=0.4.0 -pipestat>=0.9.2 -peppy>=0.40.0,<=0.40.2 +pipestat>=0.12.0a1 +peppy>=0.40.6 pyyaml>=3.12 rich>=9.10.0 ubiquerg>=0.8.1a1 diff --git a/setup.py b/setup.py index db8d94595..08c455ba4 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def get_static(name, condition=None): license="BSD2", entry_points={ "console_scripts": [ - "looper = looper.cli_pydantic:main", + "looper = looper.cli_pydantic:main_cli", "divvy = looper.__main__:divvy_main", ], }, diff --git a/tests/conftest.py b/tests/conftest.py index ef2176feb..960a98b44 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -141,7 +141,7 @@ def test_args_expansion(pth=None, cmd=None, appendix=list(), dry=True) -> List[s if cmd: x.append(cmd) if pth: - x.append("--looper-config") + x.append("--config") x.append(pth) if dry: x.append("--dry-run") @@ -204,7 +204,7 @@ def prep_temp_pep(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "advanced") + advanced_dir = os.path.join(d, "pytesting/advanced_test") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -220,7 +220,7 @@ def prep_temp_pep_basic(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "intermediate") + advanced_dir = os.path.join(d, "pytesting/intermediate_test") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -236,7 +236,7 @@ def prep_temp_pep_csv(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "csv") + advanced_dir = os.path.join(d, "looper_csv_example") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -274,7 +274,7 @@ def prep_temp_pep_pipestat(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "pipestat") + advanced_dir = os.path.join(d, "pytesting/pipestat_test") path_to_looper_config = os.path.join(advanced_dir, ".looper.yaml") return path_to_looper_config @@ -291,7 +291,7 @@ def prep_temp_pep_pipestat_advanced(example_pep_piface_path): d = tempfile.mkdtemp() shutil.copytree(hello_looper_dir_path, d, dirs_exist_ok=True) - advanced_dir = os.path.join(d, "advanced") + advanced_dir = os.path.join(d, "pytesting/advanced_test") path_to_looper_config = os.path.join(advanced_dir, ".looper_advanced_pipestat.yaml") return path_to_looper_config diff --git a/tests/data/hello_looper-dev/.looper.yaml b/tests/data/hello_looper-dev/.looper.yaml deleted file mode 100644 index e812a1ea8..000000000 --- a/tests/data/hello_looper-dev/.looper.yaml +++ /dev/null @@ -1,4 +0,0 @@ -pep_config: ./project/project_config.yaml # pephub registry path or local path -output_dir: "./results" -pipeline_interfaces: - sample: ../pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/README.md b/tests/data/hello_looper-dev/README.md index 6c213b1a9..fc4b86926 100644 --- a/tests/data/hello_looper-dev/README.md +++ b/tests/data/hello_looper-dev/README.md @@ -4,12 +4,11 @@ This repository provides minimal working examples for the [looper pipeline submi This repository contains examples -1. `/minimal` - A basic example pipeline and project. -2. `/intermediate` - An intermediate example pipeline and project with a couple extra options. -3. `/advanced` - A more advanced example, showcasing the capabilities of Looper. -4. `/pephub` - Example of how to point looper to PEPhub. -5. `/pipestat` - Example of a pipeline that uses pipestat for recording results. -6. `/csv` - How to use a pipeline with a CSV sample table (no YAML config) +1. `/looper_csv_example` - A minimal example using _only_ csv for metadata. +2. `/pep_derived_attrs` - An basic example utilizing the PEP specification for metadata and deriving attributes from the metadata +3. `/pephub` - Example of how to point looper to a PEP stored on PEPhub and running a pipeline. +4. `/pipestat_example` - Example on how to use pipestat to report pipeline results when using looper. +5. `/input_schema_example` - Example on how to use input schemas when using looper. Each example contains: @@ -17,4 +16,4 @@ Each example contains: 2. Sample data plus metadata in PEP format (or pointer to PEPhub). 3. A looper-compatible pipeline. -Explanation and results of running the above examples can be found at [Looper: Hello World](https://pep.databio.org/looper/code/hello-world/) +Explanation and results of running the above examples can be found at [Looper: User Tutorial](https://pep.databio.org/looper/user-tutorial/initialize/) diff --git a/tests/data/hello_looper-dev/advanced/.looper.yaml b/tests/data/hello_looper-dev/advanced/.looper.yaml deleted file mode 100644 index d2c5797f8..000000000 --- a/tests/data/hello_looper-dev/advanced/.looper.yaml +++ /dev/null @@ -1,10 +0,0 @@ -pep_config: project/project_config.yaml -output_dir: "results" -pipeline_interfaces: - sample: - - ../pipeline/pipeline_interface1_sample.yaml - - ../pipeline/pipeline_interface2_sample.yaml - project: - - ../pipeline/pipeline_interface1_project.yaml - - ../pipeline/pipeline_interface2_project.yaml - diff --git a/tests/data/hello_looper-dev/csv/.looper.yaml b/tests/data/hello_looper-dev/csv/.looper.yaml deleted file mode 100644 index c88f0c9a5..000000000 --- a/tests/data/hello_looper-dev/csv/.looper.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pep_config: project/sample_annotation.csv # local path to CSV -# pep_config: pepkit/hello_looper:default # you can also use a pephub registry path -output_dir: "results" -pipeline_interfaces: - sample: pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/csv/data/frog1_data.txt b/tests/data/hello_looper-dev/csv/data/frog1_data.txt deleted file mode 100644 index 815c0cf7c..000000000 --- a/tests/data/hello_looper-dev/csv/data/frog1_data.txt +++ /dev/null @@ -1,4 +0,0 @@ -ribbit -ribbit -ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/csv/data/frog2_data.txt b/tests/data/hello_looper-dev/csv/data/frog2_data.txt deleted file mode 100644 index e6fdd5350..000000000 --- a/tests/data/hello_looper-dev/csv/data/frog2_data.txt +++ /dev/null @@ -1,7 +0,0 @@ -ribbit -ribbit -ribbit - -ribbit, ribbit -ribbit, ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml deleted file mode 100644 index 732e69761..000000000 --- a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -pipeline_type: sample -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} {sample.file} diff --git a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml deleted file mode 100644 index 9063c7d61..000000000 --- a/tests/data/hello_looper-dev/csv/pipeline/pipeline_interface_project.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -pipeline_type: project -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} "data/*.txt" diff --git a/tests/data/hello_looper-dev/csv/project/sample_annotation.csv b/tests/data/hello_looper-dev/csv/project/sample_annotation.csv deleted file mode 100644 index 05bf4d172..000000000 --- a/tests/data/hello_looper-dev/csv/project/sample_annotation.csv +++ /dev/null @@ -1,3 +0,0 @@ -sample_name,library,file,toggle -frog_1,anySampleType,data/frog1_data.txt,1 -frog_2,anySampleType,data/frog2_data.txt,1 diff --git a/tests/data/hello_looper-dev/input_schema_example/.looper.yaml b/tests/data/hello_looper-dev/input_schema_example/.looper.yaml new file mode 100644 index 000000000..ba4eb61d0 --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/.looper.yaml @@ -0,0 +1,4 @@ +pep_config: metadata/pep_config.yaml +output_dir: results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/input_schema_example/data/canada.txt b/tests/data/hello_looper-dev/input_schema_example/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt b/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt b/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml b/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml new file mode 100644 index 000000000..45cefe21e --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/metadata/pep_config.yaml @@ -0,0 +1,9 @@ +pep_version: 2.1.0 +sample_table: sample_table.csv +sample_modifiers: + append: + file_path: source1 + derive: + attributes: [file_path] + sources: + source1: "data/{sample_name}.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv new file mode 100644 index 000000000..dcbd2366d --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/metadata/sample_table.csv @@ -0,0 +1,5 @@ +sample_name,area_type +mexico,state +switzerland,canton +canada,province +usa, \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh b/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh new file mode 100755 index 000000000..03675d99c --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/count_lines.sh @@ -0,0 +1,4 @@ +#!/bin/bash +linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` +export area_type=$2 +echo "Number of ${area_type}s: $linecount" diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml b/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml new file mode 100644 index 000000000..1de22ea3b --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/input_schema.yaml @@ -0,0 +1,26 @@ +description: An input schema for count_lines pipeline pipeline. +properties: + samples: + type: array + items: + type: object + properties: + sample_name: + type: string + description: "Name of the sample" + file_path: + type: string + description: "Path to the input file to count" + area_type: + type: string + description: "Name of the components of the country" + tangible: + - file_path + sizing: + - file_path + required: + - sample_name + - area_type + - file_path +required: + - samples diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..6f7b156bc --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/pipeline_interface.yaml @@ -0,0 +1,10 @@ +pipeline_name: count_lines +input_schema: input_schema.yaml +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} {sample.area_type} +project_interface: + command_template: > + python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/ +compute: + size_dependent_variables: resources-sample.tsv \ No newline at end of file diff --git a/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv new file mode 100644 index 000000000..6f0553ef6 --- /dev/null +++ b/tests/data/hello_looper-dev/input_schema_example/pipeline/resources-sample.tsv @@ -0,0 +1,4 @@ +max_file_size cores mem time +0.0005 1 1000 00-01:00:00 +0.05 2 2000 00-03:00:00 +NaN 4 4000 00-05:00:00 diff --git a/tests/data/hello_looper-dev/intermediate/.looper_project.yaml b/tests/data/hello_looper-dev/intermediate/.looper_project.yaml deleted file mode 100644 index b44ef03b7..000000000 --- a/tests/data/hello_looper-dev/intermediate/.looper_project.yaml +++ /dev/null @@ -1,4 +0,0 @@ -pep_config: project/project_config.yaml # local path to pep config -output_dir: "results" -pipeline_interfaces: - project: pipeline/pipeline_interface_project.yaml diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml deleted file mode 100644 index 732e69761..000000000 --- a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -pipeline_type: sample -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} {sample.file} diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml deleted file mode 100644 index 9063c7d61..000000000 --- a/tests/data/hello_looper-dev/intermediate/pipeline/pipeline_interface_project.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -pipeline_type: project -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} "data/*.txt" diff --git a/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml b/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml new file mode 100644 index 000000000..381d1819a --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/.looper.yaml @@ -0,0 +1,4 @@ +pep_config: metadata/sample_table.csv +output_dir: results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt b/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt b/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt b/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv new file mode 100644 index 000000000..6aa56d5bd --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/metadata/sample_table.csv @@ -0,0 +1,4 @@ +sample_name,area_type,file_path +mexico,state,data/mexico.txt +switzerland,canton,data/switzerland.txt +canada,province,data/canada.txt \ No newline at end of file diff --git a/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh b/tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh similarity index 69% rename from tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh rename to tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh index 71b887fe7..0a39a4a87 100755 --- a/tests/data/hello_looper-dev/intermediate/pipeline/count_lines.sh +++ b/tests/data/hello_looper-dev/looper_csv_example/pipeline/count_lines.sh @@ -1,3 +1,3 @@ #!/bin/bash linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` -echo "Number of lines: $linecount" +echo "Number of lines: $linecount" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..dc58569b8 --- /dev/null +++ b/tests/data/hello_looper-dev/looper_csv_example/pipeline/pipeline_interface.yaml @@ -0,0 +1,4 @@ +pipeline_name: count_lines +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} diff --git a/tests/data/hello_looper-dev/minimal/.looper.yaml b/tests/data/hello_looper-dev/minimal/.looper.yaml deleted file mode 100644 index 19fac81d4..000000000 --- a/tests/data/hello_looper-dev/minimal/.looper.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pep_config: project/project_config.yaml # local path to pep config -# pep_config: pepkit/hello_looper:default # you can also use a pephub registry path -output_dir: "results" -pipeline_interfaces: - sample: pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml deleted file mode 100644 index 732e69761..000000000 --- a/tests/data/hello_looper-dev/minimal/pipeline/pipeline_interface.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -pipeline_type: sample -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} {sample.file} diff --git a/tests/data/hello_looper-dev/minimal/project/project_config.yaml b/tests/data/hello_looper-dev/minimal/project/project_config.yaml deleted file mode 100644 index 5456cca30..000000000 --- a/tests/data/hello_looper-dev/minimal/project/project_config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -pep_version: 2.0.0 -sample_table: sample_annotation.csv \ No newline at end of file diff --git a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv b/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv deleted file mode 100644 index 97f223700..000000000 --- a/tests/data/hello_looper-dev/minimal/project/sample_annotation.csv +++ /dev/null @@ -1,3 +0,0 @@ -sample_name,library,file,toggle -frog_1,anySampleType,data/frog_1.txt,1 -frog_2,anySampleType,data/frog_2.txt,1 diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml new file mode 100644 index 000000000..ba4eb61d0 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/.looper.yaml @@ -0,0 +1,4 @@ +pep_config: metadata/pep_config.yaml +output_dir: results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt b/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml new file mode 100644 index 000000000..45cefe21e --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/pep_config.yaml @@ -0,0 +1,9 @@ +pep_version: 2.1.0 +sample_table: sample_table.csv +sample_modifiers: + append: + file_path: source1 + derive: + attributes: [file_path] + sources: + source1: "data/{sample_name}.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv new file mode 100644 index 000000000..85f38a908 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/metadata/sample_table.csv @@ -0,0 +1,4 @@ +sample_name,area_type +mexico,state +switzerland,canton +canada,province \ No newline at end of file diff --git a/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh similarity index 69% rename from tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh rename to tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh index 71b887fe7..0a39a4a87 100755 --- a/tests/data/hello_looper-dev/minimal/pipeline/count_lines.sh +++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines.sh @@ -1,3 +1,3 @@ #!/bin/bash linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` -echo "Number of lines: $linecount" +echo "Number of lines: $linecount" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py new file mode 100644 index 000000000..398c1c02a --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/count_lines_plot.py @@ -0,0 +1,32 @@ +import matplotlib.pyplot as plt +import os +import sys + +results_dir = sys.argv[ + 1 +] # Obtain the looper results directory passed via the looper command template + +# Extract the previously reported sample-level data from the .log files +countries = [] +number_of_regions = [] +for filename in os.listdir(results_dir): + if filename.endswith(".log"): + file = os.path.join(results_dir, filename) + with open(file, "r") as f: + for line in f: + if line.startswith("Number of lines:"): + region_count = int(line.split(":")[1].strip()) + number_of_regions.append(region_count) + country = filename.split("_")[2].split(".")[0] + countries.append(country) + +# Create a bar chart of regions per country +plt.figure(figsize=(8, 5)) +plt.bar(countries, number_of_regions, color=["blue", "green", "purple"]) +plt.xlabel("Countries") +plt.ylabel("Number of regions") +plt.title("Number of regions per country") + +# Save the image locally +save_location = os.path.join(os.path.dirname(results_dir), "regions_per_country.png") +plt.savefig(save_location, dpi=150) diff --git a/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..dacc65891 --- /dev/null +++ b/tests/data/hello_looper-dev/pep_derived_attrs/pipeline/pipeline_interface.yaml @@ -0,0 +1,7 @@ +pipeline_name: count_lines +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} +project_interface: + command_template: > + python3 {looper.piface_dir}/count_lines_plot.py {looper.output_dir}/submission/ diff --git a/tests/data/hello_looper-dev/pephub/.looper.yaml b/tests/data/hello_looper-dev/pephub/.looper.yaml index 00e60ded6..654c5427e 100644 --- a/tests/data/hello_looper-dev/pephub/.looper.yaml +++ b/tests/data/hello_looper-dev/pephub/.looper.yaml @@ -1,4 +1,4 @@ -pep_config: pepkit/hello_looper:default # pephub registry path or local path +pep_config: databio/hello_looper:default output_dir: results pipeline_interfaces: - sample: pipeline/pipeline_interface.yaml + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/pephub/data/canada.txt b/tests/data/hello_looper-dev/pephub/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/pephub/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/pephub/data/frog1_data.txt b/tests/data/hello_looper-dev/pephub/data/frog1_data.txt deleted file mode 100644 index 815c0cf7c..000000000 --- a/tests/data/hello_looper-dev/pephub/data/frog1_data.txt +++ /dev/null @@ -1,4 +0,0 @@ -ribbit -ribbit -ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pephub/data/frog2_data.txt b/tests/data/hello_looper-dev/pephub/data/frog2_data.txt deleted file mode 100644 index e6fdd5350..000000000 --- a/tests/data/hello_looper-dev/pephub/data/frog2_data.txt +++ /dev/null @@ -1,7 +0,0 @@ -ribbit -ribbit -ribbit - -ribbit, ribbit -ribbit, ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pephub/data/mexico.txt b/tests/data/hello_looper-dev/pephub/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/pephub/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pephub/data/switzerland.txt b/tests/data/hello_looper-dev/pephub/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/pephub/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml index 732e69761..dc58569b8 100644 --- a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml +++ b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface.yaml @@ -1,6 +1,4 @@ pipeline_name: count_lines -pipeline_type: sample -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} {sample.file} +sample_interface: + command_template: > + pipeline/count_lines.sh {sample.file_path} diff --git a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml deleted file mode 100644 index 9063c7d61..000000000 --- a/tests/data/hello_looper-dev/pephub/pipeline/pipeline_interface_project.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pipeline_name: count_lines -pipeline_type: project -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} "data/*.txt" diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt b/tests/data/hello_looper-dev/pipestat/data/frog_1.txt deleted file mode 100644 index 815c0cf7c..000000000 --- a/tests/data/hello_looper-dev/pipestat/data/frog_1.txt +++ /dev/null @@ -1,4 +0,0 @@ -ribbit -ribbit -ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt b/tests/data/hello_looper-dev/pipestat/data/frog_2.txt deleted file mode 100644 index e6fdd5350..000000000 --- a/tests/data/hello_looper-dev/pipestat/data/frog_2.txt +++ /dev/null @@ -1,7 +0,0 @@ -ribbit -ribbit -ribbit - -ribbit, ribbit -ribbit, ribbit -CROAK! diff --git a/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml b/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml deleted file mode 100644 index 0a04ac6f9..000000000 --- a/tests/data/hello_looper-dev/pipestat/looper_pipestat_config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -results_file_path: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results.yaml -flag_file_dir: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results/flags -output_dir: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./results -record_identifier: frog_2 -schema_path: /home/drc/GITHUB/hello_looper/hello_looper/pipestat/./pipeline_pipestat/pipestat_output_schema.yaml -pipeline_name: test_pipe -pipeline_type: sample diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml deleted file mode 100644 index e5a144027..000000000 --- a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pipeline_name: example_pipestat_pipeline -pipeline_type: sample -output_schema: pipestat_output_schema.yaml -command_template: > - python3 {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml deleted file mode 100644 index 2237c2f39..000000000 --- a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_project.yaml +++ /dev/null @@ -1,8 +0,0 @@ -pipeline_name: example_pipestat_project_pipeline -pipeline_type: project -output_schema: pipestat_output_schema.yaml -var_templates: - pipeline: '{looper.piface_dir}/count_lines.sh' -command_template: > - {pipeline.var_templates.pipeline} "data/*.txt" - diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml deleted file mode 100644 index 82df8b942..000000000 --- a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipeline_interface_shell.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pipeline_name: example_pipestat_pipeline -pipeline_type: sample -output_schema: pipestat_output_schema.yaml -command_template: > - {looper.piface_dir}/count_lines_pipestat.sh {sample.file} {sample.sample_name} {pipestat.config_file} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/.looper.yaml b/tests/data/hello_looper-dev/pipestat_example/.looper.yaml new file mode 100644 index 000000000..dba1fbe9a --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/.looper.yaml @@ -0,0 +1,8 @@ +pep_config: ./metadata/pep_config.yaml # pephub registry path or local path +output_dir: ./results +pipeline_interfaces: + - pipeline/pipeline_interface.yaml +pipestat: + project_name: count_lines + results_file_path: results.yaml + flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml new file mode 100644 index 000000000..29a397194 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/.looper_pipestat_shell.yaml @@ -0,0 +1,7 @@ +pep_config: ./metadata/pep_config.yaml # pephub registry path or local path +output_dir: ./results +pipeline_interfaces: + - pipeline/pipeline_interface_shell.yaml +pipestat: + results_file_path: results.yaml + flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/data/canada.txt b/tests/data/hello_looper-dev/pipestat_example/data/canada.txt new file mode 100644 index 000000000..dd70801c1 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/data/canada.txt @@ -0,0 +1,10 @@ +Alberta +British Columbia +Manitoba +New Brunswick +Newfoundland and Labrador +Nova Scotia +Ontario +Prince Edward Island +Quebec +Saskatchewan diff --git a/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt b/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt new file mode 100644 index 000000000..66cf49d5c --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/data/mexico.txt @@ -0,0 +1,31 @@ +Aguascalientes +Baja California +Baja California Sur +Campeche +Chiapas +Chihuahua +Coahuila +Colima +Durango +Guanajuato +Guerrero +Hidalgo +Jalisco +México (Estado de México) +Michoacán +Morelos +Nayarit +Nuevo León +Oaxaca +Puebla +Querétaro +Quintana Roo +San Luis Potosí +Sinaloa +Sonora +Tabasco +Tamaulipas +Tlaxcala +Veracruz +Yucatán +Zacatecas \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt b/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt new file mode 100644 index 000000000..b7159d55f --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/data/switzerland.txt @@ -0,0 +1,23 @@ +Zürich +Bern +Luzern +Uri +Schwyz +Unterwalden +Glarus +Zug +Freiburg +Solothurn +Basel +Schaffhausen +Appenzell +Sankt Gallen +Graubünden +Aargau +Thurgau +Ticino +Vaud +Valais +Neuchâtel +Genève +Jura diff --git a/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml b/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml new file mode 100644 index 000000000..45cefe21e --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/metadata/pep_config.yaml @@ -0,0 +1,9 @@ +pep_version: 2.1.0 +sample_table: sample_table.csv +sample_modifiers: + append: + file_path: source1 + derive: + attributes: [file_path] + sources: + source1: "data/{sample_name}.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv b/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv new file mode 100644 index 000000000..85f38a908 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/metadata/sample_table.csv @@ -0,0 +1,4 @@ +sample_name,area_type +mexico,state +switzerland,canton +canada,province \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.py similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines.py rename to tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.py diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.sh similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/count_lines_pipestat.sh rename to tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines.sh diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py new file mode 100644 index 000000000..bc3a2bce3 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/count_lines_plot.py @@ -0,0 +1,45 @@ +import matplotlib.pyplot as plt # be sure to `pip install matplotlib` +import os +import pipestat +import sys + +# A pipeline that retrieves previously reported pipestat results +# and plots them in a bar chart +results_file = sys.argv[1] +schema_path = sys.argv[2] + +# Create pipestat manager +psm = pipestat.PipestatManager( + schema_path=schema_path, results_file_path=results_file, pipeline_type="project" +) + +# Extract the previously reported data +results = ( + psm.select_records() +) # pipestat object holds the data after reading the results file +countries = [record["record_identifier"] for record in results["records"]] +number_of_regions = [record["number_of_lines"] for record in results["records"]] + +# Create a bar chart of regions per country +plt.figure(figsize=(8, 5)) +plt.bar(countries, number_of_regions, color=["blue", "green", "purple"]) +plt.xlabel("Countries") +plt.ylabel("Number of regions") +plt.title("Number of regions per country") +# plt.show() # Showing the figure and then saving it causes issues, so leave this commented out. + +# Save the image locally AND report that location via pipestat +# we can place it next to the results file for now +save_location = os.path.join(os.path.dirname(results_file), "regions_per_country.png") + +plt.savefig(save_location, dpi=150) + +result_to_report = { + "regions_plot": { + "path": save_location, + "thumbnail_path": save_location, + "title": "Regions Plot", + } +} + +psm.report(record_identifier="count_lines", values=result_to_report) diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..275af085a --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface.yaml @@ -0,0 +1,8 @@ +pipeline_name: count_lines +output_schema: pipestat_output_schema.yaml +sample_interface: + command_template: > + python3 {looper.piface_dir}/count_lines.py {sample.file_path} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema} +project_interface: + command_template: > + python3 {looper.piface_dir}/count_lines_plot.py {pipestat.results_file} {pipestat.output_schema} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml new file mode 100644 index 000000000..c3b930fb3 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipeline_interface_shell.yaml @@ -0,0 +1,5 @@ +pipeline_name: count_lines +output_schema: pipestat_output_schema.yaml +sample_interface: + command_template: > + {looper.piface_dir}/count_lines.sh {sample.file_path} {sample.sample_name} {pipestat.config_file} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml new file mode 100644 index 000000000..be8007a57 --- /dev/null +++ b/tests/data/hello_looper-dev/pipestat_example/pipeline/pipestat_output_schema.yaml @@ -0,0 +1,31 @@ +title: Pipestat output schema for counting lines +description: A pipeline that uses pipestat to report sample level results. +type: object +properties: + pipeline_name: count_lines + samples: + type: array + items: + type: object + properties: + number_of_lines: + type: integer + description: "Number of lines in the input file." + project: + type: object + properties: + regions_plot: + description: "This a path to the output image" + type: object + object_type: image + properties: + path: + type: string + thumbnail_path: + type: string + title: + type: string + required: + - path + - thumbnail_path + - title \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pytesting/README.md b/tests/data/hello_looper-dev/pytesting/README.md new file mode 100644 index 000000000..1f0c5cc7a --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/README.md @@ -0,0 +1 @@ +The examples in this pytesting directory are for Looper testing purposes. \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml new file mode 100644 index 000000000..26942bd98 --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper.yaml @@ -0,0 +1,8 @@ +pep_config: project/project_config.yaml +output_dir: "results" +pipeline_interfaces: + - pipeline/pipeline_interface1_sample.yaml + - pipeline/pipeline_interface2_sample.yaml + - pipeline/pipeline_interface1_project.yaml + - pipeline/pipeline_interface2_project.yaml + diff --git a/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml similarity index 55% rename from tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml index 74da1a3fb..3c4963c20 100644 --- a/tests/data/hello_looper-dev/advanced/.looper_advanced_pipestat.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/.looper_advanced_pipestat.yaml @@ -1,9 +1,8 @@ pep_config: project/project_config.yaml output_dir: "results" pipeline_interfaces: - sample: - - ../pipeline/pipestat_pipeline_interface1_sample.yaml - - ../pipeline/pipestat_pipeline_interface2_sample.yaml + - pipeline/pipestat_pipeline_interface1_sample.yaml + - pipeline/pipestat_pipeline_interface2_sample.yaml pipestat: results_file_path: results.yaml flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/advanced/pipeline/col_pipeline1.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline1.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/col_pipeline1.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline1.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/col_pipeline2.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline2.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/col_pipeline2.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/col_pipeline2.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/other_pipeline2.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/other_pipeline2.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/other_pipeline2.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/other_pipeline2.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/output_schema.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/output_schema.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/output_schema.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline1.py b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline1.py similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline1.py rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline1.py diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml similarity index 52% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml index 2a23d3214..534905cad 100644 --- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_project.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_project.yaml @@ -1,9 +1,9 @@ pipeline_name: PIPELINE1 -pipeline_type: project output_schema: output_schema.yaml var_templates: path: "{looper.piface_dir}/col_pipeline1.py" -command_template: > - python3 {pipeline.var_templates.path} --project-name {project.name} +project_interface: + command_template: > + python3 {pipeline.var_templates.path} --project-name {project.name} diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml similarity index 63% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml index 8e79b7ae7..d0d608498 100644 --- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface1_sample.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface1_sample.yaml @@ -1,5 +1,4 @@ pipeline_name: PIPELINE1 -pipeline_type: sample input_schema: https://schema.databio.org/pep/2.0.0.yaml output_schema: output_schema.yaml var_templates: @@ -7,7 +6,8 @@ var_templates: pre_submit: python_functions: - looper.write_sample_yaml -command_template: > - python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} +sample_interface: + command_template: > + python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml similarity index 62% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml index 824b7e09b..df557d820 100644 --- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_project.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_project.yaml @@ -1,10 +1,10 @@ pipeline_name: OTHER_PIPELINE2 -pipeline_type: project output_schema: output_schema.yaml var_templates: path: "{looper.piface_dir}/col_pipeline2.py" -command_template: > - python3 {pipeline.var_templates.path} --project-name {project.name} +project_interface: + command_template: > + python3 {pipeline.var_templates.path} --project-name {project.name} compute: size_dependent_variables: resources-project.tsv diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml similarity index 64% rename from tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml index 589aef6dc..0329d33a2 100644 --- a/tests/data/hello_looper-dev/advanced/pipeline/pipeline_interface2_sample.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipeline_interface2_sample.yaml @@ -1,13 +1,13 @@ pipeline_name: OTHER_PIPELINE2 -pipeline_type: sample output_schema: output_schema.yaml var_templates: path: "{looper.piface_dir}/other_pipeline2.py" pre_submit: python_functions: - looper.write_sample_yaml -command_template: > - python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} +sample_interface: + command_template: > + python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} compute: size_dependent_variables: resources-sample.tsv diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_output_schema.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_output_schema.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_output_schema.yaml diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml similarity index 65% rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml index e687ea0d3..4bdbab1fc 100644 --- a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface1_sample.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface1_sample.yaml @@ -1,5 +1,4 @@ pipeline_name: example_pipestat_pipeline -pipeline_type: sample input_schema: https://schema.databio.org/pep/2.0.0.yaml output_schema: pipestat_output_schema.yaml var_templates: @@ -7,7 +6,8 @@ var_templates: pre_submit: python_functions: - looper.write_sample_yaml -command_template: > - python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} +sample_interface: + command_template: > + python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} diff --git a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml similarity index 69% rename from tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml index bac3ea3d4..3fa6829c5 100644 --- a/tests/data/hello_looper-dev/advanced/pipeline/pipestat_pipeline_interface2_sample.yaml +++ b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/pipestat_pipeline_interface2_sample.yaml @@ -1,5 +1,4 @@ pipeline_name: example_pipestat_pipeline -pipeline_type: sample input_schema: https://schema.databio.org/pep/2.0.0.yaml output_schema: pipestat_output_schema.yaml var_templates: @@ -7,8 +6,9 @@ var_templates: pre_submit: python_functions: - looper.write_sample_yaml -command_template: > - python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} +sample_interface: + command_template: > + python3 {pipeline.var_templates.path} --sample-name {sample.sample_name} --req-attr {sample.attr} compute: size_dependent_variables: resources-sample.tsv diff --git a/tests/data/hello_looper-dev/advanced/pipeline/readData.R b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/readData.R similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/readData.R rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/readData.R diff --git a/tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-project.tsv similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/resources-project.tsv rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-project.tsv diff --git a/tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv b/tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-sample.tsv similarity index 100% rename from tests/data/hello_looper-dev/advanced/pipeline/resources-sample.tsv rename to tests/data/hello_looper-dev/pytesting/advanced_test/pipeline/resources-sample.tsv diff --git a/tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv b/tests/data/hello_looper-dev/pytesting/advanced_test/project/annotation_sheet.csv similarity index 100% rename from tests/data/hello_looper-dev/advanced/project/annotation_sheet.csv rename to tests/data/hello_looper-dev/pytesting/advanced_test/project/annotation_sheet.csv diff --git a/tests/data/hello_looper-dev/advanced/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/advanced_test/project/project_config.yaml similarity index 100% rename from tests/data/hello_looper-dev/advanced/project/project_config.yaml rename to tests/data/hello_looper-dev/pytesting/advanced_test/project/project_config.yaml diff --git a/tests/data/hello_looper-dev/intermediate/.looper.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml similarity index 81% rename from tests/data/hello_looper-dev/intermediate/.looper.yaml rename to tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml index 19fac81d4..4fcf56725 100644 --- a/tests/data/hello_looper-dev/intermediate/.looper.yaml +++ b/tests/data/hello_looper-dev/pytesting/intermediate_test/.looper.yaml @@ -2,4 +2,4 @@ pep_config: project/project_config.yaml # local path to pep config # pep_config: pepkit/hello_looper:default # you can also use a pephub registry path output_dir: "results" pipeline_interfaces: - sample: pipeline/pipeline_interface.yaml + - pipeline/pipeline_interface.yaml diff --git a/tests/data/hello_looper-dev/intermediate/data/frog_1.txt b/tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_1.txt similarity index 100% rename from tests/data/hello_looper-dev/intermediate/data/frog_1.txt rename to tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_1.txt diff --git a/tests/data/hello_looper-dev/intermediate/data/frog_2.txt b/tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_2.txt similarity index 100% rename from tests/data/hello_looper-dev/intermediate/data/frog_2.txt rename to tests/data/hello_looper-dev/pytesting/intermediate_test/data/frog_2.txt diff --git a/tests/data/hello_looper-dev/csv/pipeline/count_lines.sh b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/count_lines.sh similarity index 100% rename from tests/data/hello_looper-dev/csv/pipeline/count_lines.sh rename to tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/count_lines.sh diff --git a/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml new file mode 100644 index 000000000..dde7c3937 --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/intermediate_test/pipeline/pipeline_interface.yaml @@ -0,0 +1,9 @@ +pipeline_name: count_lines +var_templates: + pipeline: '{looper.piface_dir}/count_lines.sh' +sample_interface: + command_template: > + {pipeline.var_templates.pipeline} {sample.file} +project_interface: + command_template: > + {pipeline.var_templates.pipeline} "data/*.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/intermediate/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/intermediate_test/project/project_config.yaml similarity index 100% rename from tests/data/hello_looper-dev/intermediate/project/project_config.yaml rename to tests/data/hello_looper-dev/pytesting/intermediate_test/project/project_config.yaml diff --git a/tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv b/tests/data/hello_looper-dev/pytesting/intermediate_test/project/sample_annotation.csv similarity index 100% rename from tests/data/hello_looper-dev/intermediate/project/sample_annotation.csv rename to tests/data/hello_looper-dev/pytesting/intermediate_test/project/sample_annotation.csv diff --git a/tests/data/hello_looper-dev/pipestat/.looper.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml similarity index 62% rename from tests/data/hello_looper-dev/pipestat/.looper.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml index 852c6fa41..3fa1947c4 100644 --- a/tests/data/hello_looper-dev/pipestat/.looper.yaml +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper.yaml @@ -1,8 +1,7 @@ pep_config: ./project/project_config.yaml # pephub registry path or local path output_dir: ./results pipeline_interfaces: - sample: ./pipeline_pipestat/pipeline_interface.yaml - project: ./pipeline_pipestat/pipeline_interface_project.yaml + - pipeline_pipestat/pipeline_interface.yaml pipestat: results_file_path: results.yaml flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml similarity index 76% rename from tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml index fb645a9bd..6ecbdfc16 100644 --- a/tests/data/hello_looper-dev/pipestat/.looper_pipestat_shell.yaml +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/.looper_pipestat_shell.yaml @@ -1,7 +1,7 @@ pep_config: ./project/project_config.yaml # pephub registry path or local path output_dir: ./results pipeline_interfaces: - sample: ./pipeline_pipestat/pipeline_interface_shell.yaml + - pipeline_pipestat/pipeline_interface_shell.yaml pipestat: results_file_path: results.yaml flag_file_dir: results/flags \ No newline at end of file diff --git a/tests/data/hello_looper-dev/minimal/data/frog_1.txt b/tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_1.txt similarity index 100% rename from tests/data/hello_looper-dev/minimal/data/frog_1.txt rename to tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_1.txt diff --git a/tests/data/hello_looper-dev/minimal/data/frog_2.txt b/tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_2.txt similarity index 100% rename from tests/data/hello_looper-dev/minimal/data/frog_2.txt rename to tests/data/hello_looper-dev/pytesting/pipestat_test/data/frog_2.txt diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py new file mode 100755 index 000000000..6f6a4ab8f --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines.py @@ -0,0 +1,31 @@ +import os.path + +import pipestat +import sys + +# Very simple pipeline that calls pipestat +# takes arguments invoked during looper submission via command templates +text_file = sys.argv[ + 1 +] # this is the sample we wish to process by reading the number of lines +sample_name = sys.argv[2] +results_file = sys.argv[3] +schema_path = sys.argv[4] + +# Create pipestat manager and then report values +psm = pipestat.PipestatManager( + schema_path=schema_path, + results_file_path=results_file, + record_identifier=sample_name, +) + + +text_file = os.path.abspath(text_file) +# Read text file and count lines +with open(text_file, "r") as f: + result = {"number_of_lines": len(f.readlines())} + +# The results are defined in the pipestat output schema. +psm.report(record_identifier=sample_name, values=result) + +# end of pipeline diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh new file mode 100755 index 000000000..99f83f906 --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/count_lines_pipestat.sh @@ -0,0 +1,4 @@ +#!/bin/bash +linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '` +pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3 +echo "Number of lines: $linecount" diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml new file mode 100644 index 000000000..ee9a8d5c2 --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface.yaml @@ -0,0 +1,8 @@ +pipeline_name: example_pipestat_pipeline +output_schema: pipestat_output_schema.yaml +sample_interface: + command_template: > + python3 {looper.piface_dir}/count_lines.py {sample.file} {sample.sample_name} {pipestat.results_file} {pipestat.output_schema} +project_interface: + command_template: > + {pipeline.var_templates.pipeline} "data/*.txt" \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml new file mode 100644 index 000000000..2fed02857 --- /dev/null +++ b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipeline_interface_shell.yaml @@ -0,0 +1,5 @@ +pipeline_name: example_pipestat_pipeline +output_schema: pipestat_output_schema.yaml +sample_interface: + command_template: > + {looper.piface_dir}/count_lines_pipestat.sh {sample.file} {sample.sample_name} {pipestat.config_file} \ No newline at end of file diff --git a/tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipestat_output_schema.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/pipeline_pipestat/pipestat_output_schema.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/pipeline_pipestat/pipestat_output_schema.yaml diff --git a/tests/data/hello_looper-dev/pipestat/project/project_config.yaml b/tests/data/hello_looper-dev/pytesting/pipestat_test/project/project_config.yaml similarity index 100% rename from tests/data/hello_looper-dev/pipestat/project/project_config.yaml rename to tests/data/hello_looper-dev/pytesting/pipestat_test/project/project_config.yaml diff --git a/tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv b/tests/data/hello_looper-dev/pytesting/pipestat_test/project/sample_annotation.csv similarity index 100% rename from tests/data/hello_looper-dev/pipestat/project/sample_annotation.csv rename to tests/data/hello_looper-dev/pytesting/pipestat_test/project/sample_annotation.csv diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py index f7795696a..5770661f7 100644 --- a/tests/divvytests/test_divvy_simple.py +++ b/tests/divvytests/test_divvy_simple.py @@ -4,7 +4,7 @@ from collections import OrderedDict from yacman import YacAttMap -from divvy import select_divvy_config +from looper.divvy import select_divvy_config # For interactive debugging: # import logmuse diff --git a/tests/smoketests/test_other.py b/tests/smoketests/test_other.py index b90e9b61b..bc23bfb64 100644 --- a/tests/smoketests/test_other.py +++ b/tests/smoketests/test_other.py @@ -3,7 +3,11 @@ import pytest from peppy import Project -from looper.exceptions import PipestatConfigurationException, MisconfigurationException +from looper.exceptions import ( + PipestatConfigurationException, + MisconfigurationException, + LooperReportError, +) from tests.conftest import * from looper.cli_pydantic import main import pandas as pd @@ -64,7 +68,7 @@ class TestLooperPipestat: def test_fail_no_pipestat_config(self, prep_temp_pep, cmd): "report, table, and check should fail if pipestat is NOT configured." tp = prep_temp_pep - x = [cmd, "--looper-config", tp] + x = [cmd, "--config", tp] with pytest.raises(PipestatConfigurationException): main(test_args=x) @@ -73,17 +77,23 @@ def test_pipestat_configured(self, prep_temp_pep_pipestat, cmd): tp = prep_temp_pep_pipestat if cmd in ["run", "runp"]: - x = [cmd, "--looper-config", tp, "--dry-run"] + x = [cmd, "--config", tp, "--dry-run"] else: # Not every command supports dry run - x = [cmd, "--looper-config", tp] - - try: - result = main(test_args=x) - if cmd == "run": - assert result["Pipestat compatible"] is True - except Exception: - raise pytest.fail("DID RAISE {0}".format(Exception)) + x = [cmd, "--config", tp] + + if cmd not in ["report"]: + try: + result = main(test_args=x) + if cmd == "run": + assert result["Pipestat compatible"] is True + except Exception: + raise pytest.fail("DID RAISE {0}".format(Exception)) + else: + with pytest.raises( + expected_exception=LooperReportError + ): # Looper report will and should raise exception if there are no results reported. + result = main(test_args=x) class TestLooperRerun: @@ -120,7 +130,7 @@ def test_pipestat_rerun(self, prep_temp_pep_pipestat, pipeline_name, flags): with open(pipestat_project_file, "w") as f: dump(pipestat_project_data, f) - x = ["rerun", "--looper-config", tp] + x = ["rerun", "--config", tp] try: result = main(test_args=x) except Exception: @@ -137,7 +147,7 @@ def test_rerun_no_pipestat(self, prep_temp_pep, pipeline_name, flags): tp = prep_temp_pep _make_flags(tp, flags, pipeline_name) - x = ["rerun", "--looper-config", tp] + x = ["rerun", "--config", tp] try: result = main(test_args=x) except Exception: @@ -157,7 +167,7 @@ def test_check_works(self, prep_temp_pep_pipestat, flag_id, pipeline_name): tp = prep_temp_pep_pipestat _make_flags_pipestat(tp, flag_id, pipeline_name) - x = ["check", "--looper-config", tp] + x = ["check", "--config", tp] try: results = main(test_args=x) @@ -176,7 +186,7 @@ def test_check_multi(self, prep_temp_pep_pipestat, flag_id, pipeline_name): _make_flags_pipestat(tp, flag_id, pipeline_name) _make_flags_pipestat(tp, FLAGS[1], pipeline_name) - x = ["check", "--looper-config", tp] + x = ["check", "--config", tp] # Multiple flag files SHOULD cause pipestat to throw an assertion error if flag_id != FLAGS[1]: with pytest.raises(AssertionError): @@ -189,7 +199,7 @@ def test_check_bogus(self, prep_temp_pep_pipestat, flag_id, pipeline_name): tp = prep_temp_pep_pipestat _make_flags_pipestat(tp, flag_id, pipeline_name) - x = ["check", "--looper-config", tp] + x = ["check", "--config", tp] try: results = main(test_args=x) result_key = list(results.keys())[0] @@ -232,7 +242,7 @@ def test_selecting_flags_works( f.write(FLAGS[count]) count += 1 - x = ["run", "--looper-config", tp, "--sel-flag", "completed", "--dry-run"] + x = ["run", "--config", tp, "--sel-flag", "completed", "--dry-run"] try: results = main(test_args=x) @@ -273,7 +283,7 @@ def test_excluding_flags_works( f.write(FLAGS[count]) count += 1 - x = ["run", "--looper-config", tp, "--exc-flag", "running", "--dry-run"] + x = ["run", "--config", tp, "--exc-flag", "running", "--dry-run"] try: results = main(test_args=x) @@ -318,7 +328,7 @@ def test_excluding_multi_flags_works( x = [ "run", - "--looper-config", + "--config", tp, "--exc-flag", "completed", @@ -368,7 +378,7 @@ def test_selecting_multi_flags_works( x = [ "run", "--dry-run", - "--looper-config", + "--config", tp, "--sel-flag", "completed", @@ -419,7 +429,7 @@ def test_selecting_attr_and_flags_works( x = [ "run", "--dry-run", - "--looper-config", + "--config", tp, "--sel-flag", "completed", @@ -472,7 +482,7 @@ def test_excluding_attr_and_flags_works( x = [ "run", "--dry-run", - "--looper-config", + "--config", tp, "--exc-flag", "completed", @@ -534,7 +544,7 @@ def test_excluding_toggle_attr( x = [ "run", "--dry-run", - "--looper-config", + "--config", tp, "--sel-attr", "toggle", @@ -596,7 +606,7 @@ def test_including_toggle_attr( x = [ "run", "--dry-run", - "--looper-config", + "--config", tp, "--sel-attr", "toggle", @@ -620,7 +630,7 @@ class TestLooperInspect: def test_inspect_config(self, prep_temp_pep, cmd): "Checks inspect command" tp = prep_temp_pep - x = [cmd, "--looper-config", tp] + x = [cmd, "--config", tp] try: results = main(test_args=x) except Exception: diff --git a/tests/smoketests/test_run.py b/tests/smoketests/test_run.py index 6c0bdd8d8..c35d59470 100644 --- a/tests/smoketests/test_run.py +++ b/tests/smoketests/test_run.py @@ -16,7 +16,7 @@ def test_cli(prep_temp_pep): tp = prep_temp_pep - x = ["run", "--looper-config", tp, "--dry-run"] + x = ["run", "--config", tp, "--dry-run"] try: main(test_args=x) except Exception: @@ -26,20 +26,20 @@ def test_cli(prep_temp_pep): def test_cli_shortform(prep_temp_pep): tp = prep_temp_pep - x = ["run", "--looper-config", tp, "-d"] + x = ["run", "--config", tp, "-d"] try: main(test_args=x) except Exception: raise pytest.fail("DID RAISE {0}".format(Exception)) - x = ["run", "--looper-config", tp, "-d", "-l", "2"] + x = ["run", "--config", tp, "-d", "-l", "2"] try: main(test_args=x) except Exception: raise pytest.fail("DID RAISE {0}".format(Exception)) tp = prep_temp_pep - x = ["run", "--looper-config", tp, "-d", "-n", "2"] + x = ["run", "--config", tp, "-d", "-n", "2"] try: main(test_args=x) except Exception: @@ -49,7 +49,7 @@ def test_cli_shortform(prep_temp_pep): def test_running_csv_pep(prep_temp_pep_csv): tp = prep_temp_pep_csv - x = ["run", "--looper-config", tp, "--dry-run"] + x = ["run", "--config", tp, "--dry-run"] try: main(test_args=x) except Exception: @@ -83,9 +83,7 @@ class TestLooperBothRuns: def test_looper_cfg_invalid(self, cmd): """Verify looper does not accept invalid cfg paths""" - x = test_args_expansion( - cmd, "--looper-config", "jdfskfds/dsjfklds/dsjklsf.yaml" - ) + x = test_args_expansion(cmd, "--config", "jdfskfds/dsjfklds/dsjklsf.yaml") with pytest.raises(SystemExit): result = main(test_args=x) print(result) @@ -164,8 +162,7 @@ def test_looper_single_pipeline(self, prep_temp_pep): with mod_yaml_data(tp) as config_data: pifaces = config_data[PIPELINE_INTERFACES_KEY] - config_data[PIPELINE_INTERFACES_KEY]["sample"] = pifaces["sample"][1] - del config_data[PIPELINE_INTERFACES_KEY]["project"] + config_data[PIPELINE_INTERFACES_KEY] = pifaces[0] x = test_args_expansion(tp, "run") try: @@ -195,7 +192,7 @@ def test_looper_cli_pipeline(self, prep_temp_pep): tp = prep_temp_pep with mod_yaml_data(tp) as config_data: pifaces = config_data[PIPELINE_INTERFACES_KEY] - pi_pth = pifaces["sample"][1] + pi_pth = pifaces[1] x = test_args_expansion(tp, "run", ["--sample-pipeline-interfaces", pi_pth]) try: result = main(test_args=x) @@ -226,7 +223,7 @@ def test_looper_pipeline_not_found(self, prep_temp_pep): """ tp = prep_temp_pep with mod_yaml_data(tp) as config_data: - config_data[PIPELINE_INTERFACES_KEY]["sample"] = ["bogus"] + config_data[PIPELINE_INTERFACES_KEY] = ["bogus"] x = test_args_expansion(tp, "run") try: result = main(test_args=x) @@ -268,7 +265,7 @@ def test_looper_toggle(self, prep_temp_pep): project_config_path = get_project_config_path(tp) with mod_yaml_data(project_config_path) as project_config_data: - project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY][SAMPLE_TOGGLE_ATTR] = 0 + project_config_data[SAMPLE_MODS_KEY][APPEND_KEY][SAMPLE_TOGGLE_ATTR] = 0 x = test_args_expansion(tp, "run") x.pop(-1) # remove dry run for this test @@ -289,7 +286,7 @@ def test_cmd_extra_sample(self, prep_temp_pep, arg): project_config_path = get_project_config_path(tp) with mod_yaml_data(project_config_path) as project_config_data: - project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg + project_config_data[SAMPLE_MODS_KEY][APPEND_KEY]["command_extra"] = arg x = test_args_expansion(tp, "run") try: main(test_args=x) @@ -310,7 +307,7 @@ def test_cmd_extra_override_sample(self, prep_temp_pep, arg): project_config_path = get_project_config_path(tp) with mod_yaml_data(project_config_path) as project_config_data: - project_config_data[SAMPLE_MODS_KEY][CONSTANT_KEY]["command_extra"] = arg + project_config_data[SAMPLE_MODS_KEY][APPEND_KEY]["command_extra"] = arg x = test_args_expansion(tp, "run", ["--command-extra-override='different'"]) try: @@ -346,10 +343,9 @@ def test_looper_single_pipeline(self, prep_temp_pep): with mod_yaml_data(tp) as config_data: # Modifying in this way due to https://github.com/pepkit/looper/issues/474 - config_data[PIPELINE_INTERFACES_KEY]["project"] = os.path.join( + config_data[PIPELINE_INTERFACES_KEY] = os.path.join( os.path.dirname(tp), "pipeline/pipeline_interface1_project.yaml" ) - del config_data[PIPELINE_INTERFACES_KEY]["sample"] print(tp) x = test_args_expansion(tp, "runp") @@ -593,6 +589,9 @@ def test_cli_compute_overwrites_yaml_settings_spec(self, prep_temp_pep, cmd): assert_content_not_in_any_files(subs_list, "testin_mem") +@pytest.mark.skip( + reason="This functionality requires input from the user. Causing pytest to error if run without -s flag" +) class TestLooperConfig: def test_init_config_file(self, prep_temp_pep): @@ -635,4 +634,4 @@ def test_init_project_using_csv(self, prep_temp_pep_csv): pep_config_csv = os.path.join(os.path.dirname(tp), pep_config_csv) init_project = Project(cfg=pep_config_csv) - assert len(init_project.samples) == 2 + assert len(init_project.samples) == 3 diff --git a/tests/test_comprehensive.py b/tests/test_comprehensive.py index 9b857f8f7..41c73ea0c 100644 --- a/tests/test_comprehensive.py +++ b/tests/test_comprehensive.py @@ -23,7 +23,7 @@ def test_comprehensive_advanced_looper_no_pipestat(prep_temp_pep): path_to_looper_config = prep_temp_pep - x = ["run", "--looper-config", path_to_looper_config] + x = ["run", "--config", path_to_looper_config] try: results = main(test_args=x) @@ -48,7 +48,7 @@ def test_comprehensive_looper_no_pipestat(prep_temp_pep_basic): with open(basic_project_file, "w") as f: dump(basic_project_data, f) - x = ["run", "--looper-config", path_to_looper_config] + x = ["run", "--config", path_to_looper_config] try: results = main(test_args=x) @@ -85,7 +85,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat): with open(pipestat_project_file, "w") as f: dump(pipestat_project_data, f) - x = [cmd, "--looper-config", path_to_looper_config] + x = [cmd, "--config", path_to_looper_config] try: result = main(test_args=x) @@ -110,7 +110,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat): psm.set_status(record_identifier="frog_2", status_identifier="completed") # Now use looper check to get statuses - x = ["check", "--looper-config", path_to_looper_config] + x = ["check", "--config", path_to_looper_config] try: result = main(test_args=x) @@ -119,18 +119,18 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat): raise pytest.fail("DID RAISE {0}".format(Exception)) # Now use looper check to get project level statuses - x = ["check", "--looper-config", path_to_looper_config, "--project"] + x = ["check", "--config", path_to_looper_config, "--project"] try: result = main(test_args=x) - assert result == {"example_pipestat_project_pipeline": {"project": "unknown"}} + assert result == {} except Exception: raise pytest.fail("DID RAISE {0}".format(Exception)) # TEST LOOPER REPORT - x = ["report", "--looper-config", path_to_looper_config] + x = ["report", "--config", path_to_looper_config] try: result = main(test_args=x) @@ -140,7 +140,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat): # TEST LOOPER Table - x = ["table", "--looper-config", path_to_looper_config] + x = ["table", "--config", path_to_looper_config] try: result = main(test_args=x) @@ -153,7 +153,7 @@ def test_comprehensive_looper_pipestat(prep_temp_pep_pipestat): x = [ "destroy", - "--looper-config", + "--config", path_to_looper_config, "--force-yes", ] # Must force yes or pytest will throw an exception "OSError: pytest: reading from stdin while output is captured!" @@ -178,7 +178,7 @@ def test_comprehensive_looper_pephub(prep_temp_pep_pephub): # TODO need to add way to check if user is logged into pephub and then run test otherwise skip path_to_looper_config = prep_temp_pep_pephub - x = ["run", "--looper-config", path_to_looper_config] + x = ["run", "--config", path_to_looper_config] try: results = main(test_args=x)