diff --git a/looper/__init__.py b/looper/__init__.py index fe751d02..0be3b8cb 100644 --- a/looper/__init__.py +++ b/looper/__init__.py @@ -26,7 +26,6 @@ write_sample_yaml_prj, write_custom_template, ) -from .const import * from .pipeline_interface import PipelineInterface from .project import Project diff --git a/looper/cli_pydantic.py b/looper/cli_pydantic.py index 3ec094d0..fb06b687 100644 --- a/looper/cli_pydantic.py +++ b/looper/cli_pydantic.py @@ -17,6 +17,7 @@ # with types. from __future__ import annotations +import os import sys import logmuse @@ -35,11 +36,32 @@ TopLevelParser, add_short_arguments, ) -from .const import * +from .const import ( + CLI_KEY, + CLI_PROJ_ATTRS, + EXAMPLE_COMPUTE_SPEC_FMT, + PipelineLevel, + PROJECT_PL_ARG, + SAMPLE_EXCLUSION_OPTNAME, + SAMPLE_INCLUSION_OPTNAME, + SAMPLE_PL_ARG, +) from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config -from .exceptions import * -from .looper import * -from .parser_types import * +from .exceptions import ( + MisconfigurationException, + PipestatConfigurationException, + SampleFailedException, +) +from .looper import ( + Checker, + Cleaner, + Collator, + Destroyer, + Linker, + Reporter, + Runner, + Tabulator, +) from .project import Project, ProjectContext from .utils import ( dotfile_path, diff --git a/looper/conductor.py b/looper/conductor.py index 268db543..cf438845 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -12,7 +12,7 @@ from math import ceil from json import loads from subprocess import check_output -from typing import * +from typing import Optional from eido import read_schema, get_input_files_size from eido.const import INPUT_FILE_SIZE_KEY, MISSING_KEY @@ -25,7 +25,24 @@ from yaml import dump from yacman import FutureYAMLConfigManager as YAMLConfigManager -from .const import * +from .const import ( + EXTRA_PROJECT_CMD_TEMPLATE, + EXTRA_SAMPLE_CMD_TEMPLATE, + JOB_NAME_KEY, + NOT_SUB_MSG, + OUTDIR_KEY, + OUTPUT_SCHEMA_KEY, + PipelineLevel, + PRE_SUBMIT_CMD_KEY, + PRE_SUBMIT_HOOK_KEY, + PRE_SUBMIT_PY_FUN_KEY, + PROJECT_PL_KEY, + RESULTS_SUBDIR_KEY, + SAMPLE_CWL_YAML_PATH_KEY, + SAMPLE_PL_KEY, + SUBMISSION_SUBDIR_KEY, + VAR_TEMPL_KEY, +) from .exceptions import JobSubmissionException from .processed_project import populate_sample_paths from .utils import ( @@ -33,7 +50,6 @@ jinja_render_template_strictly, expand_nested_var_templates, ) -from .const import PipelineLevel _LOGGER = logging.getLogger(__name__) diff --git a/looper/looper.py b/looper/looper.py index cb3cb301..fa0fd124 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -27,7 +27,6 @@ from eido import validate_config, validate_sample from eido.exceptions import EidoValidationError from jsonschema import ValidationError -from peppy.const import * from peppy.exceptions import RemoteYAMLError from rich.color import Color from rich.console import Console @@ -37,8 +36,19 @@ from .conductor import SubmissionConductor -from .exceptions import * -from .const import * +from .exceptions import ( + JobSubmissionException, + LooperReportError, + MisconfigurationException, + SampleFailedException, +) +from .const import ( + DEBUG_COMMANDS, + DEBUG_EIDO_VALIDATION, + DEBUG_JOBS, + NOT_SUB_MSG, + SUBMISSION_FAILURE_MESSAGE, +) from .project import Project from .utils import ( desired_samples_range_skipped, diff --git a/looper/pipeline_interface.py b/looper/pipeline_interface.py index f7f0793e..ee76f790 100644 --- a/looper/pipeline_interface.py +++ b/looper/pipeline_interface.py @@ -12,7 +12,19 @@ from ubiquerg import expandpath, is_url from yacman import load_yaml, YAMLConfigManager -from .const import * +from .const import ( + COMPUTE_KEY, + DYN_VARS_KEY, + FILE_SIZE_COLNAME, + ID_COLNAME, + INPUT_SCHEMA_KEY, + LOOPER_KEY, + PIFACE_SCHEMA_SRC, + PIPELINE_INTERFACE_PIPELINE_NAME_KEY, + RESOURCES_KEY, + SIZE_DEP_VARS_KEY, + VAR_TEMPL_KEY, +) from .exceptions import ( InvalidResourceSpecificationException, PipelineInterfaceConfigError, diff --git a/looper/plugins.py b/looper/plugins.py index dc34283e..d4cc5265 100644 --- a/looper/plugins.py +++ b/looper/plugins.py @@ -1,6 +1,10 @@ import logging import os -from .const import * +from .const import ( + SAMPLE_CWL_YAML_PATH_KEY, + SAMPLE_YAML_PATH_KEY, + SAMPLE_YAML_PRJ_PATH_KEY, +) from .conductor import _get_yaml_path _LOGGER = logging.getLogger(__name__) diff --git a/looper/processed_project.py b/looper/processed_project.py index 39b87fa0..9222dadc 100644 --- a/looper/processed_project.py +++ b/looper/processed_project.py @@ -128,8 +128,8 @@ import os from logging import getLogger -from eido.const import * -from eido.exceptions import * +from eido.const import PROP_KEY +from eido.exceptions import EidoSchemaInvalidError from peppy.project import Project from peppy.sample import Sample diff --git a/looper/project.py b/looper/project.py index 88de52e0..18fb1a4b 100644 --- a/looper/project.py +++ b/looper/project.py @@ -2,6 +2,7 @@ import itertools import os +from typing import List, NoReturn, Union from yaml import safe_load @@ -15,16 +16,42 @@ from eido import PathAttrNotFoundError, read_schema from jsonschema import ValidationError from pandas.core.common import flatten +from peppy import Project as peppyProject from peppy.utils import make_abs_via_cfg +from peppy.const import CONFIG_KEY from pipestat import PipestatManager from .conductor import write_pipestat_config -from .exceptions import * +from .exceptions import MisconfigurationException, PipelineInterfaceConfigError from .pipeline_interface import PipelineInterface from .processed_project import populate_project_paths, populate_sample_paths -from .utils import * -from .const import PipelineLevel +from .utils import ( + expandpath, + fetch_sample_flags, + get_sample_status, + getLogger, + is_pephub_registry_path, +) +from .const import ( + CLI_PROJ_ATTRS, + COMPUTE_PACKAGE_KEY, + DRY_RUN_KEY, + EXTRA_KEY, + FILE_CHECKS_KEY, + INPUT_SCHEMA_KEY, + LOOPER_KEY, + OUTDIR_KEY, + OUTPUT_SCHEMA_KEY, + PIFACE_KEY_SELECTOR, + PIPELINE_INTERFACE_PIPELINE_NAME_KEY, + PIPELINE_INTERFACES_KEY, + PipelineLevel, + PIPESTAT_KEY, + RESULTS_SUBDIR_KEY, + SAMPLE_PL_ARG, + SUBMISSION_SUBDIR_KEY, +) __all__ = ["Project"] diff --git a/looper/utils.py b/looper/utils.py index b5d904c5..78de020c 100644 --- a/looper/utils.py +++ b/looper/utils.py @@ -6,20 +6,37 @@ import itertools from logging import getLogger import os -from typing import * +from typing import Iterable, List, Optional, Tuple, Union import re import jinja2 import yaml from peppy import Project as peppyProject -from peppy.const import * +from peppy.const import AMENDMENTS_KEY, CONFIG_KEY, NAME_KEY, SAMPLE_MODS_KEY from ubiquerg import convert_value, expandpath, parse_registry_path, deep_update from pephubclient.constants import RegistryPath from pydantic import ValidationError from yacman import load_yaml from yaml.parser import ParserError -from .const import * +from .const import ( + ALL_SUBCMD_KEY, + CLI_KEY, + FLAGS, + LOOPER_DOTFILE_NAME, + LOOPER_GENERIC_COUNT_LINES, + LOOPER_GENERIC_OUTPUT_SCHEMA, + LOOPER_GENERIC_PIPELINE, + LOOPER_KEY, + OUTDIR_KEY, + PEP_CONFIG_KEY, + PIPELINE_INTERFACES_KEY, + PIPESTAT_KEY, + POSITIONAL, + PROJECT_PL_ARG, + PipelineLevel, + SAMPLE_PL_ARG, +) from .command_models.commands import SUPPORTED_COMMANDS from .exceptions import MisconfigurationException, PipelineInterfaceConfigError from rich.console import Console