Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion looper/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
write_sample_yaml_prj,
write_custom_template,
)
from .const import *
from .pipeline_interface import PipelineInterface
from .project import Project

Expand Down
30 changes: 26 additions & 4 deletions looper/cli_pydantic.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# with types.
from __future__ import annotations

import os
import sys

import logmuse
Expand All @@ -35,11 +36,32 @@
TopLevelParser,
add_short_arguments,
)
from .const import *
from .const import (
CLI_KEY,
CLI_PROJ_ATTRS,
EXAMPLE_COMPUTE_SPEC_FMT,
PipelineLevel,
PROJECT_PL_ARG,
SAMPLE_EXCLUSION_OPTNAME,
SAMPLE_INCLUSION_OPTNAME,
SAMPLE_PL_ARG,
)
from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
from .exceptions import *
from .looper import *
from .parser_types import *
from .exceptions import (
MisconfigurationException,
PipestatConfigurationException,
SampleFailedException,
)
from .looper import (
Checker,
Cleaner,
Collator,
Destroyer,
Linker,
Reporter,
Runner,
Tabulator,
)
from .project import Project, ProjectContext
from .utils import (
dotfile_path,
Expand Down
22 changes: 19 additions & 3 deletions looper/conductor.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from math import ceil
from json import loads
from subprocess import check_output
from typing import *
from typing import Optional

from eido import read_schema, get_input_files_size
from eido.const import INPUT_FILE_SIZE_KEY, MISSING_KEY
Expand All @@ -25,15 +25,31 @@
from yaml import dump
from yacman import FutureYAMLConfigManager as YAMLConfigManager

from .const import *
from .const import (
EXTRA_PROJECT_CMD_TEMPLATE,
EXTRA_SAMPLE_CMD_TEMPLATE,
JOB_NAME_KEY,
NOT_SUB_MSG,
OUTDIR_KEY,
OUTPUT_SCHEMA_KEY,
PipelineLevel,
PRE_SUBMIT_CMD_KEY,
PRE_SUBMIT_HOOK_KEY,
PRE_SUBMIT_PY_FUN_KEY,
PROJECT_PL_KEY,
RESULTS_SUBDIR_KEY,
SAMPLE_CWL_YAML_PATH_KEY,
SAMPLE_PL_KEY,
SUBMISSION_SUBDIR_KEY,
VAR_TEMPL_KEY,
)
from .exceptions import JobSubmissionException
from .processed_project import populate_sample_paths
from .utils import (
fetch_sample_flags,
jinja_render_template_strictly,
expand_nested_var_templates,
)
from .const import PipelineLevel


_LOGGER = logging.getLogger(__name__)
Expand Down
16 changes: 13 additions & 3 deletions looper/looper.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from eido import validate_config, validate_sample
from eido.exceptions import EidoValidationError
from jsonschema import ValidationError
from peppy.const import *
from peppy.exceptions import RemoteYAMLError
from rich.color import Color
from rich.console import Console
Expand All @@ -37,8 +36,19 @@

from .conductor import SubmissionConductor

from .exceptions import *
from .const import *
from .exceptions import (
JobSubmissionException,
LooperReportError,
MisconfigurationException,
SampleFailedException,
)
from .const import (
DEBUG_COMMANDS,
DEBUG_EIDO_VALIDATION,
DEBUG_JOBS,
NOT_SUB_MSG,
SUBMISSION_FAILURE_MESSAGE,
)
from .project import Project
from .utils import (
desired_samples_range_skipped,
Expand Down
14 changes: 13 additions & 1 deletion looper/pipeline_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,19 @@
from ubiquerg import expandpath, is_url
from yacman import load_yaml, YAMLConfigManager

from .const import *
from .const import (
COMPUTE_KEY,
DYN_VARS_KEY,
FILE_SIZE_COLNAME,
ID_COLNAME,
INPUT_SCHEMA_KEY,
LOOPER_KEY,
PIFACE_SCHEMA_SRC,
PIPELINE_INTERFACE_PIPELINE_NAME_KEY,
RESOURCES_KEY,
SIZE_DEP_VARS_KEY,
VAR_TEMPL_KEY,
)
from .exceptions import (
InvalidResourceSpecificationException,
PipelineInterfaceConfigError,
Expand Down
6 changes: 5 additions & 1 deletion looper/plugins.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import logging
import os
from .const import *
from .const import (
SAMPLE_CWL_YAML_PATH_KEY,
SAMPLE_YAML_PATH_KEY,
SAMPLE_YAML_PRJ_PATH_KEY,
)
from .conductor import _get_yaml_path

_LOGGER = logging.getLogger(__name__)
Expand Down
4 changes: 2 additions & 2 deletions looper/processed_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@
import os
from logging import getLogger

from eido.const import *
from eido.exceptions import *
from eido.const import PROP_KEY
from eido.exceptions import EidoSchemaInvalidError
from peppy.project import Project
from peppy.sample import Sample

Expand Down
33 changes: 30 additions & 3 deletions looper/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import itertools
import os
from typing import List, NoReturn, Union

from yaml import safe_load

Expand All @@ -15,16 +16,42 @@
from eido import PathAttrNotFoundError, read_schema
from jsonschema import ValidationError
from pandas.core.common import flatten
from peppy import Project as peppyProject
from peppy.utils import make_abs_via_cfg
from peppy.const import CONFIG_KEY
from pipestat import PipestatManager

from .conductor import write_pipestat_config

from .exceptions import *
from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
from .pipeline_interface import PipelineInterface
from .processed_project import populate_project_paths, populate_sample_paths
from .utils import *
from .const import PipelineLevel
from .utils import (
expandpath,
fetch_sample_flags,
get_sample_status,
getLogger,
is_pephub_registry_path,
)
from .const import (
CLI_PROJ_ATTRS,
COMPUTE_PACKAGE_KEY,
DRY_RUN_KEY,
EXTRA_KEY,
FILE_CHECKS_KEY,
INPUT_SCHEMA_KEY,
LOOPER_KEY,
OUTDIR_KEY,
OUTPUT_SCHEMA_KEY,
PIFACE_KEY_SELECTOR,
PIPELINE_INTERFACE_PIPELINE_NAME_KEY,
PIPELINE_INTERFACES_KEY,
PipelineLevel,
PIPESTAT_KEY,
RESULTS_SUBDIR_KEY,
SAMPLE_PL_ARG,
SUBMISSION_SUBDIR_KEY,
)

__all__ = ["Project"]

Expand Down
23 changes: 20 additions & 3 deletions looper/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,37 @@
import itertools
from logging import getLogger
import os
from typing import *
from typing import Iterable, List, Optional, Tuple, Union
import re

import jinja2
import yaml
from peppy import Project as peppyProject
from peppy.const import *
from peppy.const import AMENDMENTS_KEY, CONFIG_KEY, NAME_KEY, SAMPLE_MODS_KEY
from ubiquerg import convert_value, expandpath, parse_registry_path, deep_update
from pephubclient.constants import RegistryPath
from pydantic import ValidationError
from yacman import load_yaml
from yaml.parser import ParserError

from .const import *
from .const import (
ALL_SUBCMD_KEY,
CLI_KEY,
FLAGS,
LOOPER_DOTFILE_NAME,
LOOPER_GENERIC_COUNT_LINES,
LOOPER_GENERIC_OUTPUT_SCHEMA,
LOOPER_GENERIC_PIPELINE,
LOOPER_KEY,
OUTDIR_KEY,
PEP_CONFIG_KEY,
PIPELINE_INTERFACES_KEY,
PIPESTAT_KEY,
POSITIONAL,
PROJECT_PL_ARG,
PipelineLevel,
SAMPLE_PL_ARG,
)
from .command_models.commands import SUPPORTED_COMMANDS
from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
from rich.console import Console
Expand Down