Skip to content

Overhauling validation results to get them closer to cover different types of validators #1514

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yarikoptic merged 6 commits into from
Apr 4, 2025
Merged

Overhauling validation results to get them closer to cover different types of validators #1514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
666b8f4
feat: improve `ValidationResult`
yarikoptic Oct 18, 2024
5240543
fix: add `eval_type_backport` dependency
candleindark Mar 21, 2025
5c12e8b
fix: avoid catching unexpected errors in NWB validation
candleindark Mar 27, 2025
3b31ee7
Merge branch 'master' into ext-validation
candleindark Apr 2, 2025
699005d
rf: define and reuse some common `origin` objects
candleindark Apr 3, 2025
cf401cd
fix: relabel validation result of Zarr data by `dandi-cli` itself
candleindark Apr 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 17 additions & 13 deletions dandi/cli/tests/test_cmd_validate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,14 @@

from ..cmd_validate import _process_issues, validate
from ...tests.fixtures import BIDS_ERROR_TESTDATA_SELECTION
from ...validate_types import Scope, Severity, ValidationOrigin, ValidationResult
from ...validate_types import (
Origin,
OriginType,
Scope,
Severity,
ValidationResult,
Validator,
)


@pytest.mark.parametrize("dataset", BIDS_ERROR_TESTDATA_SELECTION)
Expand Down Expand Up @@ -68,35 +75,32 @@ def test_validate_nwb_path_grouping(organized_nwb_dir4: Path) -> None:


def test_process_issues(capsys):
origin_validation_nwbinspector = Origin(
type=OriginType.VALIDATION,
validator=Validator.nwbinspector,
validator_version="",
)

issues = [
ValidationResult(
id="NWBI.check_data_orientation",
origin=ValidationOrigin(
name="nwbinspector",
version="",
),
origin=origin_validation_nwbinspector,
scope=Scope.FILE,
message="Data may be in the wrong orientation.",
path=Path("dir0/sub-mouse004/sub-mouse004.nwb"),
severity=Severity.WARNING,
),
ValidationResult(
id="NWBI.check_data_orientation",
origin=ValidationOrigin(
name="nwbinspector",
version="",
),
origin=origin_validation_nwbinspector,
scope=Scope.FILE,
message="Data may be in the wrong orientation.",
path=Path("dir1/sub-mouse001/sub-mouse001.nwb"),
severity=Severity.WARNING,
),
ValidationResult(
id="NWBI.check_missing_unit",
origin=ValidationOrigin(
name="nwbinspector",
version="",
),
origin=origin_validation_nwbinspector,
scope=Scope.FILE,
message="Missing text for attribute 'unit'.",
path=Path("dir1/sub-mouse001/sub-mouse001.nwb"),
Expand Down
49 changes: 29 additions & 20 deletions dandi/files/bases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from xml.etree.ElementTree import fromstring

import dandischema
from dandischema.consts import DANDI_SCHEMA_VERSION
from dandischema.digests.dandietag import DandiETag
from dandischema.models import BareAsset, CommonModel
from dandischema.models import Dandiset as DandisetMeta
Expand All @@ -27,7 +28,17 @@
from dandi.metadata.core import get_default_metadata
from dandi.misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, P
from dandi.utils import post_upload_size_check, pre_upload_size_check, yaml_load
from dandi.validate_types import Scope, Severity, ValidationOrigin, ValidationResult
from dandi.validate_types import (
ORIGIN_INTERNAL_DANDI,
ORIGIN_VALIDATION_DANDI,
Origin,
OriginType,
Scope,
Severity,
Standard,
ValidationResult,
Validator,
)

lgr = dandi.get_logger()

Expand Down Expand Up @@ -201,13 +212,11 @@ def get_validation_errors(
)
return [
ValidationResult(
origin=ValidationOrigin(
name="dandi",
version=dandi.__version__,
),
origin=ORIGIN_INTERNAL_DANDI,
severity=Severity.ERROR,
id="dandi.SOFTWARE_ERROR",
scope=Scope.FILE,
origin_result=e,
# metadata=metadata,
path=self.filepath, # note that it is not relative .path
message=f"Failed to read metadata: {e}",
Expand Down Expand Up @@ -526,9 +535,10 @@ def get_validation_errors(
else:
# make sure that we have some basic metadata fields we require
try:
origin = ValidationOrigin(
name="nwbinspector",
version=str(_get_nwb_inspector_version()),
origin_validation_nwbinspector = Origin(
type=OriginType.VALIDATION,
validator=Validator.nwbinspector,
validator_version=str(_get_nwb_inspector_version()),
)

for error in inspect_nwbfile(
Expand All @@ -549,10 +559,11 @@ def get_validation_errors(
}
errors.append(
ValidationResult(
origin=origin,
origin=origin_validation_nwbinspector,
severity=severity,
id=f"NWBI.{error.check_function_name}",
scope=Scope.FILE,
origin_result=error,
path=Path(error.file_path),
message=error.message,
# Assuming multiple sessions per multiple subjects,
Expand Down Expand Up @@ -694,10 +705,7 @@ def _check_required_fields(
message = f"Required field {f!r} has no value"
errors.append(
ValidationResult(
origin=ValidationOrigin(
name="dandischema",
version=dandischema.__version__, # type: ignore[attr-defined]
),
origin=ORIGIN_VALIDATION_DANDI,
severity=Severity.ERROR,
id="dandischema.requred_field",
scope=Scope.FILE,
Expand All @@ -709,10 +717,7 @@ def _check_required_fields(
message = f"Required field {f!r} has value {v!r}"
errors.append(
ValidationResult(
origin=ValidationOrigin(
name="dandischema",
version=dandischema.__version__, # type: ignore[attr-defined]
),
origin=ORIGIN_VALIDATION_DANDI,
severity=Severity.WARNING,
id="dandischema.placeholder_value",
scope=Scope.FILE,
Expand Down Expand Up @@ -793,13 +798,17 @@ def _pydantic_errors_to_validation_results(
message = e.get("message", e.get("msg", None))
out.append(
ValidationResult(
origin=ValidationOrigin(
name="dandischema",
version=dandischema.__version__, # type: ignore[attr-defined]
origin=Origin(
type=OriginType.VALIDATION,
validator=Validator.dandischema,
validator_version=dandischema.__version__, # type: ignore[attr-defined]
standard=Standard.DANDI_SCHEMA,
standard_version=DANDI_SCHEMA_VERSION,
),
severity=Severity.ERROR,
id=id,
scope=scope,
origin_result=e,
path=file_path,
message=message,
# TODO? dataset_path=dataset_path,
Expand Down
2 changes: 1 addition & 1 deletion dandi/files/bids.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def _validate(self) -> None:
self._asset_metadata[bids_path] = prepare_metadata(
result.metadata
)
self._bids_version = result.origin.bids_version
self._bids_version = result.origin.standard_version

def get_asset_errors(self, asset: BIDSAsset) -> list[ValidationResult]:
""":meta private:"""
Expand Down
68 changes: 52 additions & 16 deletions dandi/files/zarr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,47 @@
)

from .bases import LocalDirectoryAsset
from ..validate_types import Scope, Severity, ValidationOrigin, ValidationResult
from ..validate_types import (
ORIGIN_VALIDATION_DANDI_ZARR,
Origin,
OriginType,
Scope,
Severity,
Standard,
ValidationResult,
Validator,
)

lgr = get_logger()


def get_zarr_format_version(data: Any) -> str:
"""
Get the Zarr storage specification version from a Zarr data object

Parameters
----------
data : zarr.core.Array or zarr.hierarchy.Group
The Zarr data object from which to extract the storage specification version

Returns
-------
str
The Zarr storage specification version,
https://zarr-specs.readthedocs.io/en/latest/specs.html, used in the Zarr data
object
"""
import zarr # Delay heavy import

Check warning on line 75 in dandi/files/zarr.py

View check run for this annotation

Codecov / codecov/patch

dandi/files/zarr.py#L75 

Added line #L75 was not covered by tests
Copy link
Copy Markdown
Member Author

@yarikoptic yarikoptic Apr 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hm, apparently we have no single test on validation of zarrs? that's odd :-/

filed dedicated

for that


if isinstance(data, zarr.Group):
meta = json.loads(data.store.get(".zgroup"))
elif isinstance(data, zarr.Array):
meta = json.loads(data.store.get(".zarray"))

Check warning on line 80 in dandi/files/zarr.py

View check run for this annotation

Codecov / codecov/patch

dandi/files/zarr.py#L77-L80 

Added lines #L77 - L80 were not covered by tests
else:
raise TypeError("`data` must be a `zarr.core.Array` or `zarr.hierarchy.Group`")
return str(meta["zarr_format"])

Check warning on line 83 in dandi/files/zarr.py

View check run for this annotation

Codecov / codecov/patch

dandi/files/zarr.py#L82-L83 

Added lines #L82 - L83 were not covered by tests


@dataclass
class LocalZarrEntry(BasePath):
"""A file or directory within a `ZarrAsset`"""
Expand Down Expand Up @@ -209,34 +245,37 @@
import zarr

errors: list[ValidationResult] = []
origin_internal_zarr: Origin = Origin(
type=OriginType.INTERNAL,
validator=Validator.zarr,
validator_version=zarr.__version__,
standard=Standard.ZARR,
)

try:
data = zarr.open(str(self.filepath))
except Exception:
except Exception as e:

Check warning on line 257 in dandi/files/zarr.py

View check run for this annotation

Codecov / codecov/patch

dandi/files/zarr.py#L257 

Added line #L257 was not covered by tests
if devel_debug:
raise
errors.append(
ValidationResult(
origin=ValidationOrigin(
name="zarr",
version=zarr.version.version,
),
origin=origin_internal_zarr,
severity=Severity.ERROR,
id="zarr.cannot_open",
scope=Scope.FILE,
origin_result=e,
path=self.filepath,
message="Error opening file.",
)
)
data = None

if isinstance(data, zarr.Group) and not data:
errors.append(
ValidationResult(
origin=ValidationOrigin(
name="zarr",
version=zarr.version.version,
),
origin=ORIGIN_VALIDATION_DANDI_ZARR,
severity=Severity.ERROR,
id="zarr.empty_group",
id="dandi_zarr.empty_group",
scope=Scope.FILE,
path=self.filepath,
message="Zarr group is empty.",
Expand All @@ -248,12 +287,9 @@
raise ValueError(msg)
errors.append(
ValidationResult(
origin=ValidationOrigin(
name="zarr",
version=zarr.version.version,
),
origin=ORIGIN_VALIDATION_DANDI_ZARR,
severity=Severity.ERROR,
id="zarr.tree_depth_exceeded",
id="dandi_zarr.tree_depth_exceeded",
scope=Scope.FILE,
path=self.filepath,
message=msg,
Expand Down
15 changes: 10 additions & 5 deletions dandi/organize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import ruamel.yaml

from . import __version__, get_logger
from . import get_logger
from .consts import dandi_layout_fields
from .dandiset import Dandiset
from .exceptions import OrganizeImpossibleError
Expand All @@ -35,7 +35,12 @@
move_file,
yaml_load,
)
from .validate_types import Scope, Severity, ValidationOrigin, ValidationResult
from .validate_types import (
ORIGIN_VALIDATION_DANDI_LAYOUT,
Scope,
Severity,
ValidationResult,
)

lgr = get_logger()

Expand Down Expand Up @@ -1119,7 +1124,7 @@ def validate_organized_path(
errors.append(
ValidationResult(
id="DANDI.NON_DANDI_FILENAME",
origin=ValidationOrigin(name="dandi", version=__version__),
origin=ORIGIN_VALIDATION_DANDI_LAYOUT,
severity=Severity.ERROR,
scope=Scope.FILE,
path=filepath,
Expand All @@ -1135,7 +1140,7 @@ def validate_organized_path(
errors.append(
ValidationResult(
id="DANDI.NON_DANDI_FOLDERNAME",
origin=ValidationOrigin(name="dandi", version=__version__),
origin=ORIGIN_VALIDATION_DANDI_LAYOUT,
severity=Severity.ERROR,
scope=Scope.FOLDER,
path=filepath,
Expand All @@ -1151,7 +1156,7 @@ def validate_organized_path(
errors.append(
ValidationResult(
id="DANDI.METADATA_MISMATCH_SUBJECT",
origin=ValidationOrigin(name="dandi", version=__version__),
origin=ORIGIN_VALIDATION_DANDI_LAYOUT,
severity=Severity.ERROR,
scope=Scope.FILE,
path=filepath,
Expand Down
Loading
Loading