From 241bc08e73e7140b34684deafe64a12bb8ede973 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 14 Nov 2024 18:59:32 -0800 Subject: [PATCH 1/7] feat: add model validator to remove `"@context"` key in `Asset` and `Dandiset` This allows validation of data instances of `Asset` and `Dandiset` to contain the `"@context"` key --- dandischema/models.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dandischema/models.py b/dandischema/models.py index eaa98263..e885f2d8 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -76,6 +76,20 @@ def diff_models(model1: M, model2: M) -> None: print(f"{field} is different") +def get_dict_without_context(d: Any) -> Any: + """ + If a given object is a dictionary, return a copy of it without the + `@context` key. Otherwise, return the input object as is. + + :param d: The given object + :return: If the object is a dictionary, a copy of it without the `@context` key; + otherwise, the input object as is. + """ + if isinstance(d, dict): + return {k: v for k, v in d.items() if k != "@context"} + return d + + class AccessType(Enum): """An enumeration of access status options""" @@ -1683,6 +1697,10 @@ def contributor_musthave_contact( "nskey": "dandi", } + # Model validator to remove the `"@context"` key from data instance before + # "base" validation is performed. + _remove_context_key = model_validator(mode="before")(get_dict_without_context) + class BareAsset(CommonModel): """Metadata used to describe an asset anywhere (local or server). @@ -1815,6 +1833,10 @@ class Asset(BareAsset): json_schema_extra={"readOnly": True, "nskey": "schema"} ) + # Model validator to remove the `"@context"` key from data instance before + # "base" validation is performed. + _remove_context_key = model_validator(mode="before")(get_dict_without_context) + class Publishable(DandiBaseModel): publishedBy: Union[AnyHttpUrl, PublishActivity] = Field( From eda8db0e9012130a975973003e7664c9ea6ca3a6 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 18 Nov 2024 10:14:00 -0800 Subject: [PATCH 2/7] feat: Add the `"@context"` key to the generated JSON schema for `Asset` and `Dandiset` This requires `"@context"` field at the JSON level --- dandischema/models.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/dandischema/models.py b/dandischema/models.py index e885f2d8..ced5b059 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -32,7 +32,7 @@ field_validator, model_validator, ) -from pydantic.json_schema import JsonSchemaValue +from pydantic.json_schema import JsonDict, JsonSchemaValue, JsonValue from pydantic_core import CoreSchema from zarr_checksum.checksum import InvalidZarrChecksum, ZarrDirectoryDigest @@ -90,6 +90,33 @@ def get_dict_without_context(d: Any) -> Any: return d +def add_context(json_schema: JsonDict) -> None: + """ + Add the `@context` key to the given JSON schema as a required key represented as a + dictionary. + + :param json_schema: The dictionary representing the JSON schema + + raises: ValueError if the `@context` key is already present in the given dictionary + """ + context_key = "@context" + context_key_title = "@Context" + properties: JsonDict = json_schema.get("properties", {}) + required: list[JsonValue] = json_schema.get("required", []) + + if context_key in properties or context_key in required: + msg = f"The '{context_key}' key is already present in the given JSON schema." + raise ValueError(msg) + + properties[context_key] = { + "format": "uri", + "minLength": 1, + "title": context_key_title, + "type": "string", + } + required.append(context_key) + + class AccessType(Enum): """An enumeration of access status options""" @@ -1601,8 +1628,6 @@ class CommonModel(DandiBaseModel): class Dandiset(CommonModel): """A body of structured information describing a DANDI dataset.""" - model_config = ConfigDict(extra="allow") - @field_validator("contributor") @classmethod def contributor_musthave_contact( @@ -1701,6 +1726,8 @@ def contributor_musthave_contact( # "base" validation is performed. _remove_context_key = model_validator(mode="before")(get_dict_without_context) + model_config = ConfigDict(extra="allow", json_schema_extra=add_context) + class BareAsset(CommonModel): """Metadata used to describe an asset anywhere (local or server). @@ -1837,6 +1864,8 @@ class Asset(BareAsset): # "base" validation is performed. _remove_context_key = model_validator(mode="before")(get_dict_without_context) + model_config = ConfigDict(json_schema_extra=add_context) + class Publishable(DandiBaseModel): publishedBy: Union[AnyHttpUrl, PublishActivity] = Field( From f1409d8d36a63edd48edd14f081833ff586a4969 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 18 Nov 2024 10:42:42 -0800 Subject: [PATCH 3/7] feat: Disallow extra field in all Pydantic DANDI models This also sets the `"additionalProperties"` of each corresponding model in JSON schema to `false` --- dandischema/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dandischema/models.py b/dandischema/models.py index ced5b059..476f5b30 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -648,6 +648,8 @@ def __get_pydantic_json_schema__( return schema + model_config = ConfigDict(extra="forbid") + class PropertyValue(DandiBaseModel): maxValue: Optional[float] = Field(None, json_schema_extra={"nskey": "schema"}) @@ -1726,7 +1728,7 @@ def contributor_musthave_contact( # "base" validation is performed. _remove_context_key = model_validator(mode="before")(get_dict_without_context) - model_config = ConfigDict(extra="allow", json_schema_extra=add_context) + model_config = ConfigDict(json_schema_extra=add_context) class BareAsset(CommonModel): From 0a69296c7857890642668dedd9cdd254d32b8b66 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 18 Nov 2024 16:38:40 -0800 Subject: [PATCH 4/7] fix: remove use of `JsonDict` and `JsonValue` These are not made available in `__all__`. Use of them triggers complains in the IDE. --- dandischema/models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dandischema/models.py b/dandischema/models.py index 476f5b30..7f67c448 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -15,6 +15,7 @@ Type, TypeVar, Union, + cast, ) from warnings import warn @@ -32,7 +33,7 @@ field_validator, model_validator, ) -from pydantic.json_schema import JsonDict, JsonSchemaValue, JsonValue +from pydantic.json_schema import JsonSchemaValue from pydantic_core import CoreSchema from zarr_checksum.checksum import InvalidZarrChecksum, ZarrDirectoryDigest @@ -90,19 +91,18 @@ def get_dict_without_context(d: Any) -> Any: return d -def add_context(json_schema: JsonDict) -> None: +def add_context(json_schema: dict) -> None: """ - Add the `@context` key to the given JSON schema as a required key represented as a - dictionary. + Add the `@context` key to the given JSON schema as a required key :param json_schema: The dictionary representing the JSON schema - raises: ValueError if the `@context` key is already present in the given dictionary + raises: ValueError if the `@context` key is already present in the given schema """ context_key = "@context" context_key_title = "@Context" - properties: JsonDict = json_schema.get("properties", {}) - required: list[JsonValue] = json_schema.get("required", []) + properties = cast(dict, json_schema.get("properties", {})) + required = cast(list, json_schema.get("required", [])) if context_key in properties or context_key in required: msg = f"The '{context_key}' key is already present in the given JSON schema." From 99a66edb8ba394aaf85d75acc7b3c5a6329e2efd Mon Sep 17 00:00:00 2001 From: Isaac To Date: Mon, 18 Nov 2024 17:08:18 -0800 Subject: [PATCH 5/7] fix: update `"properties"` and `"required"` keys of JSON schema This is needed to handle the case in which the keys are newly created --- dandischema/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dandischema/models.py b/dandischema/models.py index 7f67c448..e99d89c8 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -116,6 +116,11 @@ def add_context(json_schema: dict) -> None: } required.append(context_key) + # Update the schema + # This is needed to handle the case in which the keys are newly created + json_schema["properties"] = properties + json_schema["required"] = required + class AccessType(Enum): """An enumeration of access status options""" From 391aa1a4dcde148396ea54a74ab27939edea0ba0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 27 Nov 2024 12:46:41 -0500 Subject: [PATCH 6/7] Make @context optional (for now at least) at jsonschema level --- dandischema/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandischema/models.py b/dandischema/models.py index e99d89c8..7e844d05 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -114,7 +114,7 @@ def add_context(json_schema: dict) -> None: "title": context_key_title, "type": "string", } - required.append(context_key) + # required.append(context_key) # Update the schema # This is needed to handle the case in which the keys are newly created From fd3df903f2ed89129ad246fe4b0d2e22b8c7c0cc Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 19 Jan 2025 18:38:39 -0800 Subject: [PATCH 7/7] doc: update `add_context` docstring To reflect that `@context` is not set to be required --- dandischema/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dandischema/models.py b/dandischema/models.py index 7e844d05..2aa559d4 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -93,7 +93,7 @@ def get_dict_without_context(d: Any) -> Any: def add_context(json_schema: dict) -> None: """ - Add the `@context` key to the given JSON schema as a required key + Add the `@context` key to the given JSON schema :param json_schema: The dictionary representing the JSON schema @@ -114,7 +114,7 @@ def add_context(json_schema: dict) -> None: "title": context_key_title, "type": "string", } - # required.append(context_key) + # required.append(context_key) # Uncomment this line to make `@context` required # Update the schema # This is needed to handle the case in which the keys are newly created