merge: Merge branch 'main' of https://github.com/NHSDigital/data-validation-engine into feature/gr-dep008-913_uplift_python_version

Author: georgeRobertson

CHANGELOG.md

@@ -1,4 +1,18 @@
-## 1.0.0 (2025-10-09)
+## 0.1.0 (2025-11-10)
+
+### Feat
+
+- Added ability to define custom error codes and templated messages for data contract feedback messages
+- Added new JSON readers
+- Added SparkCSVReader
+- Added PolarsToDuckDBCSVReader and DuckDBCSVRepeatingReader
+- Added quotechar option to DuckDBCSVReader
+
+### Fix
+
+- Fixed issues with refdata loader table implementations
+- Fixed duckdb try_cast statements in data contract phase
+- Allowed use of entity type in file transformation
 
 ### Refactor
 

README.md

@@ -21,7 +21,7 @@ Additionally, if you'd like to contribute a new backend implementation into the
 
 ## Installation and usage
 
-The DVE is a Python package and can be installed using `pip`. As of release v1.0.0 we currently only supports Python 3.7, with Spark version 3.2.1 and DuckDB version of 1.1.0. We are currently working on upgrading the DVE to work on Python 3.11+ and this will be made available asap with version 2.0.0 release.
+The DVE is a Python package and can be installed using `pip`. As of release v0.1.0 we currently only supports Python 3.7, with Spark version 3.2.1 and DuckDB version of 1.1.0. We are currently working on upgrading the DVE to work on Python 3.11+ and this will be made available asap with version 1.0.0 release.
 
 In addition to a working Python 3.7+ installation you will need OpenJDK 11 installed if you're planning to use the Spark backend implementation.
 
@@ -30,7 +30,7 @@ Python dependencies are listed in `pyproject.toml`.
 To install the DVE package you can simply install using a package manager such as [pip](https://pypi.org/project/pip/).
 
 ```
-pip install git+https://github.com/NHSDigital/data-validation-engine.git@v1.0.0
+pip install git+https://github.com/NHSDigital/data-validation-engine.git@v0.1.0
 ```
 
 Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema json document (configuration), please read the [documentation](./docs/).
@@ -48,8 +48,8 @@ If you have feature request then please follow the same process whilst using the
 Below is a list of features that we would like to implement or have been requested.
 | Feature | Release Version | Released? |
 | ------- | --------------- | --------- |
-| Open source release | 1.0.0 | Yes |
-| Uplift to Python 3.11 | 2.0.0 | No |
+| Open source release | 0.1.0 | Yes |
+| Uplift to Python 3.11 | 1.0.0 | No |
 | Upgrade to Pydantic 2.0 | Not yet confirmed | No |
 | Create a more user friendly interface for building and modifying dischema files | Not yet confirmed | No |
 

docs/README.md

@@ -18,7 +18,7 @@ DVE configuration can be instantiated from a json (dischema) file which might be
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {},
         "schemas": {},
         "datasets": {

docs/detailed_guidance/data_contract.md

@@ -4,7 +4,7 @@ Lets look at the data contract configuration from [Introduction to DVE](../READM
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {},
         "schemas": {},
         "datasets": {
@@ -78,7 +78,7 @@ Here we have only filled out datasets. We've added a few more fields such as `Pe
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {
             "isodate": {
                 "description": "an isoformatted date type",
@@ -172,7 +172,7 @@ We can see here that the Activity has a number of fields. `startdate`, `enddate`
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {
             "isodate": {
                 "description": "an isoformatted date type",

docs/json_schemas/contract/components/contact_error_details.schema.json

@@ -0,0 +1,10 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/contract_error_details.schema.json",
+    "title": "base_entity",
+    "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)",
+    "type": "object",
+    "additionalProperties": {
+        "$ref": "field_error_type.schema.json"
+    }
+}

docs/json_schemas/contract/components/field_error_detail.schema.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_detail.schema.json",
+    "title": "field_error_detail",
+    "description": "The custom details to be used for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the field and error type specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be used for the field and error type specified. This can include templating (specified using jinja2 conventions). During templating, the full record will be available with an additional __error_value to easily obtain nested offending values.",
+            "type": "string",
+            "enum": [
+                "record_rejection",
+                "file_rejection",
+                "warning"
+            ]
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}

docs/json_schemas/contract/components/field_error_type.schema copy.json

@@ -0,0 +1,21 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_type.schema.json",
+    "title": "field_error_detail",
+    "description": "The error type for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_type": {
+            "description": "The type of error the details are for",
+            "type": "string",
+            "enum": [
+                "Blank",
+                "Bad value",
+                "Wrong format"
+            ],
+            "additionalProperties": {
+                "$ref": "field_error_detail.schema.json"
+            }
+        }
+    }
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nhs_dve"
-version = "1.0.0"
+version = "0.1.0"
 description = "`nhs data validation engine` is a framework used to validate data"
 authors = ["NHS England <england.contactus@nhs.net>"]
 readme = "README.md"

src/dve/core_engine/backends/base/core.py

@@ -77,7 +77,7 @@ def __init__(
                 raise ValueError(f"Entity name cannot start with 'refdata_', got {entity_name!r}")
             self.entities[entity_name] = entity
 
-        self.reference_data = reference_data or {}
+        self.reference_data = reference_data if reference_data is not None else {}
         """The reference data mapping."""
 
     @staticmethod

src/dve/core_engine/backends/implementations/duckdb/__init__.py

@@ -0,0 +1,25 @@
+"""Implementation of duckdb backend"""
+from dve.core_engine.backends.implementations.duckdb.readers.json import DuckDBJSONReader
+from dve.core_engine.backends.readers import register_reader
+
+from .contract import DuckDBDataContract
+from .readers import (
+    DuckDBCSVReader,
+    DuckDBCSVRepeatingHeaderReader,
+    DuckDBXMLStreamReader,
+    PolarsToDuckDBCSVReader,
+)
+from .reference_data import DuckDBRefDataLoader
+from .rules import DuckDBStepImplementations
+
+register_reader(DuckDBCSVReader)
+register_reader(DuckDBCSVRepeatingHeaderReader)
+register_reader(DuckDBJSONReader)
+register_reader(DuckDBXMLStreamReader)
+register_reader(PolarsToDuckDBCSVReader)
+
+__all__ = [
+    "DuckDBDataContract",
+    "DuckDBRefDataLoader",
+    "DuckDBStepImplementations",
+]