v1.1.0 - custom error codes for data contract, new readers and various bugfixes (#7)

* feat: initial work to add custom error messages to data contract

* feat: added support for nested fields when configuring custom error details for data contract. Includes accessing nested error values in error messages.

* feat: bug fixes with readers and base pipeline. Added spark csv reader. Added new dischema files for tests

* feat: small fixes and movies dataset working up to end of data contract

* feat: further refinement of movies test dataset.

* feat: improved movies dataset test coverage. Added testing for spark and duckdb refdata loaders when table config specified.

* feat: sorted linting. Added json schema docs

* feat: tweaked json schemas for contract error details. Fixed failing unit test

* style: fix mypy error with entity.write

* feature: more duckdb csv readers (#6)

* feat: add new duckdb csv readers

* style: fix polars typing issue and white space in docstrings

* bump: version 1.0.0 → 1.1.0

* docs: Amended changelog

---------

Co-authored-by: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>

Author: stevenhsd

CHANGELOG.md

@@ -1,3 +1,18 @@
+## v1.1.0 (2025-10-28)
+
+### Feat
+
+- Added ability to define custom error codes and templated messages for data contract feedback messages
+- Added new JSON readers
+- Added SparkCSVReader
+- Added PolarsToDuckDBCSVReader and DuckDBCSVRepeatingReader
+- Added quotechar option to DuckDBCSVReader
+
+### Fix
+- Fixed issues with refdata loader table implementations
+- Fixed duckdb try_cast statements in data contract phase
+- Allowed use of entity type in file transformation
+
 ## 1.0.0 (2025-10-09)
 
 ### Refactor

docs/README.md

@@ -18,7 +18,7 @@ DVE configuration can be instantiated from a json (dischema) file which might be
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {},
         "schemas": {},
         "datasets": {

docs/detailed_guidance/data_contract.md

@@ -4,7 +4,7 @@ Lets look at the data contract configuration from [Introduction to DVE](../READM
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {},
         "schemas": {},
         "datasets": {
@@ -78,7 +78,7 @@ Here we have only filled out datasets. We've added a few more fields such as `Pe
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {
             "isodate": {
                 "description": "an isoformatted date type",
@@ -172,7 +172,7 @@ We can see here that the Activity has a number of fields. `startdate`, `enddate`
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {
             "isodate": {
                 "description": "an isoformatted date type",

docs/json_schemas/contract/components/contact_error_details.schema.json

@@ -0,0 +1,10 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/contract_error_details.schema.json",
+    "title": "base_entity",
+    "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)",
+    "type": "object",
+    "additionalProperties": {
+        "$ref": "field_error_type.schema.json"
+    }
+}

docs/json_schemas/contract/components/field_error_detail.schema.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_detail.schema.json",
+    "title": "field_error_detail",
+    "description": "The custom details to be used for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the field and error type specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be used for the field and error type specified. This can include templating (specified using jinja2 conventions). During templating, the full record will be available with an additional __error_value to easily obtain nested offending values.",
+            "type": "string",
+            "enum": [
+                "record_rejection",
+                "file_rejection",
+                "warning"
+            ]
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}

docs/json_schemas/contract/components/field_error_type.schema copy.json

@@ -0,0 +1,21 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_type.schema.json",
+    "title": "field_error_detail",
+    "description": "The error type for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_type": {
+            "description": "The type of error the details are for",
+            "type": "string",
+            "enum": [
+                "Blank",
+                "Bad value",
+                "Wrong format"
+            ],
+            "additionalProperties": {
+                "$ref": "field_error_detail.schema.json"
+            }
+        }
+    }
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nhs_dve"
-version = "1.0.0"
+version = "1.1.0"
 description = "`nhs data validation engine` is a framework used to validate data"
 authors = ["NHS England <england.contactus@nhs.net>"]
 readme = "README.md"

src/dve/core_engine/backends/implementations/duckdb/__init__.py

@@ -0,0 +1,25 @@
+"""Implementation of duckdb backend"""
+from dve.core_engine.backends.implementations.duckdb.readers.json import DuckDBJSONReader
+from dve.core_engine.backends.readers import register_reader
+
+from .contract import DuckDBDataContract
+from .readers import (
+    DuckDBCSVReader,
+    DuckDBCSVRepeatingHeaderReader,
+    DuckDBXMLStreamReader,
+    PolarsToDuckDBCSVReader
+)
+from .reference_data import DuckDBRefDataLoader
+from .rules import DuckDBStepImplementations
+
+register_reader(DuckDBCSVReader)
+register_reader(DuckDBCSVRepeatingHeaderReader)
+register_reader(DuckDBJSONReader)
+register_reader(DuckDBXMLStreamReader)
+register_reader(PolarsToDuckDBCSVReader)
+
+__all__ = [
+    "DuckDBDataContract",
+    "DuckDBRefDataLoader",
+    "DuckDBStepImplementations",
+]

src/dve/core_engine/backends/implementations/duckdb/auditing.py

@@ -13,9 +13,9 @@
 )
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
     PYTHON_TYPE_TO_DUCKDB_TYPE,
-    PYTHON_TYPE_TO_POLARS_TYPE,
     table_exists,
 )
+from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE
 from dve.core_engine.models import (
     AuditRecord,
     ProcessingStatusRecord,

src/dve/core_engine/backends/implementations/duckdb/contract.py

@@ -20,13 +20,12 @@
     duckdb_read_parquet,
     duckdb_write_parquet,
     get_duckdb_type_from_annotation,
-    get_polars_type_from_annotation,
     relation_is_empty,
 )
 from dve.core_engine.backends.implementations.duckdb.types import DuckDBEntities
 from dve.core_engine.backends.metadata.contract import DataContractMetadata
 from dve.core_engine.backends.types import StageSuccessful
-from dve.core_engine.backends.utilities import stringify_model
+from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, Messages
 from dve.core_engine.validation import RowValidator
@@ -95,8 +94,8 @@ def generate_ddb_cast_statement(
         Current duckdb python API doesn't play well with this currently.
         """
         if not null_flag:
-            return f"try_cast({column_name} AS {dtype}) AS {column_name}"
-        return f"cast(NULL AS {dtype}) AS {column_name}"
+            return f'try_cast("{column_name}" AS {dtype}) AS "{column_name}"'
+        return f'cast(NULL AS {dtype}) AS "{column_name}"'
 
     def apply_data_contract(
         self, entities: DuckDBEntities, contract_metadata: DataContractMetadata