merge: changes from feature/ndit-512_dc_codes_granular_null_vs_format

Author: georgeRobertson

docs/README.md

@@ -18,7 +18,7 @@ DVE configuration can be instantiated from a json (dischema) file which might be
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {},
         "schemas": {},
         "datasets": {

docs/detailed_guidance/data_contract.md

@@ -4,7 +4,7 @@ Lets look at the data contract configuration from [Introduction to DVE](../READM
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {},
         "schemas": {},
         "datasets": {
@@ -78,7 +78,7 @@ Here we have only filled out datasets. We've added a few more fields such as `Pe
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {
             "isodate": {
                 "description": "an isoformatted date type",
@@ -172,7 +172,7 @@ We can see here that the Activity has a number of fields. `startdate`, `enddate`
 {
     "contract": {
         "cache_originals": true,
-        "contract_error_codes": null,
+        "error_details": null,
         "types": {
             "isodate": {
                 "description": "an isoformatted date type",

docs/json_schemas/contract/components/contact_error_details.schema.json

@@ -0,0 +1,10 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/contract_error_details.schema.json",
+    "title": "base_entity",
+    "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)",
+    "type": "object",
+    "additionalProperties": {
+        "$ref": "field_error_type.schema.json"
+    }
+}

docs/json_schemas/contract/components/field_error_detail.schema.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_detail.schema.json",
+    "title": "field_error_detail",
+    "description": "The custom details to be used for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the field and error type specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be used for the field and error type specified. This can include templating (specified using jinja2 conventions). During templating, the full record will be available with an additional __error_value to easily obtain nested offending values.",
+            "type": "string",
+            "enum": [
+                "record_rejection",
+                "file_rejection",
+                "warning"
+            ]
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}

docs/json_schemas/contract/components/field_error_type.schema copy.json

@@ -0,0 +1,21 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_type.schema.json",
+    "title": "field_error_detail",
+    "description": "The error type for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_type": {
+            "description": "The type of error the details are for",
+            "type": "string",
+            "enum": [
+                "Blank",
+                "Bad value",
+                "Wrong format"
+            ],
+            "additionalProperties": {
+                "$ref": "field_error_detail.schema.json"
+            }
+        }
+    }
+}

src/dve/core_engine/backends/implementations/duckdb/__init__.py

@@ -1,3 +1,4 @@
+"""Implementation of duckdb backend"""
 from dve.core_engine.backends.implementations.duckdb.readers.json import DuckDBJSONReader
 from dve.core_engine.backends.readers import register_reader
 

src/dve/core_engine/backends/implementations/duckdb/auditing.py

@@ -15,14 +15,14 @@
     PYTHON_TYPE_TO_DUCKDB_TYPE,
     table_exists,
 )
+from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE
 from dve.core_engine.models import (
     AuditRecord,
     ProcessingStatusRecord,
     SubmissionInfo,
     SubmissionStatisticsRecord,
     TransferRecord,
 )
-from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE
 from dve.core_engine.type_hints import URI, ExecutorType
 
 

src/dve/core_engine/backends/implementations/duckdb/contract.py

@@ -25,7 +25,7 @@
 from dve.core_engine.backends.implementations.duckdb.types import DuckDBEntities
 from dve.core_engine.backends.metadata.contract import DataContractMetadata
 from dve.core_engine.backends.types import StageSuccessful
-from dve.core_engine.backends.utilities import stringify_model, get_polars_type_from_annotation
+from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, Messages
 from dve.core_engine.validation import RowValidator

src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -11,11 +11,9 @@
 
 import duckdb.typing as ddbtyp
 import numpy as np
-import polars as pl  # type: ignore
 from duckdb import DuckDBPyConnection, DuckDBPyRelation
 from duckdb.typing import DuckDBPyType
 from pandas import DataFrame
-
 from pydantic import BaseModel
 from typing_extensions import Annotated, get_args, get_origin, get_type_hints
 
@@ -91,6 +89,7 @@ def __call__(self):
 }
 """A mapping of Python types to the equivalent DuckDB types."""
 
+
 def table_exists(connection: DuckDBPyConnection, table_name: str) -> bool:
     """check if a table exists in a given DuckDBPyConnection"""
     return table_name in map(lambda x: x[0], connection.sql("SHOW TABLES").fetchall())
@@ -190,6 +189,7 @@ def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType:
             return duck_type
     raise ValueError(f"No equivalent DuckDB type for {type_annotation!r}")
 
+
 def coerce_inferred_numpy_array_to_list(pandas_df: DataFrame) -> DataFrame:
     """Function to modify numpy inferred array when cnverting from duckdb relation to
     pandas dataframe - these cause issues with pydantic models
@@ -224,23 +224,20 @@ def _ddb_read_parquet(
 
 
 def _ddb_write_parquet(  # pylint: disable=unused-argument
-    self,
-    entity: Union[Iterator[Dict[str, Any]],
-                  DuckDBPyRelation],
-    target_location: URI,
-    **kwargs
+    self, entity: Union[Iterator[Dict[str, Any]], DuckDBPyRelation], target_location: URI, **kwargs
 ) -> URI:
     """Method to write parquet files from type cast entities
     following data contract application
     """
     if isinstance(_get_implementation(target_location), LocalFilesystemImplementation):
         Path(target_location).parent.mkdir(parents=True, exist_ok=True)
-    
+
     if isinstance(entity, Generator):
-        entity = self._connection.query("select dta.* from (select unnest($data) as dta)",
-                                                  params={"data": list(entity)})
+        entity = self._connection.query(
+            "select dta.* from (select unnest($data) as dta)", params={"data": list(entity)}
+        )
 
-    entity.to_parquet(file_name=target_location, compression="snappy", **kwargs)
+    entity.to_parquet(file_name=target_location, compression="snappy", **kwargs)  # type: ignore
     return target_location
 
 

src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -5,7 +5,7 @@
 
 import duckdb as ddb
 import polars as pl
-from duckdb import DuckDBPyConnection, DuckDBPyRelation, read_csv, default_connection
+from duckdb import DuckDBPyConnection, DuckDBPyRelation, default_connection, read_csv
 from pydantic import BaseModel
 
 from dve.core_engine.backends.base.reader import BaseFileReader, read_function
@@ -20,6 +20,7 @@
 from dve.core_engine.type_hints import URI, EntityName
 from dve.parser.file_handling import get_content_length
 
+
 @duckdb_write_parquet
 class DuckDBCSVReader(BaseFileReader):
     """A reader for CSV files"""