feat: sorted linting. Added json schema docs

Author: stevenhsd

docs/json_schemas/contract/components/contact_error_details.schema.json

@@ -0,0 +1,16 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/contract_error_details.schema.json",
+    "title": "base_entity",
+    "description": "An optional specification of custom error codes and messages for fields for the data contract phase of validation",
+    "type": "object",
+    "properties": {
+        "field_name": {
+            "type": "object",
+            "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)",
+            "additionalProperties": {
+                "$ref": "field_error_type.schema.json"
+            }
+        }
+    }
+}

docs/json_schemas/contract/components/field_error_detail.schema.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_detail.schema.json",
+    "title": "field_error_detail",
+    "description": "The custom details to be used for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the field and error type specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be used for the field and error type specified. This can include templating (specified using jinja2 conventions). During templating, the full record will be available with an additional __error_value to easily obtain nested offending values.",
+            "type": "string",
+            "enum": [
+                "record_rejection",
+                "file_rejection",
+                "warning"
+            ]
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}

docs/json_schemas/contract/components/field_error_type.schema copy.json

@@ -0,0 +1,21 @@
+{
+    "$schema": "https://json-schema.org/draft-07/schema",
+    "$id": "data-ingest:contract/components/field_error_type.schema.json",
+    "title": "field_error_detail",
+    "description": "The error type for a field when a validation error is raised during the data contract phase",
+    "type": "object",
+    "properties": {
+        "error_type": {
+            "description": "The type of error the details are for",
+            "type": "string",
+            "enum": [
+                "Blank",
+                "Bad value",
+                "Wrong format"
+            ],
+            "additionalProperties": {
+                "$ref": "field_error_detail.schema.json"
+            }
+        }
+    }
+}

src/dve/core_engine/backends/implementations/duckdb/__init__.py

@@ -1,3 +1,4 @@
+"""Implementation of duckdb backend"""
 from dve.core_engine.backends.implementations.duckdb.readers.json import DuckDBJSONReader
 from dve.core_engine.backends.readers import register_reader
 

src/dve/core_engine/backends/implementations/duckdb/auditing.py

@@ -15,14 +15,14 @@
     PYTHON_TYPE_TO_DUCKDB_TYPE,
     table_exists,
 )
+from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE
 from dve.core_engine.models import (
     AuditRecord,
     ProcessingStatusRecord,
     SubmissionInfo,
     SubmissionStatisticsRecord,
     TransferRecord,
 )
-from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE
 from dve.core_engine.type_hints import URI, ExecutorType
 
 

src/dve/core_engine/backends/implementations/duckdb/contract.py

@@ -25,7 +25,7 @@
 from dve.core_engine.backends.implementations.duckdb.types import DuckDBEntities
 from dve.core_engine.backends.metadata.contract import DataContractMetadata
 from dve.core_engine.backends.types import StageSuccessful
-from dve.core_engine.backends.utilities import stringify_model, get_polars_type_from_annotation
+from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, Messages
 from dve.core_engine.validation import RowValidator

src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -11,11 +11,9 @@
 
 import duckdb.typing as ddbtyp
 import numpy as np
-import polars as pl  # type: ignore
 from duckdb import DuckDBPyConnection, DuckDBPyRelation
 from duckdb.typing import DuckDBPyType
 from pandas import DataFrame
-
 from pydantic import BaseModel
 from typing_extensions import Annotated, get_args, get_origin, get_type_hints
 
@@ -91,6 +89,7 @@ def __call__(self):
 }
 """A mapping of Python types to the equivalent DuckDB types."""
 
+
 def table_exists(connection: DuckDBPyConnection, table_name: str) -> bool:
     """check if a table exists in a given DuckDBPyConnection"""
     return table_name in map(lambda x: x[0], connection.sql("SHOW TABLES").fetchall())
@@ -190,6 +189,7 @@ def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType:
             return duck_type
     raise ValueError(f"No equivalent DuckDB type for {type_annotation!r}")
 
+
 def coerce_inferred_numpy_array_to_list(pandas_df: DataFrame) -> DataFrame:
     """Function to modify numpy inferred array when cnverting from duckdb relation to
     pandas dataframe - these cause issues with pydantic models
@@ -224,23 +224,20 @@ def _ddb_read_parquet(
 
 
 def _ddb_write_parquet(  # pylint: disable=unused-argument
-    self,
-    entity: Union[Iterator[Dict[str, Any]],
-                  DuckDBPyRelation],
-    target_location: URI,
-    **kwargs
+    self, entity: Union[Iterator[Dict[str, Any]], DuckDBPyRelation], target_location: URI, **kwargs
 ) -> URI:
     """Method to write parquet files from type cast entities
     following data contract application
     """
     if isinstance(_get_implementation(target_location), LocalFilesystemImplementation):
         Path(target_location).parent.mkdir(parents=True, exist_ok=True)
-    
+
     if isinstance(entity, Generator):
-        entity = self._connection.query("select dta.* from (select unnest($data) as dta)",
-                                                  params={"data": list(entity)})
+        entity = self._connection.query(
+            "select dta.* from (select unnest($data) as dta)", params={"data": list(entity)}
+        )
 
-    entity.to_parquet(file_name=target_location, compression="snappy", **kwargs)
+    entity.to_parquet(file_name=target_location, compression="snappy", **kwargs) # type: ignore
     return target_location
 
 

src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -3,7 +3,7 @@
 # pylint: disable=arguments-differ
 from typing import Any, Dict, Iterator, Type
 
-from duckdb import DuckDBPyConnection, DuckDBPyRelation, read_csv, default_connection
+from duckdb import DuckDBPyConnection, DuckDBPyRelation, default_connection, read_csv
 from pydantic import BaseModel
 
 from dve.core_engine.backends.base.reader import BaseFileReader, read_function
@@ -14,6 +14,7 @@
 from dve.core_engine.backends.implementations.duckdb.types import SQLType
 from dve.core_engine.type_hints import URI, EntityName
 
+
 @duckdb_write_parquet
 class DuckDBCSVReader(BaseFileReader):
     """A reader for CSV files"""

src/dve/core_engine/backends/implementations/duckdb/readers/json.py

@@ -5,7 +5,6 @@
 
 from duckdb import DuckDBPyRelation, read_json
 from pydantic import BaseModel
-from typing_extensions import Literal
 
 from dve.core_engine.backends.base.reader import BaseFileReader, read_function
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
@@ -19,12 +18,9 @@
 @duckdb_write_parquet
 class DuckDBJSONReader(BaseFileReader):
     """A reader for JSON files"""
-    
-    def __init__(
-        self,
-        format: Optional[str] = "array"
-    ):
-        self._format = format
+
+    def __init__(self, json_format: Optional[str] = "array"):
+        self._json_format = json_format
 
         super().__init__()
 
@@ -45,6 +41,4 @@ def read_to_relation(  # pylint: disable=unused-argument
             for fld in schema.__fields__.values()
         }
 
-        return read_json(resource,
-                         columns=ddb_schema,
-                         format=self._format)
+        return read_json(resource, columns=ddb_schema, format=self._json_format) # type: ignore

src/dve/core_engine/backends/implementations/duckdb/readers/xml.py

@@ -3,27 +3,22 @@
 
 from typing import Dict, Optional, Type
 
-import pandas as pd
 import polars as pl
 from duckdb import DuckDBPyConnection, DuckDBPyRelation, default_connection
 from pydantic import BaseModel
 
 from dve.core_engine.backends.base.reader import read_function
-from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
-    duckdb_write_parquet,
-)
+from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import duckdb_write_parquet
 from dve.core_engine.backends.readers.xml import XMLStreamReader
-from dve.core_engine.backends.utilities import stringify_model, get_polars_type_from_annotation
+from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
 from dve.core_engine.type_hints import URI
 
 
 @duckdb_write_parquet
 class DuckDBXMLStreamReader(XMLStreamReader):
     """A reader for XML files"""
 
-    def __init__(self,
-                 ddb_connection: Optional[DuckDBPyConnection] = None,
-                 **kwargs):
+    def __init__(self, ddb_connection: Optional[DuckDBPyConnection] = None, **kwargs):
         self.ddb_connection = ddb_connection if ddb_connection else default_connection
         super().__init__(**kwargs)