style: fix linting

Author: georgeRobertson

src/dve/core_engine/backends/implementations/duckdb/readers/json.py

@@ -1,7 +1,8 @@
 """A csv reader to create duckdb relations"""
 
 # pylint: disable=arguments-differ
-from typing import Any, Dict, Iterator, Optional, Type
+from collections.abc import Iterator
+from typing import Any, Optional
 
 from duckdb import DuckDBPyRelation, read_json
 from pydantic import BaseModel
@@ -25,18 +26,18 @@ def __init__(self, json_format: Optional[str] = "array"):
         super().__init__()
 
     def read_to_py_iterator(
-        self, resource: URI, entity_name: EntityName, schema: Type[BaseModel]
-    ) -> Iterator[Dict[str, Any]]:
+        self, resource: URI, entity_name: EntityName, schema: type[BaseModel]
+    ) -> Iterator[dict[str, Any]]:
         """Creates an iterable object of rows as dictionaries"""
         return self.read_to_relation(resource, entity_name, schema).pl().iter_rows(named=True)
 
     @read_function(DuckDBPyRelation)
     def read_to_relation(  # pylint: disable=unused-argument
-        self, resource: URI, entity_name: EntityName, schema: Type[BaseModel]
+        self, resource: URI, entity_name: EntityName, schema: type[BaseModel]
     ) -> DuckDBPyRelation:
         """Returns a relation object from the source json"""
 
-        ddb_schema: Dict[str, SQLType] = {
+        ddb_schema: dict[str, SQLType] = {
             fld.name: str(get_duckdb_type_from_annotation(fld.annotation))  # type: ignore
             for fld in schema.__fields__.values()
         }

src/dve/core_engine/backends/implementations/spark/readers/csv.py

@@ -1,7 +1,7 @@
 """A reader implementation using the Databricks Spark CSV reader."""
 
-
-from typing import Any, Dict, Iterator, Type
+from collections.abc import Iterator
+from typing import Any, Optional
 
 from pydantic import BaseModel
 from pyspark.sql import DataFrame, SparkSession
@@ -30,7 +30,7 @@ def __init__(
         header: bool = True,
         multi_line: bool = False,
         encoding: str = "utf-8-sig",
-        spark_session: SparkSession = None,
+        spark_session: Optional[SparkSession] = None,
     ) -> None:
 
         self.delimiter = delimiter
@@ -39,13 +39,13 @@ def __init__(
         self.quote_char = quote_char
         self.header = header
         self.multi_line = multi_line
-        self.spark_session = spark_session if spark_session else SparkSession.builder.getOrCreate()
+        self.spark_session = spark_session if spark_session else SparkSession.builder.getOrCreate()  # type: ignore  # pylint: disable=C0301
 
         super().__init__()
 
     def read_to_py_iterator(
-        self, resource: URI, entity_name: EntityName, schema: Type[BaseModel]
-    ) -> Iterator[Dict[URI, Any]]:
+        self, resource: URI, entity_name: EntityName, schema: type[BaseModel]
+    ) -> Iterator[dict[URI, Any]]:
         df = self.read_to_dataframe(resource, entity_name, schema)
         yield from (record.asDict(True) for record in df.toLocalIterator())
 
@@ -54,7 +54,7 @@ def read_to_dataframe(
         self,
         resource: URI,
         entity_name: EntityName,  # pylint: disable=unused-argument
-        schema: Type[BaseModel],
+        schema: type[BaseModel],
     ) -> DataFrame:
         """Read a CSV file directly to a Spark DataFrame."""
         if get_content_length(resource) == 0:

src/dve/core_engine/backends/implementations/spark/readers/json.py

@@ -1,7 +1,7 @@
 """A reader implementation using the Databricks Spark JSON reader."""
 
-
-from typing import Any, Dict, Iterator, Optional, Type
+from collections.abc import Iterator
+from typing import Any, Optional
 
 from pydantic import BaseModel
 from pyspark.sql import DataFrame, SparkSession
@@ -31,13 +31,13 @@ def __init__(
 
         self.encoding = encoding
         self.multi_line = multi_line
-        self.spark_session = spark_session if spark_session else SparkSession.builder.getOrCreate()
+        self.spark_session = spark_session if spark_session else SparkSession.builder.getOrCreate()  # type: ignore  # pylint: disable=C0301
 
         super().__init__()
 
     def read_to_py_iterator(
-        self, resource: URI, entity_name: EntityName, schema: Type[BaseModel]
-    ) -> Iterator[Dict[URI, Any]]:
+        self, resource: URI, entity_name: EntityName, schema: type[BaseModel]
+    ) -> Iterator[dict[URI, Any]]:
         df = self.read_to_dataframe(resource, entity_name, schema)
         yield from (record.asDict(True) for record in df.toLocalIterator())
 
@@ -46,7 +46,7 @@ def read_to_dataframe(
         self,
         resource: URI,
         entity_name: EntityName,  # pylint: disable=unused-argument
-        schema: Type[BaseModel],
+        schema: type[BaseModel],
     ) -> DataFrame:
         """Read a JSON file directly to a Spark DataFrame."""
         if get_content_length(resource) == 0:

src/dve/pipeline/pipeline.py

@@ -532,9 +532,9 @@ def apply_business_rules(self, submission_info: SubmissionInfo, failed: bool):
                     entity_name,
                 ),
             )
-            entity_manager.entities[entity_name] = self.step_implementations.read_parquet(
+            entity_manager.entities[entity_name] = self.step_implementations.read_parquet(  # type: ignore
                 projected
-            )  # type: ignore
+            )
 
         status = SubmissionStatus(
             failed=failed,