merge: merge from main

Author: georgeRobertson

.github/workflows/ci_linting.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: Install extra dependencies for a python 3.7.17 install
+      - name: Install extra dependencies for a python install
         run: |
           sudo apt-get update
           sudo apt -y install --no-install-recommends liblzma-dev libbz2-dev libreadline-dev
@@ -26,6 +26,9 @@ jobs:
 
       - name: reshim asdf
         run: asdf reshim
+      
+      - name: ensure poetry using desired python version
+        run: poetry env use $(asdf which python)
 
       - name: Cache Poetry virtualenv
         uses: actions/cache@v4

.github/workflows/ci_testing.yml

@@ -13,11 +13,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
-
-      - name: Install extra dependencies for a python 3.7.17 install
+      
+      - name: Install extra dependencies for a python install
         run: |
           sudo apt-get update
           sudo apt -y install --no-install-recommends liblzma-dev libbz2-dev libreadline-dev
+
       - name: Install asdf cli
         uses: asdf-vm/actions/setup@v4
 
@@ -26,6 +27,9 @@ jobs:
 
       - name: reshim asdf
         run: asdf reshim
+      
+      - name: ensure poetry using desired python version
+        run: poetry env use $(asdf which python)
 
       - name: Cache Poetry virtualenv
         uses: actions/cache@v4
@@ -42,7 +46,6 @@ jobs:
       - name: Run pytest and coverage
         run: |
           export JAVA_HOME=$(asdf where java)
-          echo "JAVA_HOME - $JAVA_HOME"
           make coverage
       
       - name: Upload Coverage Report
@@ -54,5 +57,4 @@ jobs:
       - name: Run behave tests
         run: |
           export JAVA_HOME=$(asdf where java)
-          echo "JAVA_HOME - $JAVA_HOME"
           make behave

.mise.toml

@@ -1,4 +1,4 @@
 [tools]
-python="3.7.17"
-poetry="1.4.2"
+python="3.11"
+poetry="2.2"
 java="liberica-1.8.0"

.tool-versions

@@ -1,3 +1,3 @@
-python 3.7.17
-poetry 1.4.2
+python 3.11.14
+poetry 2.2.0
 java liberica-1.8.0

CHANGELOG.md

@@ -1,5 +1,38 @@
+## v0.3.0 (2025-11-19)
+
+### Feat
+
+- new domain type formattedtime for time only data
+
+### Refactor
+
+- small tweak to allow use of dynamic fields in select rules
+
+## v0.2.0 (2025-11-12)
+
+### Refactor
+
+- ensure dve working on python 3.10
+- ensure dve working on python 3.11
+
+### BREAKING CHANGE
+
+- Numerous typing updates that will make this codebase unusable below python 3.9
+
+note - this does not mean the package will work on python 3.9. Minimum working version is 3.10.
+
+### Feat
+
+- added functionality to allow error messages in business rules t… (#8)
+
+### Refactor
+
+- bump pylint to work correctly with py3.11 and fix numerous linting issues
+
 ## 0.1.0 (2025-11-10)
 
+*NB - This was previously v1.0.0 and v1.1.0 but has been rolled back into a 0.1.0 release to reflect lack of package stability.*
+
 ### Feat
 
 - Added ability to define custom error codes and templated messages for data contract feedback messages

Makefile

@@ -3,7 +3,7 @@ activate = poetry run
 # dev
 install:
 	poetry lock
-	poetry install --with dev,test
+	poetry install --with dev
 
 # dist
 wheel:
@@ -27,6 +27,15 @@ coverage:
 	$(activate) coverage report
 	$(activate) coverage xml
 
+# lint
+pylint:
+	${activate} pylint src/
+
+mypy:
+	${activate} mypy src/
+
+lint: mypy pylint
+
 # pre-commit
 pre-commit-all:
 	${activate} pre-commit run --all-files

README.md

@@ -1,6 +1,6 @@
 # Data Validation Engine
 
-The Data Validation Engine (DVE) is a configuration driven data validation library built and utilised by NHS England.
+The Data Validation Engine (DVE) is a configuration driven data validation library built and utilised by NHS England. Currently the package has been reverted from v1.0.0 release to a 0.x as we feel the package is not yet mature enough to be considered a 1.0.0 release. So please bear this in mind if reading through the commits and references to a v1+ release when on v0.x.
 
 As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` file and example files can be accessed [here](./tests/testdata/). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here](./docs/).
 
@@ -9,7 +9,7 @@ Once a dischema file has been defined, you are ready to use the DVE. The DVE is
 |    | Service | Purpose |
 | -- | ------- | ------- |
 | 1. | File Transformation | This service will take submitted files and turn them into stringified parquet file(s) to ensure that a consistent data structure can be passed through the other services. |
-| 2. | Data Contract | This service will validate and peform type casting against a stringified parquet file using [pydantic models](https://docs.pydantic.dev/1.10/). |
+| 2. | Data Contract | This service will validate and perform type casting against a stringified parquet file using [pydantic models](https://docs.pydantic.dev/1.10/). |
 | 3. | Business Rules | The business rules service will perform more complex validations such as comparisons between fields and tables, aggregations, filters etc to generate new entities. |
 | 4. | Error Reports | The error reports service will take all the errors raised in previous services and surface them into a readable format for a downstream users/service. Currently, this implemented to be an excel spreadsheet but could be reconfigured to meet other requirements/use cases. |
 
@@ -21,7 +21,7 @@ Additionally, if you'd like to contribute a new backend implementation into the
 
 ## Installation and usage
 
-The DVE is a Python package and can be installed using `pip`. As of release v0.1.0 we currently only supports Python 3.7, with Spark version 3.2.1 and DuckDB version of 1.1.0. We are currently working on upgrading the DVE to work on Python 3.11+ and this will be made available asap with version 1.0.0 release.
+The DVE is a Python package and can be installed using `pip`. As of release v0.1.x we currently only supports Python 3.7, with Spark version 3.2.1 and DuckDB version of 1.1.0. We are currently working on upgrading the DVE to work on Python 3.10-3.11 and this will be made available with version v0.2.x release.
 
 In addition to a working Python 3.7+ installation you will need OpenJDK 11 installed if you're planning to use the Spark backend implementation.
 
@@ -33,7 +33,7 @@ To install the DVE package you can simply install using a package manager such a
 pip install git+https://github.com/NHSDigital/data-validation-engine.git@v0.1.0
 ```
 
-Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema json document (configuration), please read the [documentation](./docs/).
+Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema JSON document (configuration), please read the [documentation](./docs/).
 
 Please note - The long term aim is to make the DVE available via PyPi and Conda but we are not quite there yet. Once available this documentation will be updated to contain the new installation options.
 
@@ -49,7 +49,7 @@ Below is a list of features that we would like to implement or have been request
 | Feature | Release Version | Released? |
 | ------- | --------------- | --------- |
 | Open source release | 0.1.0 | Yes |
-| Uplift to Python 3.11 | 1.0.0 | No |
+| Uplift to Python 3.11 | 0.2.0 | Yes |
 | Upgrade to Pydantic 2.0 | Not yet confirmed | No |
 | Create a more user friendly interface for building and modifying dischema files | Not yet confirmed | No |
 

docs/detailed_guidance/domain_types.md

@@ -4,24 +4,24 @@ Domain types are custom defined pydantic types that solve common problems with u
 This might include Postcodes, NHS Numbers, dates with specific formats etc.
 
 Below is a list of defined types, their output type and any contraints. Nested beneath them are any constraints that area allowed and their default values if there are any.
-| Defined Type | Output Type | Contraints & Defaults |
-| ------------ | ----------- | --------------------- |
-| NHSNumber | str |
-| permissive_nhs_number | str | <li> warn_on_test_numbers = False </li> |
-| Postcode | str |
-| OrgId | str |
-| conformatteddate | date | <li>date_format: str</li><li>ge: date</li><li>le: date</li><li>gt: date</li><li>lt: date</li> |
-| formatteddatetime | datetime | <li>date_format: str </li><li>timezone_treatment: one_of ["forbid", "permit", "require] = "permit"</li> |
-| reportingperiod | date | <li>reporting_period_type: one_of ["start", "end"]</li><li>date_format: str = "%Y-%m-%d"</li> |
-| alphanumeric | str | <li>min_digits : NonNegativeInt = 1</li><li>max_digits: PositiveInt = 1</li> |
-| identifier | str | <li>min_digits : NonNegativeInt = 1</li><li>max_digits: PositiveInt = 1</li>
+| Defined Type | Output Type | Contraints & Defaults | Supported Implementations |
+| ------------ | ----------- | --------------------- | ------------------------- |
+| NHSNumber | str | | Spark, DuckDB |
+| permissive_nhs_number | str | <li> warn_on_test_numbers = False </li> | Spark, DuckDB |
+| Postcode | str | | Spark, DuckDB |
+| OrgId | str | | Spark, DuckDB |
+| conformatteddate | date | <li>date_format: str</li><li>ge: date</li><li>le: date</li><li>gt: date</li><li>lt: date</li> | Spark, DuckDB |
+| formatteddatetime | datetime | <li>date_format: str </li><li>timezone_treatment: one_of ["forbid", "permit", "require] = "permit"</li> | Spark, DuckDB |
+| formattedtime | time | <li>time_format: str </li><li>timezone_treatment: one_of ["forbid", "permit", "require"] = "permit" | DuckDB |
+| reportingperiod | date | <li>reporting_period_type: one_of ["start", "end"]</li><li>date_format: str = "%Y-%m-%d"</li> | Spark, DuckDB |
+| alphanumeric | str | <li>min_digits : NonNegativeInt = 1</li><li>max_digits: PositiveInt = 1</li> | Spark, DuckDB |
+| identifier | str | <li>min_digits : NonNegativeInt = 1</li><li>max_digits: PositiveInt = 1</li> | Spark, DuckDB |
 
-Other types that are allowed include:
+**Other types that are allowed include:**
 - str
 - int
 - date
 - datetime
 - Decimal
 - float
-
-And any types that are included in [pydantic version 1.10](https://docs.pydantic.dev/1.10/usage/types/#pydantic-types)
+- Any types that are included in [pydantic version 1.10](https://docs.pydantic.dev/1.10/usage/types/#pydantic-types)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nhs_dve"
-version = "0.1.0"
+version = "0.3.0"
 description = "`nhs data validation engine` is a framework used to validate data"
 authors = ["NHS England <england.contactus@nhs.net>"]
 readme = "README.md"
@@ -9,58 +9,73 @@ packages = [
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Operating System :: OS Independent",
     "Topic :: Software Development :: Libraries",
     "Typing :: Typed",
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.7.2,<3.8"
-boto3 = "1.28.47" # Boto3 will no longer support Python 3.7 starting December 13, 2023
-botocore = "1.31.47"
-delta-spark = "1.1.0"
+python = ">=3.10,<3.12"
+boto3 = "1.34.162"
+botocore = "1.34.162"
+delta-spark = "2.4.0"
 duckdb = "1.1.0" # mitigates security vuln in < 1.1.0
 formulas = "1.2.4"
 idna = "3.7"   # Downstream dep of requests but has security vuln < 3.7
 Jinja2 = "3.1.6"   # mitigates security vuln in < 3.1.6
 lxml = "4.9.1"
 openpyxl = "3.1.0"
-pandas = "1.3.5"
-polars = "0.17.14"
-pyarrow = "7.0.0"
+pandas = "2.2.2"
+polars = "0.20.14"
+pyarrow = "17.0.0"
 pydantic = "1.10.15"  # Mitigates security vuln in < 1.10.13
 pymongo = "4.6.3"
-pyspark = "3.2.1"
+pyspark = "3.4.4"
 pytz = "2022.1"
-PyYAML = "5.4"
-requests = "2.31.0"
+PyYAML = "6.0.3"
+requests = "2.32.4"  # Mitigates security vuln in < 2.31.0
 schedula = "1.2.19"
 sqlalchemy = "2.0.19"
 typing_extensions = "4.6.2"
-urllib3 = "1.26.19"  # Used transiently, but has security vuln < 1.26.19
+urllib3 = "2.5.0"  # Mitigates security vuln in < 1.26.19
 xmltodict = "0.13.0"
 
+[tool.poetry.group.dev]
+optional = true
+include-groups = [
+    "test",
+    "lint"
+]
+
 [tool.poetry.group.dev.dependencies]
-commitizen = "3.9.1"  # latest version to support Python 3.7.17
-pre-commit = "2.21.0" # latest version to support Python 3.7.17
+commitizen = "4.9.1"
+pre-commit = "4.3.0"
+
+[tool.poetry.group.test]
+optional = true
 
 [tool.poetry.group.test.dependencies]
 faker = "18.11.1"
-behave = "1.2.6"
-coverage = "6.4.3"
-moto = {extras = ["s3"], version = "3.1.18"}
+behave = "1.3.3"
+coverage = "7.11.0"
+moto = {extras = ["s3"], version = "4.0.13"}
+Werkzeug = "3.0.6"  # Dependency of moto which needs 3.0.6 for security vuln mitigation
 mongomock = "4.1.2"
-pytest = "7.4.4"
-pytest-lazy-fixture = "0.6.3"
+pytest = "8.4.2"
+pytest-lazy-fixtures = "1.4.0"  # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
 xlsx2csv = "0.8.2"
 
+[tool.poetry.group.lint]
+optional = true
+
 [tool.poetry.group.lint.dependencies]
-black = "22.6.0"
-astroid = "2.11.7"
+black = "24.3.0"
+astroid = "2.14.2"
 isort = "5.11.5"
-pylint = "2.14.5"
-mypy = "0.982"
+pylint = "2.16.4"
+mypy = "0.991"
 boto3-stubs = {extras = ["essential"], version = "1.26.72"}
 botocore-stubs = "1.29.72"
 pandas-stubs = "1.2.0.62"
@@ -112,9 +127,8 @@ source_pkgs = [
 show_missing = true
 
 [tool.pylint]
-# Can't add support for custom checker until running on Python 3.9+ again.
-# init-hook = "import sys; sys.path.append('./pylint_checkers')"
-# load-plugins = "check_typing_imports"
+init-hook = "import sys; sys.path.append('./pylint_checkers')"
+load-plugins = "check_typing_imports"
 
 [tool.pylint.main]
 # Analyse import fallback blocks. This can be used to support both Python 2 and 3
@@ -189,7 +203,7 @@ persistent = true
 
 # Minimum Python version to use for version dependent checks. Will default to the
 # version used to run pylint.
-py-version = "3.7"
+py-version = "3.11"
 
 # Discover python modules and packages in the file system subtree.
 # recursive =