Merge pull request #1 from NHSDigital/rest_of_the_code

feat: rest of the codebase

Author: georgeRobertson

.gitallowed

@@ -0,0 +1,19 @@
+# base
+.*git-secrets/.*:*
+.*terraform.tfstate.*:*
+.*.svg:*
+.*.jquery.*.js:*
+.*engine_version*
+.*127.0.0.1*
+.*assembly: AssemblyVersion*
+.*assembly: AssemblyFileVersion*
+.*Version=*
+
+# poetry dependencies false flags
+pandas-stubs
+types-openpyxl
+types-PyYAML
+types-requests
+types-setuptools
+types-urllib3
+types-xmltodict

.gitignore

@@ -0,0 +1,159 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Mesh folders
+mesh*/
+
+# Spark warehouse
+spark-warehouse/*
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+poetry.lock
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Editor stuff:
+.vscode/
+.idea/
+.devcontainer/
+
+# Exclude outputs from dve.pipelines
+/outputs/
+.~lock*
+
+metastore_db/
+
+mesh_folder/*
+in/*
+/runtime-image*/engine_package.tgz
+/run_configuration/
+/runtime-image-test*/
+/nhs_data_validation_engine-*-py3-none-any.whl
+whl/
+spark-warehouse/
+
+#test ipynb
+testing.ipynb

.mise.toml

@@ -0,0 +1,4 @@
+[tools]
+python="3.7.17"
+poetry="1.4.2"
+java="liberica-1.8.0"

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_install_hook_types: [
+  commit-msg,
+  pre-push,
+]
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      # - id: trailing-whitespace
+      - id: end-of-file-fixer
+        stages: [commit-msg]
+      # - id: check-yaml
+      # - id: check-added-large-files
+  - repo: local
+    hooks:
+      - id: git-secrets
+        name: Git Secrets
+        description: git-secrets scans commits, commit messages, and --no-ff merges to prevent adding secrets into your git repositories.
+        entry: 'scripts/git-secrets/pre-commit.sh'
+        language: script
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+      - id: black
+        stages: [commit-msg]
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        stages: [commit-msg]
+  - repo: https://github.com/commitizen-tools/commitizen
+    rev: v3.9.1
+    hooks:
+      - id: commitizen
+        stages: [commit-msg]

.tool-versions

@@ -0,0 +1,3 @@
+python 3.7.17
+poetry 1.4.2
+java liberica-1.8.0

CHANGELOG.md

@@ -0,0 +1,5 @@
+## 1.0.0 (2025-10-09)
+
+### Refactor
+
+- release initial dve source code

CONTRIBUTE.md

@@ -0,0 +1,62 @@
+# DVE Contributing guidelines
+
+# Developer information
+
+## Getting started
+
+To begin with I would recommend that users read all the documentation available within the [docs](./docs/). It gives an overview of how the DVE works and how to work with the dischema json document.
+
+## General requirements
+To start contributing to the DVE project you will need the following tooling available:
+| Tool | Version | Reason |
+| ---- | ------- | ------ |
+| `Python` | 3.7.17 | Currently supported version of `Python` for the DVE. |
+| `Poetry` | 1.4.2 | Build and venv tool used for the DVE. |
+| `Java` | java liberica-1.8.0 | `Java` version required for `PySpark`. |
+| `pre-commit` | 2.21.0 | Currently installed as part of the `poetry` venv but seperate installation is fine. |
+| `commitizen` | 3.9.1 | Like `pre-commit`, installed as part of the `poetry` venv but seperate installation is fine. This is used to manage commits and automated semantic versioning. |
+| `git-secrets` | Latest | Utilised as part of the `pre-commit` to ensure that no secrets are commited to the repository. There is a helper installation script within [scripts](/scripts/git-secrets/). |
+
+Additionally, we have created a [asdf support](.tool-versions) and [mise-en-toml](.mise.toml) for those utilising `asdf` or `mise-en-toml` software.
+
+## Testing Requirements
+
+Testing requirements are given in [pyproject.toml](./poetry.toml#48) under `tool.poetry.group.test.dependencies`. These are always pinned versions for consistency, but should be updated regularly if new versions are released. The following core packages are used for testing:
+ - [pytest](https://docs.pytest.org/en/stable/): Used for Python unit tests, and some small e2e tests which check coverage.
+ - [behave](https://github.com/behave/behave): Used for full, business-driven end-to-end tests.
+ - [coverage](https://coverage.readthedocs.io/en/7.10.7/): Used to get coverage for `pytest` tests.
+
+## Linting/Formatting/Type Checking Requirements
+
+Additional dependencies for linting, type checking, and formatting are listed within the [pyproject.toml](./poetry.toml#58) under `tool.poetry.group.lint.dependencies`. Like the testing requirements, we use pinned versions for these.
+
+This mostly breaks down to:
+- [black](https://github.com/psf/black): a tool to format Python, which ensures consistency of formatting across the project. We use a line length of 100.
+- [isort](https://pycqa.github.io/isort/): a tool to organise imports in a consistent way across the project.
+- [mypy](https://github.com/python/mypy): a type checker for Python. This ensures that our function signatures remain accurate and is very useful for spotting type-related issues (a significant category of bugs).
+- [pylint](https://pylint.readthedocs.io/en/stable/): a (very aggressive) linter for Python, which prevents errors and code style violations across the project. This pushes us to document the code and is often quite good at highlighting overly complex code.
+
+We use these tools to ensure that code quality is not excessively compromised, even when working at pace.
+
+## Installation for Development
+
+We are utilising Poetry for build dependency management and packaging. If you're on a system that has `Make` available, you can simply run `make install` to setup a local virtual environment with all the dependencies installed (this won't install Poetry for you).
+
+## Testing
+
+Tests should be run after installing the package for development as outlined above.
+- To run unit tests without coverage, run `poetry run pytest tests/`
+- To run the unit tests with coverage, run `poetry run coverage run`
+- To check the coverage run `poetry run coverage report -m`
+- To run the behave tests, run `poetry run behave tests/features` (these are not included in coverage calculations)
+
+## Submitting a pull request
+
+If you want to contribute to the DVE then please follow the steps below:
+1. Fork the repository.
+2. Configure and install the dependencies.
+3. Ensure that new changes are fully tested and that you are reguarlly checking for changes within the DVE repository to sort any potential merge conflicts.
+4. Ensure linting passes.
+5. Push and then create a pull request from your fork to our repository.
+
+Your pull request will then be reviewed. You may receive some feedback and suggested changes before it can be approved and merged.

Makefile

@@ -0,0 +1,32 @@
+activate = poetry run
+
+# dev
+install:
+	poetry lock
+	poetry install --with dev,test
+
+# dist
+wheel:
+	poetry build -f wheel
+	poetry build -f sdist
+
+dist: wheel
+
+# testing
+behave:
+	${activate} behave
+
+pytest:
+	${activate} pytest tests/
+
+all-tests: pytest behave
+
+coverage:
+	$(activate) coverage run
+	$(activate) coverage combine
+	$(activate) coverage report
+	$(activate) coverage xml
+
+# pre-commit
+pre-commit-all:
+	${activate} pre-commit run --all-files

behave.ini

@@ -0,0 +1,2 @@
+[behave]
+paths=tests/features