diff --git a/docs/source/basics.md b/docs/source/basics.md index d71f110c4..31d3d721a 100644 --- a/docs/source/basics.md +++ b/docs/source/basics.md @@ -1021,7 +1021,7 @@ stages: has_premium: true ``` -## Including external files +### Including external files Even with being able to use anchors within the same file, there is often some data which either you want to keep in a separate (possibly autogenerated) file, @@ -1074,6 +1074,109 @@ The environment variable TAVERN_INCLUDE can contain a : separated list of paths to search for include files. Each path in TAVERN_INCLUDE has environment variables expanded before it is searched. +### Default document merge-down + +When multiple tests are defined in a single Tavern file, you may want to share common configuration across all tests +without repeating it. Tavern supports this via the `is_defaults: true` flag in a top-level document. Any document with +this flag set will have its contents merged into all subsequent test documents in the same file. + +This is useful for: + +- Shared includes (common variables, stages, or configuration) +- Default MQTT connection settings +- Common authentication or headers +- Shared test setup stages + +#### Example: Shared includes and configuration + +```yaml +--- +is_defaults: true + +includes: + - !include common.yaml + +paho-mqtt: + auth: + username: tavern + password: tavern + connect: + host: localhost + port: 9001 + client: + transport: websockets +--- + +test_name: Test mqtt message echo json + +stages: + - name: Echo json + mqtt_publish: + topic: /device/test/echo + json: + message: hello world + mqtt_response: + topic: /device/test/echo/response + json: + message: hello world + timeout: 5 +--- + +test_name: Test mqtt message echo binary + +stages: + - name: Echo binary + mqtt_publish: + topic: /device/test/echo + payload: hello world + mqtt_response: + topic: /device/test/echo/response + payload: hello world + timeout: 5 +``` + +In this example, both tests inherit the MQTT connection settings and includes from the defaults document, avoiding +duplication. + +#### Example: Shared HTTP test configuration + +```yaml +--- +is_defaults: true + +includes: + - !include common.yaml +--- + +test_name: Test redirecting loops + +stages: + - name: Expect a 302 without setting the flag + max_retries: 2 + request: + follow_redirects: true + url: "{host}/redirect/loop" + response: + status_code: 200 + +--- + +test_name: Using a shared stage from common.yaml + +stages: + - type: ref + id: typetoken-anything-match +``` + +Both tests automatically include the shared configuration from `common.yaml` without needing to specify it individually. + +Note: + +- Only the first document in a file can use `is_defaults: true` +- The defaults document cannot contain test definitions (no `test_name` or `stages`) +- Values in the defaults document are merged with subsequent documents, with the test document taking precedence for + conflicting keys + ### Including global configuration files If you do want to run the same tests with a different input data, this can be @@ -1876,10 +1979,12 @@ stages: ##### Skipping stages with simpleeval expressions -Stages can be skipped by using a `skip` key that contains a [simpleeval](https://pypi.org/project/simpleeval/) expression. +Stages can be skipped by using a `skip` key that contains a [simpleeval](https://pypi.org/project/simpleeval/) +expression. This allows for more complex conditional logic to determine if a stage should be skipped. Example: + ```yaml stages: - name: Skip based on variable value diff --git a/example/mqtt/tests/test_mqtt_merge_down.tavern.yaml b/example/mqtt/tests/test_mqtt_merge_down.tavern.yaml new file mode 100644 index 000000000..6b43c6538 --- /dev/null +++ b/example/mqtt/tests/test_mqtt_merge_down.tavern.yaml @@ -0,0 +1,50 @@ +--- +is_defaults: true + +paho-mqtt: + auth: + username: tavern + password: tavern + # tls: + # enable: true + connect: + host: localhost + port: 9001 + timeout: 3 + client: + transport: websockets + client_id: tavern-tester-{random_device_id} + +includes: + - !include common.yaml + - stages: + - id: create_test_device + name: create device + request: + url: "{host}/create_device" + method: PUT + json: + device_id: "{random_device_id}" + clean: True + response: + status_code: 201 + +--- + +test_name: Test mqtt message echo json + +stages: + - type: ref + id: create_test_device + + - name: Echo json + mqtt_publish: + topic: /device/{random_device_id}/echo + json: + message: hello world + mqtt_response: + topic: /device/{random_device_id}/echo/response + json: + message: hello world + timeout: 5 + qos: 1 diff --git a/tavern/_core/dict_util.py b/tavern/_core/dict_util.py index 04aaf6ba9..e1231f716 100644 --- a/tavern/_core/dict_util.py +++ b/tavern/_core/dict_util.py @@ -237,7 +237,6 @@ def deep_dict_merge(initial_dct: dict, merge_dct: Mapping) -> dict: dict_merge recurses down into dicts nested to an arbitrary depth and returns the merged dict. Keys values present in merge_dct take precedence over values in initial_dct. - Modified from: https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 Params: initial_dct: dict onto which the merge is executed @@ -246,15 +245,9 @@ def deep_dict_merge(initial_dct: dict, merge_dct: Mapping) -> dict: Returns: recursively merged dict """ - dct = initial_dct.copy() - - for k in merge_dct: - if k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], Mapping): - dct[k] = deep_dict_merge(dct[k], merge_dct[k]) - else: - dct[k] = merge_dct[k] - - return dct + initial_box = Box(initial_dct) + initial_box.merge_update(merge_dct) + return initial_box.to_dict() _CanCheck = Sequence | Mapping | set | Collection diff --git a/tavern/_core/plugins.py b/tavern/_core/plugins.py index c2ca409fa..3dfa3ac43 100644 --- a/tavern/_core/plugins.py +++ b/tavern/_core/plugins.py @@ -172,7 +172,7 @@ def get_extra_sessions(test_spec: Mapping, test_block_config: TestConfig) -> dic sessions = {} - plugins = load_plugins(test_block_config) + plugins: list[_Plugin] = load_plugins(test_block_config) for p in plugins: if any( @@ -182,8 +182,8 @@ def get_extra_sessions(test_spec: Mapping, test_block_config: TestConfig) -> dic logger.debug( "Initialising session for %s (%s)", p.name, p.plugin.session_type ) - session_spec = test_spec.get(p.name, {}) - formatted = format_keys(session_spec, test_block_config.variables) + session_spec: dict = test_spec.get(p.name, {}) + formatted: dict = format_keys(session_spec, test_block_config.variables) sessions[p.name] = p.plugin.session_type(**formatted) return sessions @@ -229,6 +229,7 @@ def get_request_type( # We've validated that 1 and only 1 is there, so just loop until the first # one is found + request_class: type[BaseRequest] | None = None for p in plugins: try: request_args = stage[p.plugin.request_block_name] @@ -242,6 +243,9 @@ def get_request_type( ) break + if not request_class: + raise exceptions.MissingSettingsError("No request type found") + request_maker = request_class(session, request_args, test_block_config) return request_maker diff --git a/tavern/_core/pytest/file.py b/tavern/_core/pytest/file.py index 5b2218f62..f96626126 100644 --- a/tavern/_core/pytest/file.py +++ b/tavern/_core/pytest/file.py @@ -9,7 +9,7 @@ from typing import Any, Union import pytest -import yaml +import yaml.parser from box import Box from pytest import Mark @@ -445,16 +445,62 @@ def collect(self) -> Iterator[YamlItem]: except yaml.parser.ParserError as e: raise exceptions.BadSchemaError from e - for test_spec in all_tests: + defaults_doc = None + + for document_idx, test_spec in enumerate(all_tests): if not test_spec: logger.warning("Empty document in input file '%s'", self.path) continue + # Check for explicit defaults marker and validate position + is_defaults: bool = test_spec.pop("is_defaults", False) + + if is_defaults and document_idx > 0: + raise exceptions.BadSchemaError( + f"'is_defaults' can only be used in the first YAML document, " + f"but found it in document {document_idx + 1} of '{self.path}'" + ) + + # Determine if this document is a test or defaults + is_test_doc = "test_name" in test_spec and "stages" in test_spec + + if document_idx == 0 and is_defaults: + if is_test_doc: + raise exceptions.BadSchemaError( + f"First document in '{self.path}' is marked as defaults but also contains a 'test_name' and 'stages'" + ) + + # First document is explicitly marked as defaults + logger.info( + "Using first document as defaults for %s", + self.path, + ) + defaults_doc = test_spec + continue + elif not is_test_doc: + # Document is neither a valid test nor valid defaults + if document_idx == 0: + raise exceptions.BadSchemaError( + f"First document in '{self.path}' is missing 'test_name' or 'stages'. " + f"If this is meant to be defaults for the file, add 'is_defaults: true'. " + f"If this is meant to be a test, add both 'test_name' and 'stages'." + ) + else: + raise exceptions.BadSchemaError( + f"Document {document_idx + 1} in '{self.path}' is missing 'test_name' or 'stages'" + ) + + # Merge defaults into test spec if defaults were defined + if defaults_doc: + test_spec = deep_dict_merge(defaults_doc, test_spec) + try: for i in self._generate_items(test_spec): i.initialise_fixture_attrs() yield i except (TypeError, KeyError) as e: + # If there was one of these errors, we can probably figure out + # if the error is from a bad test layout by calling verify_tests try: verify_tests(test_spec, with_plugins=False) except Exception as e2: diff --git a/tavern/_core/pytest/item.py b/tavern/_core/pytest/item.py index 317332576..fa07f328c 100644 --- a/tavern/_core/pytest/item.py +++ b/tavern/_core/pytest/item.py @@ -25,6 +25,7 @@ from tavern._core.report import attach_text from tavern._core.run import run_test from tavern._core.schema.files import verify_tests +from tavern._core.stage_lines import start_mark from .config import TestConfig from .util import load_global_cfg @@ -130,7 +131,7 @@ def initialise_fixture_attrs(self) -> None: def location(self): """get location in file""" location = super().location - location = (location[0], self.spec.start_mark.line, location[2]) + location = (location[0], start_mark(self.spec).line, location[2]) return location # Hack to stop issue with pytest-rerunfailures diff --git a/tavern/_core/schema/tests.jsonschema.yaml b/tavern/_core/schema/tests.jsonschema.yaml index 5f935573c..e9de7cbdc 100644 --- a/tavern/_core/schema/tests.jsonschema.yaml +++ b/tavern/_core/schema/tests.jsonschema.yaml @@ -525,15 +525,17 @@ definitions: type: object additionalProperties: false -required: - - test_name - - stages properties: test_name: type: string description: Name of test + is_defaults: + type: boolean + description: Whether this document contains default values to be merged with subsequent test documents + default: false + _xfail: oneOf: - type: string diff --git a/tests/integration/test_merge_down.tavern.yaml b/tests/integration/test_merge_down.tavern.yaml new file mode 100644 index 000000000..295c658f4 --- /dev/null +++ b/tests/integration/test_merge_down.tavern.yaml @@ -0,0 +1,22 @@ +--- +is_defaults: true + +includes: + - !include common.yaml +--- +test_name: Test redirecting loops + +stages: + - name: Expect a 302 without setting the flag + max_retries: 2 + request: + follow_redirects: true + url: "{host}/redirect/loop" + response: + status_code: 200 +--- +test_name: Using a shared stage from common.yaml + +stages: + - type: ref + id: typetoken-anything-match diff --git a/tests/unit/test_files.py b/tests/unit/test_files.py new file mode 100644 index 000000000..cdfe032b1 --- /dev/null +++ b/tests/unit/test_files.py @@ -0,0 +1,115 @@ +import contextlib +import dataclasses +import pathlib +import tempfile +from collections.abc import Callable, Generator +from typing import Any +from unittest.mock import Mock + +import pytest +import yaml + +from tavern._core import exceptions +from tavern._core.pytest.file import YamlFile +from tavern._core.pytest.item import YamlItem + + +@pytest.fixture(scope="function") +def tavern_test_content(): + """return some example tests""" + + test_docs = [ + {"test_name": "First test", "stages": [{"name": "stage 1"}]}, + {"test_name": "Second test", "stages": [{"name": "stage 2"}]}, + {"test_name": "Third test", "stages": [{"name": "stage 3"}]}, + ] + + return test_docs + + +@contextlib.contextmanager +def tavern_test_file(test_content: list[Any]) -> Generator[pathlib.Path, Any, None]: + """Create a temporary YAML file with multiple documents""" + + with tempfile.TemporaryDirectory() as tmpdir: + file_path = pathlib.Path(tmpdir) / "test.yaml" + + # Write the documents to the file + with file_path.open("w", encoding="utf-8") as f: + for doc in test_content: + yaml.dump(doc, f) + f.write("---\n") + + yield file_path + + +@dataclasses.dataclass +class Opener: + """Simple mock for generating items because pytest makes it hard to wrap + their internal functionality""" + + path: pathlib.Path + _generate_items: Callable[[dict], Any] + + +class TestGenerateFiles: + @pytest.mark.parametrize("with_merge_down_test", (True, False)) + def test_multiple_documents(self, tavern_test_content, with_merge_down_test): + """Verify that multiple documents in a YAML file result in multiple tests""" + + # Collect all tests + if with_merge_down_test: + tavern_test_content.insert(0, {"includes": [], "is_defaults": True}) + + def generate_yamlitem(test_spec): + mock = Mock(spec=YamlItem) + mock.name = test_spec["test_name"] + yield mock + + with tavern_test_file(tavern_test_content) as filename: + tests = list( + YamlFile.collect( + Opener( + path=filename, + _generate_items=generate_yamlitem, + ) + ) + ) + + assert len(tests) == 3 + + # Verify each test has the correct name + expected_names = ["First test", "Second test", "Third test"] + for test, expected_name in zip(tests, expected_names): + assert test.name == expected_name + + @pytest.mark.parametrize( + "content, exception", + ( + ({"kookdff": "?A?A??"}, exceptions.BadSchemaError), + ({"test_name": "name", "stages": [{"name": "lflfl"}]}, TypeError), + ), + ) + def test_reraise_exception( + self, tavern_test_content, content: dict, exception: BaseException + ): + """Verify that exceptions are properly reraised when loading YAML test files. + + Test that when an exception occurs during test generation, it is properly + reraised as a schema error if the schema is bad.""" + + def raise_error(test_spec): + raise TypeError + + tavern_test_content.insert(0, content) + + with tavern_test_file(tavern_test_content) as filename: + with pytest.raises(exception): + list( + YamlFile.collect( + Opener( + path=filename, + _generate_items=raise_error, + ) + ) + )