From 88201540e61e866abcbccae9ced7f07195a89116 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Thu, 9 Apr 2026 15:09:33 -0400 Subject: [PATCH 1/4] add target requirements --- pyrit/prompt_target/__init__.py | 2 + .../common/target_requirements.py | 45 ++++++ tests/unit/target/test_target_requirements.py | 153 ++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 pyrit/prompt_target/common/target_requirements.py create mode 100644 tests/unit/target/test_target_requirements.py diff --git a/pyrit/prompt_target/__init__.py b/pyrit/prompt_target/__init__.py index 234056c32..c71dca408 100644 --- a/pyrit/prompt_target/__init__.py +++ b/pyrit/prompt_target/__init__.py @@ -20,6 +20,7 @@ UnsupportedCapabilityBehavior, ) from pyrit.prompt_target.common.target_configuration import TargetConfiguration +from pyrit.prompt_target.common.target_requirements import TargetRequirements from pyrit.prompt_target.common.utils import limit_requests_per_minute from pyrit.prompt_target.gandalf_target import GandalfLevel, GandalfTarget from pyrit.prompt_target.http_target.http_target import HTTPTarget @@ -77,6 +78,7 @@ "RealtimeTarget", "TargetCapabilities", "TargetConfiguration", + "TargetRequirements", "UnsupportedCapabilityBehavior", "TextTarget", "WebSocketCopilotTarget", diff --git a/pyrit/prompt_target/common/target_requirements.py b/pyrit/prompt_target/common/target_requirements.py new file mode 100644 index 000000000..34c694f8d --- /dev/null +++ b/pyrit/prompt_target/common/target_requirements.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from pyrit.prompt_target.common.target_capabilities import CapabilityName + +if TYPE_CHECKING: + from pyrit.prompt_target.common.target_configuration import TargetConfiguration + + +@dataclass(frozen=True) +class TargetRequirements: + """ + Declarative description of what a consumer (attack, converter, scorer) + requires from a target. + + Consumers define their requirements once and validate them against a + ``TargetConfiguration`` at construction time. This replaces ad-hoc + ``isinstance`` checks and scattered capability branching. + """ + + # The set of capabilities the consumer requires. + required_capabilities: frozenset[CapabilityName] = field(default_factory=frozenset) + + def validate(self, *, configuration: TargetConfiguration) -> None: + """ + Validate that the target configuration can satisfy all requirements. + + Iterates over every required capability and delegates to + ``TargetConfiguration.ensure_can_handle``, which checks native support + first and then consults the handling policy. + + Args: + configuration: The target configuration to validate against. + + Raises: + ValueError: If any required capability is missing and the policy + does not allow adaptation. + """ + for capability in sorted(self.required_capabilities, key=lambda c: c.value): + configuration.ensure_can_handle(capability=capability) diff --git a/tests/unit/target/test_target_requirements.py b/tests/unit/target/test_target_requirements.py new file mode 100644 index 000000000..04c325be5 --- /dev/null +++ b/tests/unit/target/test_target_requirements.py @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest + +from pyrit.prompt_target.common.target_capabilities import ( + CapabilityHandlingPolicy, + CapabilityName, + TargetCapabilities, + UnsupportedCapabilityBehavior, +) +from pyrit.prompt_target.common.target_configuration import TargetConfiguration +from pyrit.prompt_target.common.target_requirements import TargetRequirements + + +@pytest.fixture +def adapt_all_policy(): + return CapabilityHandlingPolicy( + behaviors={ + CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT, + CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.ADAPT, + CapabilityName.JSON_SCHEMA: UnsupportedCapabilityBehavior.RAISE, + CapabilityName.JSON_OUTPUT: UnsupportedCapabilityBehavior.RAISE, + CapabilityName.MULTI_MESSAGE_PIECES: UnsupportedCapabilityBehavior.RAISE, + CapabilityName.EDITABLE_HISTORY: UnsupportedCapabilityBehavior.RAISE, + } + ) + + +# --------------------------------------------------------------------------- +# Construction +# --------------------------------------------------------------------------- + + +def test_init_default_has_empty_capabilities(): + reqs = TargetRequirements() + assert reqs.required_capabilities == frozenset() + + +def test_init_with_capabilities(): + reqs = TargetRequirements( + required_capabilities=frozenset({CapabilityName.MULTI_TURN, CapabilityName.SYSTEM_PROMPT}) + ) + assert CapabilityName.MULTI_TURN in reqs.required_capabilities + assert CapabilityName.SYSTEM_PROMPT in reqs.required_capabilities + + +def test_frozen_dataclass_is_immutable(): + reqs = TargetRequirements() + with pytest.raises(AttributeError): + reqs.required_capabilities = frozenset({CapabilityName.MULTI_TURN}) + + +# --------------------------------------------------------------------------- +# validate — all pass +# --------------------------------------------------------------------------- + + +def test_validate_passes_when_target_supports_all_natively(): + caps = TargetCapabilities(supports_multi_turn=True, supports_system_prompt=True) + config = TargetConfiguration(capabilities=caps) + reqs = TargetRequirements( + required_capabilities=frozenset({CapabilityName.MULTI_TURN, CapabilityName.SYSTEM_PROMPT}) + ) + reqs.validate(configuration=config) + + +def test_validate_passes_when_policy_is_adapt(adapt_all_policy): + caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False) + config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) + reqs = TargetRequirements( + required_capabilities=frozenset({CapabilityName.MULTI_TURN, CapabilityName.SYSTEM_PROMPT}) + ) + reqs.validate(configuration=config) + + +def test_validate_passes_with_empty_requirements(): + caps = TargetCapabilities(supports_multi_turn=True, supports_system_prompt=True) + config = TargetConfiguration(capabilities=caps) + reqs = TargetRequirements() + reqs.validate(configuration=config) + + +# --------------------------------------------------------------------------- +# validate — failures +# --------------------------------------------------------------------------- + + +def test_validate_raises_when_capability_missing_and_policy_raise(adapt_all_policy): + # Build with ADAPT so construction succeeds, then override policy for validate. + caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=True) + raise_policy = CapabilityHandlingPolicy( + behaviors={ + CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT, + CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.RAISE, + } + ) + # multi_turn is missing + RAISE → pipeline construction raises, so build with ADAPT first + config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) + # Swap in a RAISE policy to test validate behavior + config._policy = raise_policy + reqs = TargetRequirements(required_capabilities=frozenset({CapabilityName.MULTI_TURN})) + with pytest.raises(ValueError, match="supports_multi_turn"): + reqs.validate(configuration=config) + + +def test_validate_raises_for_non_normalizable_capability(adapt_all_policy): + caps = TargetCapabilities(supports_editable_history=False) + config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) + reqs = TargetRequirements(required_capabilities=frozenset({CapabilityName.EDITABLE_HISTORY})) + with pytest.raises(ValueError, match="supports_editable_history"): + reqs.validate(configuration=config) + + +def test_validate_raises_on_first_unsatisfied_capability(): + """When multiple capabilities are missing, validate raises on the first (sorted) one.""" + # Both missing, both ADAPT → construction OK, then swap to RAISE to test validate + adapt_policy = CapabilityHandlingPolicy( + behaviors={ + CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT, + CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.ADAPT, + } + ) + caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False) + config = TargetConfiguration(capabilities=caps, policy=adapt_policy) + raise_policy = CapabilityHandlingPolicy( + behaviors={ + CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.RAISE, + CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.RAISE, + } + ) + config._policy = raise_policy + reqs = TargetRequirements( + required_capabilities=frozenset({CapabilityName.MULTI_TURN, CapabilityName.SYSTEM_PROMPT}) + ) + with pytest.raises(ValueError): + reqs.validate(configuration=config) + + +def test_validate_mixed_adapt_and_raise(adapt_all_policy): + """One capability adapts but another raises — validate should raise.""" + caps = TargetCapabilities( + supports_multi_turn=False, supports_system_prompt=False, supports_json_output=False + ) + config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) + # multi_turn and system_prompt => ADAPT (OK), json_output => RAISE (fail) + reqs = TargetRequirements( + required_capabilities=frozenset( + {CapabilityName.MULTI_TURN, CapabilityName.SYSTEM_PROMPT, CapabilityName.JSON_OUTPUT} + ) + ) + with pytest.raises(ValueError, match="supports_json_output"): + reqs.validate(configuration=config) From 08168d3d45e2d6130fd54bb39d1a73d940c02ab5 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Thu, 9 Apr 2026 16:17:06 -0400 Subject: [PATCH 2/4] fix test issue and aggregate errors --- .../common/target_requirements.py | 16 ++++- tests/unit/target/test_target_requirements.py | 66 +++++++------------ 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/pyrit/prompt_target/common/target_requirements.py b/pyrit/prompt_target/common/target_requirements.py index 34c694f8d..1cd787a22 100644 --- a/pyrit/prompt_target/common/target_requirements.py +++ b/pyrit/prompt_target/common/target_requirements.py @@ -32,14 +32,24 @@ def validate(self, *, configuration: TargetConfiguration) -> None: Iterates over every required capability and delegates to ``TargetConfiguration.ensure_can_handle``, which checks native support - first and then consults the handling policy. + first and then consults the handling policy. All violations are + collected and reported in a single ``ValueError``. Args: - configuration: The target configuration to validate against. + configuration (TargetConfiguration): The target configuration to validate against. Raises: ValueError: If any required capability is missing and the policy does not allow adaptation. """ + errors: list[str] = [] for capability in sorted(self.required_capabilities, key=lambda c: c.value): - configuration.ensure_can_handle(capability=capability) + try: + configuration.ensure_can_handle(capability=capability) + except ValueError as exc: + errors.append(str(exc)) + if errors: + raise ValueError( + f"Target does not satisfy {len(errors)} required capability(ies):\n" + + "\n".join(f" - {e}" for e in errors) + ) diff --git a/tests/unit/target/test_target_requirements.py b/tests/unit/target/test_target_requirements.py index 04c325be5..30508af07 100644 --- a/tests/unit/target/test_target_requirements.py +++ b/tests/unit/target/test_target_requirements.py @@ -86,62 +86,46 @@ def test_validate_passes_with_empty_requirements(): # --------------------------------------------------------------------------- -def test_validate_raises_when_capability_missing_and_policy_raise(adapt_all_policy): - # Build with ADAPT so construction succeeds, then override policy for validate. - caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=True) - raise_policy = CapabilityHandlingPolicy( - behaviors={ - CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT, - CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.RAISE, - } - ) - # multi_turn is missing + RAISE → pipeline construction raises, so build with ADAPT first - config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) - # Swap in a RAISE policy to test validate behavior - config._policy = raise_policy - reqs = TargetRequirements(required_capabilities=frozenset({CapabilityName.MULTI_TURN})) - with pytest.raises(ValueError, match="supports_multi_turn"): +def test_validate_raises_when_capability_missing_and_no_policy(): + # EDITABLE_HISTORY has no normalizer and no handling policy — validate raises. + caps = TargetCapabilities(supports_editable_history=False, supports_multi_turn=True, supports_system_prompt=True) + config = TargetConfiguration(capabilities=caps) + reqs = TargetRequirements(required_capabilities=frozenset({CapabilityName.EDITABLE_HISTORY})) + with pytest.raises(ValueError, match="supports_editable_history"): reqs.validate(configuration=config) -def test_validate_raises_for_non_normalizable_capability(adapt_all_policy): - caps = TargetCapabilities(supports_editable_history=False) +def test_validate_raises_when_capability_missing_and_policy_raise(adapt_all_policy): + # json_output is missing and the policy is RAISE — validate raises. + caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False, supports_json_output=False) config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) - reqs = TargetRequirements(required_capabilities=frozenset({CapabilityName.EDITABLE_HISTORY})) - with pytest.raises(ValueError, match="supports_editable_history"): + reqs = TargetRequirements(required_capabilities=frozenset({CapabilityName.JSON_OUTPUT})) + with pytest.raises(ValueError, match="supports_json_output"): reqs.validate(configuration=config) -def test_validate_raises_on_first_unsatisfied_capability(): - """When multiple capabilities are missing, validate raises on the first (sorted) one.""" - # Both missing, both ADAPT → construction OK, then swap to RAISE to test validate - adapt_policy = CapabilityHandlingPolicy( - behaviors={ - CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.ADAPT, - CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.ADAPT, - } - ) - caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False) - config = TargetConfiguration(capabilities=caps, policy=adapt_policy) - raise_policy = CapabilityHandlingPolicy( - behaviors={ - CapabilityName.SYSTEM_PROMPT: UnsupportedCapabilityBehavior.RAISE, - CapabilityName.MULTI_TURN: UnsupportedCapabilityBehavior.RAISE, - } +def test_validate_collects_all_unsatisfied_capabilities(adapt_all_policy): + """When multiple capabilities are missing, validate reports all violations.""" + caps = TargetCapabilities( + supports_multi_turn=False, + supports_system_prompt=False, + supports_json_output=False, + supports_editable_history=False, ) - config._policy = raise_policy + config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) + # json_output => RAISE, editable_history => no policy (raises) reqs = TargetRequirements( - required_capabilities=frozenset({CapabilityName.MULTI_TURN, CapabilityName.SYSTEM_PROMPT}) + required_capabilities=frozenset({CapabilityName.JSON_OUTPUT, CapabilityName.EDITABLE_HISTORY}) ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="2 required capability") as exc_info: reqs.validate(configuration=config) + assert "supports_json_output" in str(exc_info.value) + assert "supports_editable_history" in str(exc_info.value) def test_validate_mixed_adapt_and_raise(adapt_all_policy): """One capability adapts but another raises — validate should raise.""" - caps = TargetCapabilities( - supports_multi_turn=False, supports_system_prompt=False, supports_json_output=False - ) + caps = TargetCapabilities(supports_multi_turn=False, supports_system_prompt=False, supports_json_output=False) config = TargetConfiguration(capabilities=caps, policy=adapt_all_policy) # multi_turn and system_prompt => ADAPT (OK), json_output => RAISE (fail) reqs = TargetRequirements( From f55031050598cb4b6b0ea752210db3efe0fec8cd Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Thu, 9 Apr 2026 16:50:16 -0400 Subject: [PATCH 3/4] pre-commit --- pyrit/prompt_target/common/target_requirements.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyrit/prompt_target/common/target_requirements.py b/pyrit/prompt_target/common/target_requirements.py index 1cd787a22..95182b47b 100644 --- a/pyrit/prompt_target/common/target_requirements.py +++ b/pyrit/prompt_target/common/target_requirements.py @@ -6,9 +6,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING -from pyrit.prompt_target.common.target_capabilities import CapabilityName - if TYPE_CHECKING: + from pyrit.prompt_target.common.target_capabilities import CapabilityName from pyrit.prompt_target.common.target_configuration import TargetConfiguration From 4902add3be2c7deaf043bf75f60a1c24656c42e9 Mon Sep 17 00:00:00 2001 From: hannahwestra25 Date: Fri, 10 Apr 2026 15:08:07 -0400 Subject: [PATCH 4/4] remove test --- tests/unit/target/test_target_requirements.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/unit/target/test_target_requirements.py b/tests/unit/target/test_target_requirements.py index 30508af07..002ccf086 100644 --- a/tests/unit/target/test_target_requirements.py +++ b/tests/unit/target/test_target_requirements.py @@ -45,12 +45,6 @@ def test_init_with_capabilities(): assert CapabilityName.SYSTEM_PROMPT in reqs.required_capabilities -def test_frozen_dataclass_is_immutable(): - reqs = TargetRequirements() - with pytest.raises(AttributeError): - reqs.required_capabilities = frozenset({CapabilityName.MULTI_TURN}) - - # --------------------------------------------------------------------------- # validate — all pass # ---------------------------------------------------------------------------