From b3c12839221b0f2c2f527b7b1cc93fd717da8769 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Thu, 12 Mar 2026 17:06:14 -0700 Subject: [PATCH 1/3] fix: skip server evaluation when no controls apply --- sdks/python/src/agent_control/evaluation.py | 53 +++++++++++--- sdks/python/tests/test_local_evaluation.py | 80 +++++++++++++++++---- 2 files changed, 110 insertions(+), 23 deletions(-) diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index e30bb3e2..979292ce 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -134,6 +134,41 @@ class _ControlAdapter: control: "ControlDefinition" +def _has_applicable_server_controls( + controls: list[dict[str, Any]], + request: EvaluationRequest, +) -> bool: + """Return whether any well-formed server control applies to this request. + + If any server control cannot be parsed locally, this returns True so the SDK + still defers to the server for authoritative handling. + """ + server_controls: list[_ControlAdapter] = [] + + for control in controls: + try: + control_def = ControlDefinition.model_validate(control["control"]) + server_controls.append( + _ControlAdapter( + id=control["id"], + name=control["name"], + control=control_def, + ) + ) + except Exception: + # Preserve existing fail-open behavior for malformed server controls. + return True + + if not server_controls: + return False + + applicable_controls = ControlEngine( + server_controls, + context="server", + ).get_applicable_controls(request) + return bool(applicable_controls) + + def _merge_results( local_result: "EvaluationResponse", server_result: "EvaluationResponse", @@ -212,7 +247,7 @@ async def check_evaluation_with_local( # Partition controls by local flag local_controls: list[_ControlAdapter] = [] parse_errors: list[ControlMatch] = [] - has_server_controls = False + server_controls: list[dict[str, Any]] = [] for control in controls: control_data = control.get("control", {}) @@ -220,7 +255,7 @@ async def check_evaluation_with_local( is_local = execution == "sdk" if not is_local: - has_server_controls = True + server_controls.append(control) continue try: @@ -272,6 +307,12 @@ async def check_evaluation_with_local( ) ) + request = EvaluationRequest( + agent_name=normalized_name, + step=step, + stage=stage, + ) + def _with_parse_errors(result: EvaluationResult) -> EvaluationResult: if not parse_errors: return result @@ -285,12 +326,6 @@ def _with_parse_errors(result: EvaluationResult) -> EvaluationResult: non_matches=result.non_matches, ) - request = EvaluationRequest( - agent_name=normalized_name, - step=step, - stage=stage, - ) - local_result: EvaluationResponse | None = None if local_controls: engine = ControlEngine(local_controls, context="sdk") @@ -317,7 +352,7 @@ def _with_parse_errors(result: EvaluationResult) -> EvaluationResult: ) ) - if has_server_controls: + if _has_applicable_server_controls(server_controls, request): request_payload = request.model_dump(mode="json", exclude_none=True) headers: dict[str, str] = {} if trace_id: diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index 94661ab0..72e21cd5 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -12,22 +12,18 @@ from unittest.mock import AsyncMock, MagicMock import pytest - +from agent_control.client import AgentControlClient +from agent_control.evaluation import ( + _merge_results, + check_evaluation_with_local, +) from agent_control_models import ( ControlMatch, EvaluationResponse, - EvaluationResult, EvaluatorResult, Step, ) -from agent_control.client import AgentControlClient -from agent_control.evaluation import ( - _merge_results, - check_evaluation_with_local, -) - - # ============================================================================= # Test Fixtures # ============================================================================= @@ -55,26 +51,36 @@ def make_control_dict( control_id: int, name: str, *, + enabled: bool = True, execution: str = "server", evaluator: str = "regex", pattern: str = r"test", action: str = "deny", step_type: str = "llm", stage: str = "pre", + step_names: list[str] | None = None, + step_name_regex: str | None = None, path: str | None = None, ) -> dict[str, Any]: """Create a control dict like what initAgent returns.""" # Default path based on payload type if path is None: path = "input" + + scope: dict[str, Any] = {"step_types": [step_type], "stages": [stage]} + if step_names is not None: + scope["step_names"] = step_names + if step_name_regex is not None: + scope["step_name_regex"] = step_name_regex + return { "id": control_id, "name": name, "control": { "description": f"Test control {name}", - "enabled": True, + "enabled": enabled, "execution": execution, - "scope": {"step_types": [step_type], "stages": [stage]}, + "scope": scope, "selector": {"path": path}, "evaluator": { "name": evaluator, @@ -254,6 +260,53 @@ async def test_server_only_controls_calls_server(self, agent_name, llm_payload): assert result.is_safe is True + @pytest.mark.asyncio + @pytest.mark.parametrize( + "control_kwargs", + [ + pytest.param({"enabled": False}, id="disabled"), + pytest.param({"stage": "post"}, id="stage_mismatch"), + pytest.param({"step_type": "tool"}, id="step_type_mismatch"), + pytest.param({"step_names": ["send_email"]}, id="step_name_mismatch"), + pytest.param({"step_name_regex": r"^send_.*$"}, id="step_name_regex_mismatch"), + ], + ) + async def test_non_applicable_server_controls_do_not_call_server( + self, + agent_name, + llm_payload, + control_kwargs, + ): + """Server evaluation should be skipped when no server control applies.""" + controls = [ + make_control_dict( + 1, + "server_ctrl", + execution="server", + **control_kwargs, + ), + ] + + client = MagicMock(spec=AgentControlClient) + mock_response = MagicMock() + mock_response.json.return_value = {"is_safe": True, "confidence": 1.0} + mock_response.raise_for_status = MagicMock() + client.http_client = AsyncMock() + client.http_client.post = AsyncMock(return_value=mock_response) + + result = await check_evaluation_with_local( + client=client, + agent_name=agent_name, + step=llm_payload, + stage="pre", + controls=controls, + ) + + client.http_client.post.assert_not_called() + assert result.is_safe is True + assert result.confidence == 1.0 + assert result.matches is None + @pytest.mark.asyncio async def test_local_deny_short_circuits(self, agent_name, llm_payload): """Local deny should return immediately without calling server.""" @@ -634,7 +687,8 @@ async def test_local_control_with_agent_scoped_evaluator_raises(self, agent_name async def test_server_control_with_missing_evaluator_allowed(self, agent_name, llm_payload): """Test that server control with unavailable evaluator is allowed (server handles it). - Given: A server control (execution="server") referencing an evaluator that doesn't exist locally + Given: A server control (execution="server") referencing an evaluator that + doesn't exist locally When: check_evaluation_with_local is called Then: No error, server is called to handle it """ @@ -755,8 +809,6 @@ async def test_local_evaluation_includes_steering_context(self, agent_name, llm_ Then: Response includes steering_context field in matches Coverage: Lines 275, 280, 298-301 in control_decorators.py """ - from agent_control_models.controls import SteeringContext - controls = [ { "id": 1, From d5562b98d2ada924be6464fbb3e1c823f5cf865c Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Thu, 12 Mar 2026 17:12:53 -0700 Subject: [PATCH 2/3] fix: skip local evaluation when no controls apply --- sdks/python/src/agent_control/evaluation.py | 31 +++-- sdks/python/tests/test_local_evaluation.py | 120 ++++++++++++++++++-- 2 files changed, 134 insertions(+), 17 deletions(-) diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index 979292ce..330cd5d4 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -134,6 +134,20 @@ class _ControlAdapter: control: "ControlDefinition" +def _get_applicable_controls( + controls: list[_ControlAdapter], + request: EvaluationRequest, + *, + context: Literal["sdk", "server"], +) -> list[_ControlAdapter]: + """Return parsed controls that apply to this request in the given context.""" + applicable_controls = ControlEngine( + controls, + context=context, + ).get_applicable_controls(request) + return cast(list[_ControlAdapter], applicable_controls) + + def _has_applicable_server_controls( controls: list[dict[str, Any]], request: EvaluationRequest, @@ -162,11 +176,7 @@ def _has_applicable_server_controls( if not server_controls: return False - applicable_controls = ControlEngine( - server_controls, - context="server", - ).get_applicable_controls(request) - return bool(applicable_controls) + return bool(_get_applicable_controls(server_controls, request, context="server")) def _merge_results( @@ -327,14 +337,19 @@ def _with_parse_errors(result: EvaluationResult) -> EvaluationResult: ) local_result: EvaluationResponse | None = None - if local_controls: - engine = ControlEngine(local_controls, context="sdk") + applicable_local_controls = _get_applicable_controls( + local_controls, + request, + context="sdk", + ) + if applicable_local_controls: + engine = ControlEngine(applicable_local_controls, context="sdk") local_result = await engine.process(request) _emit_local_events( local_result, request, - local_controls, + applicable_local_controls, trace_id, span_id, agent_name=event_agent_name, diff --git a/sdks/python/tests/test_local_evaluation.py b/sdks/python/tests/test_local_evaluation.py index 72e21cd5..ce737b0c 100644 --- a/sdks/python/tests/test_local_evaluation.py +++ b/sdks/python/tests/test_local_evaluation.py @@ -9,7 +9,7 @@ """ from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from agent_control.client import AgentControlClient @@ -91,6 +91,15 @@ def make_control_dict( } +NON_APPLICABLE_CONTROL_CASES = [ + pytest.param({"enabled": False}, id="disabled"), + pytest.param({"stage": "post"}, id="stage_mismatch"), + pytest.param({"step_type": "tool"}, id="step_type_mismatch"), + pytest.param({"step_names": ["send_email"]}, id="step_name_mismatch"), + pytest.param({"step_name_regex": r"^send_.*$"}, id="step_name_regex_mismatch"), +] + + # ============================================================================= # Test: _merge_results # ============================================================================= @@ -263,13 +272,7 @@ async def test_server_only_controls_calls_server(self, agent_name, llm_payload): @pytest.mark.asyncio @pytest.mark.parametrize( "control_kwargs", - [ - pytest.param({"enabled": False}, id="disabled"), - pytest.param({"stage": "post"}, id="stage_mismatch"), - pytest.param({"step_type": "tool"}, id="step_type_mismatch"), - pytest.param({"step_names": ["send_email"]}, id="step_name_mismatch"), - pytest.param({"step_name_regex": r"^send_.*$"}, id="step_name_regex_mismatch"), - ], + NON_APPLICABLE_CONTROL_CASES, ) async def test_non_applicable_server_controls_do_not_call_server( self, @@ -277,7 +280,11 @@ async def test_non_applicable_server_controls_do_not_call_server( llm_payload, control_kwargs, ): - """Server evaluation should be skipped when no server control applies.""" + """Given: Only server controls that do not apply to this invocation. + + When: check_evaluation_with_local is called. + Then: The SDK skips the server evaluation request entirely. + """ controls = [ make_control_dict( 1, @@ -307,6 +314,101 @@ async def test_non_applicable_server_controls_do_not_call_server( assert result.confidence == 1.0 assert result.matches is None + @pytest.mark.asyncio + @pytest.mark.parametrize( + "control_kwargs", + NON_APPLICABLE_CONTROL_CASES, + ) + async def test_non_applicable_local_controls_skip_local_evaluation( + self, + agent_name, + llm_payload, + control_kwargs, + ): + """Given: Only local controls that do not apply to this invocation. + + When: check_evaluation_with_local is called. + Then: The SDK skips local evaluation and returns a no-op safe result. + """ + controls = [ + make_control_dict( + 1, + "local_ctrl", + execution="sdk", + **control_kwargs, + ), + ] + + client = MagicMock(spec=AgentControlClient) + client.http_client = AsyncMock() + client.http_client.post = AsyncMock() + + with patch( + "agent_control.evaluation.ControlEngine.process", + side_effect=AssertionError("local evaluation should not run"), + ) as mock_process: + result = await check_evaluation_with_local( + client=client, + agent_name=agent_name, + step=llm_payload, + stage="pre", + controls=controls, + ) + + mock_process.assert_not_called() + client.http_client.post.assert_not_called() + assert result.is_safe is True + assert result.confidence == 1.0 + assert result.matches is None + + @pytest.mark.asyncio + async def test_non_applicable_local_controls_skip_local_but_still_call_server( + self, + agent_name, + llm_payload, + ): + """Given: A non-applicable local control and an applicable server control. + + When: check_evaluation_with_local is called. + Then: Local evaluation is skipped, but the applicable server control still runs. + """ + controls = [ + make_control_dict( + 1, + "local_ctrl", + execution="sdk", + step_names=["send_email"], + ), + make_control_dict( + 2, + "server_ctrl", + execution="server", + ), + ] + + client = MagicMock(spec=AgentControlClient) + mock_response = MagicMock() + mock_response.json.return_value = {"is_safe": True, "confidence": 1.0} + mock_response.raise_for_status = MagicMock() + client.http_client = AsyncMock() + client.http_client.post = AsyncMock(return_value=mock_response) + + with patch( + "agent_control.evaluation.ControlEngine.process", + side_effect=AssertionError("local evaluation should not run"), + ) as mock_process: + result = await check_evaluation_with_local( + client=client, + agent_name=agent_name, + step=llm_payload, + stage="pre", + controls=controls, + ) + + mock_process.assert_not_called() + client.http_client.post.assert_called_once() + assert result.is_safe is True + @pytest.mark.asyncio async def test_local_deny_short_circuits(self, agent_name, llm_payload): """Local deny should return immediately without calling server.""" From 91f459663de991d91d99fe185af2752b28e84729 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Fri, 13 Mar 2026 17:37:48 -0700 Subject: [PATCH 3/3] refactor: clarify server control prefilter contract --- sdks/python/src/agent_control/evaluation.py | 36 +++++++++++++-------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index 330cd5d4..9661a6e3 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -148,21 +148,25 @@ def _get_applicable_controls( return cast(list[_ControlAdapter], applicable_controls) -def _has_applicable_server_controls( - controls: list[dict[str, Any]], +def _has_applicable_prefiltered_server_controls( + server_control_payloads: list[dict[str, Any]], request: EvaluationRequest, ) -> bool: - """Return whether any well-formed server control applies to this request. + """Return whether any partitioned server control applies to this request. + + The caller is responsible for partitioning raw control payloads by + ``execution`` before calling this helper. This function only inspects the + server-control subset and does not re-check ``execution`` itself. - If any server control cannot be parsed locally, this returns True so the SDK - still defers to the server for authoritative handling. + If any server control payload cannot be parsed locally, this returns True so + the SDK still defers to the server for authoritative handling. """ - server_controls: list[_ControlAdapter] = [] + parsed_server_controls: list[_ControlAdapter] = [] - for control in controls: + for control in server_control_payloads: try: control_def = ControlDefinition.model_validate(control["control"]) - server_controls.append( + parsed_server_controls.append( _ControlAdapter( id=control["id"], name=control["name"], @@ -173,10 +177,16 @@ def _has_applicable_server_controls( # Preserve existing fail-open behavior for malformed server controls. return True - if not server_controls: + if not parsed_server_controls: return False - return bool(_get_applicable_controls(server_controls, request, context="server")) + return bool( + _get_applicable_controls( + parsed_server_controls, + request, + context="server", + ) + ) def _merge_results( @@ -257,7 +267,7 @@ async def check_evaluation_with_local( # Partition controls by local flag local_controls: list[_ControlAdapter] = [] parse_errors: list[ControlMatch] = [] - server_controls: list[dict[str, Any]] = [] + server_control_payloads: list[dict[str, Any]] = [] for control in controls: control_data = control.get("control", {}) @@ -265,7 +275,7 @@ async def check_evaluation_with_local( is_local = execution == "sdk" if not is_local: - server_controls.append(control) + server_control_payloads.append(control) continue try: @@ -367,7 +377,7 @@ def _with_parse_errors(result: EvaluationResult) -> EvaluationResult: ) ) - if _has_applicable_server_controls(server_controls, request): + if _has_applicable_prefiltered_server_controls(server_control_payloads, request): request_payload = request.model_dump(mode="json", exclude_none=True) headers: dict[str, str] = {} if trace_id: