From 602284dc3e3465c2d4b3fe5d668191dcd0dde704 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 12:14:48 -0400 Subject: [PATCH 01/10] provider specific kwargs --- libs/langchain_v1/langchain/agents/factory.py | 4 +- .../langchain/agents/structured_output.py | 30 ++++++++++-- .../unit_tests/agents/test_response_format.py | 46 +++++++++++++++++++ 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index 3af577d0a42f0..b82d214f4a7cd 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -956,10 +956,10 @@ def _get_bound_model(request: ModelRequest) -> tuple[Runnable, ResponseFormat | # Bind model based on effective response format if isinstance(effective_response_format, ProviderStrategy): # Use provider-specific structured output - kwargs = effective_response_format.to_model_kwargs() + kwargs = effective_response_format.to_model_kwargs(model=request.model) return ( request.model.bind_tools( - final_tools, strict=True, **kwargs, **request.model_settings + final_tools, **kwargs, **request.model_settings ), effective_response_format, ) diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index cd6a2fd9aed31..a8a4a45c422b4 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -254,10 +254,19 @@ def __init__( self.schema = schema self.schema_spec = _SchemaSpec(schema) - def to_model_kwargs(self) -> dict[str, Any]: - """Convert to kwargs to bind to a model to force structured output.""" - # OpenAI: - # - see https://platform.openai.com/docs/guides/structured-outputs + def to_model_kwargs(self, model: Any | None = None) -> dict[str, Any]: + """Convert to kwargs to bind to a model to force structured output. + + Args: + model: The model instance to check provider for conditional `strict` param. + + Returns: + Model kwargs with `response_format` and optionally `strict`. + """ + # Provider-specific structured output: + # - OpenAI: https://platform.openai.com/docs/guides/structured-outputs + # - Uses strict=True for schema validation + # - X.AI (Grok): Similar to OpenAI format but doesn't support strict parameter response_format = { "type": "json_schema", "json_schema": { @@ -265,7 +274,18 @@ def to_model_kwargs(self) -> dict[str, Any]: "schema": self.schema_spec.json_schema, }, } - return {"response_format": response_format} + + # Only set strict=True for OpenAI models + # Other providers (like X.AI/Grok) don't support/require the strict parameter + kwargs: dict[str, Any] = {"response_format": response_format} + + if model is not None and hasattr(model, "_get_ls_params"): + ls_params = model._get_ls_params() + provider = ls_params.get("ls_provider", "").lower() + if provider == "openai": + kwargs["strict"] = True + + return kwargs @dataclass diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index a7963ced16f57..b9c9fa710286f 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -800,3 +800,49 @@ def test_union_of_types() -> None: assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC assert len(response["messages"]) == 5 + + +def test_provider_strategy_strict_only_for_openai() -> None: + """Test that strict=True is only set for OpenAI models in ProviderStrategy.""" + from langchain.agents.structured_output import ProviderStrategy + from langchain_core.language_models.base import LangSmithParams + + # Create a mock OpenAI model + class MockOpenAIModel: + def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: + return LangSmithParams(ls_provider="openai", ls_model_type="chat") + + # Create a mock non-OpenAI model (e.g., Grok/X.AI) + class MockGrokModel: + def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: + return LangSmithParams(ls_provider="xai", ls_model_type="chat") + + # Create a mock model without _get_ls_params + class MockModelNoLSParams: + pass + + provider_strategy = ProviderStrategy(WeatherBaseModel) + + # Test OpenAI model: should include strict=True + openai_model = MockOpenAIModel() + openai_kwargs = provider_strategy.to_model_kwargs(model=openai_model) + assert "strict" in openai_kwargs + assert openai_kwargs["strict"] is True + assert "response_format" in openai_kwargs + + # Test Grok model: should NOT include strict + grok_model = MockGrokModel() + grok_kwargs = provider_strategy.to_model_kwargs(model=grok_model) + assert "strict" not in grok_kwargs + assert "response_format" in grok_kwargs + + # Test model without _get_ls_params: should NOT include strict + no_params_model = MockModelNoLSParams() + no_params_kwargs = provider_strategy.to_model_kwargs(model=no_params_model) + assert "strict" not in no_params_kwargs + assert "response_format" in no_params_kwargs + + # Test None model: should NOT include strict + none_kwargs = provider_strategy.to_model_kwargs(model=None) + assert "strict" not in none_kwargs + assert "response_format" in none_kwargs From c97e649bd3147d072282b3a7de4b7a41549356c6 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 12:44:44 -0400 Subject: [PATCH 02/10] linting, ofc --- libs/langchain_v1/langchain/agents/factory.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index b82d214f4a7cd..38fa07f57dd74 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -958,9 +958,7 @@ def _get_bound_model(request: ModelRequest) -> tuple[Runnable, ResponseFormat | # Use provider-specific structured output kwargs = effective_response_format.to_model_kwargs(model=request.model) return ( - request.model.bind_tools( - final_tools, **kwargs, **request.model_settings - ), + request.model.bind_tools(final_tools, **kwargs, **request.model_settings), effective_response_format, ) From 7157c2f69c417160f8765eafabadc4198453ba1e Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 13:12:14 -0400 Subject: [PATCH 03/10] openai and xai --- libs/langchain_v1/langchain/agents/factory.py | 13 ++++ .../langchain/agents/structured_output.py | 60 +++++++++++++++++-- .../tests/unit_tests/agents/model.py | 6 ++ .../unit_tests/agents/test_response_format.py | 53 ++++++++++------ 4 files changed, 110 insertions(+), 22 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index 38fa07f57dd74..a5348c2ff6749 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -944,6 +944,19 @@ def _get_bound_model(request: ModelRequest) -> tuple[Runnable, ResponseFormat | # User explicitly specified a strategy - preserve it effective_response_format = request.response_format + # Validate ProviderStrategy is used with supported model + if isinstance( + effective_response_format, ProviderStrategy + ) and not _supports_provider_strategy(request.model): + msg = ( + "ProviderStrategy does not support this model. " + "Supported providers: OpenAI (gpt-5, gpt-4.1, gpt-oss, o3-pro, o3-mini), " + "X.AI (Grok). " + "Consider using a raw schema (which auto-selects the best strategy) or " + "explicitly use ToolStrategy for unsupported providers." + ) + raise ValueError(msg) + # Build final tools list including structured output tools # request.tools now only contains BaseTool instances (converted from callables) # and dicts (built-ins) diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index a8a4a45c422b4..2b54ddeab34b8 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -238,7 +238,56 @@ def _iter_variants(schema: Any) -> Iterable[Any]: @dataclass(init=False) class ProviderStrategy(Generic[SchemaT]): - """Use the model provider's native structured output method.""" + """Use the model provider's native structured output method. + + `ProviderStrategy` uses provider-specific structured output APIs that enforce + JSON schema validation at the model level. This provides stronger guarantees + than tool-based approaches but is only supported by certain providers. + + Supported Providers: + - **OpenAI**: All models that support structured outputs (requires `strict=True`) + - **X.AI (Grok)**: All models that support structured outputs (requires `strict=True`) + + Important: + When using `ProviderStrategy`, the agent will validate at runtime that the + model provider is supported. If you're using an unsupported provider, consider: + + - Using a **raw schema** (recommended): Automatically selects the best strategy + based on model capabilities + - Using **`ToolStrategy`**: Explicitly use tool-based structured output for any + provider + + Example: + ```python + from langchain.agents import create_agent + from langchain.agents.structured_output import ProviderStrategy + from pydantic import BaseModel + + + class WeatherResponse(BaseModel): + temperature: float + condition: str + + + # Explicitly use provider strategy (only for OpenAI/Grok) + agent = create_agent( + model="openai:gpt-4", tools=[], response_format=ProviderStrategy(WeatherResponse) + ) + + # Or use raw schema for automatic strategy selection (recommended) + # This will auto-select ProviderStrategy for OpenAI/Grok, ToolStrategy for others + agent = create_agent( + model="openai:gpt-4", + tools=[], + response_format=WeatherResponse, # Auto-selects best strategy + ) + ``` + + Note: + `ProviderStrategy` can be used with middleware that changes the model at runtime. + Validation occurs after the model is resolved, allowing dynamic model selection + while ensuring provider compatibility. + """ schema: type[SchemaT] """Schema for native mode.""" @@ -266,7 +315,8 @@ def to_model_kwargs(self, model: Any | None = None) -> dict[str, Any]: # Provider-specific structured output: # - OpenAI: https://platform.openai.com/docs/guides/structured-outputs # - Uses strict=True for schema validation - # - X.AI (Grok): Similar to OpenAI format but doesn't support strict parameter + # - X.AI (Grok): https://docs.x.ai/docs/guides/structured-outputs + # - Uses strict=True for schema validation (required) response_format = { "type": "json_schema", "json_schema": { @@ -275,14 +325,14 @@ def to_model_kwargs(self, model: Any | None = None) -> dict[str, Any]: }, } - # Only set strict=True for OpenAI models - # Other providers (like X.AI/Grok) don't support/require the strict parameter + # Set strict=True for OpenAI and X.AI (Grok) models + # Both providers require strict=True for structured output kwargs: dict[str, Any] = {"response_format": response_format} if model is not None and hasattr(model, "_get_ls_params"): ls_params = model._get_ls_params() provider = ls_params.get("ls_provider", "").lower() - if provider == "openai": + if provider in ("openai", "xai"): kwargs["strict"] = True return kwargs diff --git a/libs/langchain_v1/tests/unit_tests/agents/model.py b/libs/langchain_v1/tests/unit_tests/agents/model.py index 07ed23995eb26..ac2a7a47f125b 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/model.py +++ b/libs/langchain_v1/tests/unit_tests/agents/model.py @@ -11,6 +11,7 @@ from langchain_core.callbacks import CallbackManagerForLLMRun from langchain_core.language_models import BaseChatModel, LanguageModelInput +from langchain_core.language_models.base import LangSmithParams from langchain_core.messages import ( AIMessage, BaseMessage, @@ -29,6 +30,7 @@ class FakeToolCallingModel(BaseChatModel, Generic[StructuredResponseT]): structured_response: StructuredResponseT | None = None index: int = 0 tool_style: Literal["openai", "anthropic"] = "openai" + ls_provider: str = "openai" def _generate( self, @@ -73,6 +75,10 @@ def _generate( def _llm_type(self) -> str: return "fake-tool-call-model" + def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: + """Get LangSmith parameters for this model.""" + return LangSmithParams(ls_provider=self.ls_provider, ls_model_type="chat") + def bind_tools( self, tools: Sequence[Union[dict[str, Any], type[BaseModel], Callable, BaseTool]], diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index b9c9fa710286f..8f26749f0199e 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -715,6 +715,12 @@ def bind_tools( self.tool_bindings.append(tools) return self + def _get_ls_params(self, **kwargs: Any): + """Return OpenAI as provider to pass ProviderStrategy validation.""" + from langchain_core.language_models.base import LangSmithParams + + return LangSmithParams(ls_provider="openai", ls_model_type="chat") + model = CustomModel( messages=iter( [ @@ -803,7 +809,7 @@ def test_union_of_types() -> None: def test_provider_strategy_strict_only_for_openai() -> None: - """Test that strict=True is only set for OpenAI models in ProviderStrategy.""" + """Test that strict=True is set for OpenAI and Grok models in ProviderStrategy.""" from langchain.agents.structured_output import ProviderStrategy from langchain_core.language_models.base import LangSmithParams @@ -812,15 +818,11 @@ class MockOpenAIModel: def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: return LangSmithParams(ls_provider="openai", ls_model_type="chat") - # Create a mock non-OpenAI model (e.g., Grok/X.AI) + # Create a mock Grok/X.AI model class MockGrokModel: def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: return LangSmithParams(ls_provider="xai", ls_model_type="chat") - # Create a mock model without _get_ls_params - class MockModelNoLSParams: - pass - provider_strategy = ProviderStrategy(WeatherBaseModel) # Test OpenAI model: should include strict=True @@ -830,19 +832,36 @@ class MockModelNoLSParams: assert openai_kwargs["strict"] is True assert "response_format" in openai_kwargs - # Test Grok model: should NOT include strict + # Test Grok model: should include strict=True (Grok requires strict) grok_model = MockGrokModel() grok_kwargs = provider_strategy.to_model_kwargs(model=grok_model) - assert "strict" not in grok_kwargs + assert "strict" in grok_kwargs + assert grok_kwargs["strict"] is True assert "response_format" in grok_kwargs - # Test model without _get_ls_params: should NOT include strict - no_params_model = MockModelNoLSParams() - no_params_kwargs = provider_strategy.to_model_kwargs(model=no_params_model) - assert "strict" not in no_params_kwargs - assert "response_format" in no_params_kwargs - # Test None model: should NOT include strict - none_kwargs = provider_strategy.to_model_kwargs(model=None) - assert "strict" not in none_kwargs - assert "response_format" in none_kwargs +def test_provider_strategy_validation() -> None: + """Test that ProviderStrategy validates provider support at agent invocation time.""" + from langchain.agents.structured_output import ProviderStrategy + from langchain_core.language_models.base import LangSmithParams + + # Create a mock model from an unsupported provider (e.g., Anthropic) + class MockAnthropicModel(FakeToolCallingModel): + def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: + return LangSmithParams(ls_provider="anthropic", ls_model_type="chat") + + # Create a mock model without _get_ls_params + class MockModelNoLSParams(FakeToolCallingModel): + def _get_ls_params(self, **kwargs: Any): + msg = "This model doesn't support _get_ls_params" + raise AttributeError(msg) + + # Test unsupported provider: should raise ValueError when invoking agent + anthropic_model = MockAnthropicModel(tool_calls=[[]]) + agent = create_agent(anthropic_model, [], response_format=ProviderStrategy(WeatherBaseModel)) + with pytest.raises(ValueError, match="does not support provider 'anthropic'"): + agent.invoke({"messages": [HumanMessage("What's the weather?")]}) + + # Test model without proper _get_ls_params: still works if model has the method + # (validation checks hasattr and calls it) + # We can't easily test the "no _get_ls_params" case without breaking BaseChatModel From 5155fbe072fc98dbaebee311100d116c9d87d3ee Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 13:20:47 -0400 Subject: [PATCH 04/10] better structure --- libs/langchain_v1/langchain/agents/factory.py | 24 +------ .../langchain/agents/structured_output.py | 34 ++++++++-- .../tests/unit_tests/agents/model.py | 2 + .../unit_tests/agents/test_response_format.py | 65 +++++-------------- 4 files changed, 47 insertions(+), 78 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index a5348c2ff6749..f6e80426f7ab6 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -42,6 +42,7 @@ ResponseFormat, StructuredOutputValidationError, ToolStrategy, + _supports_provider_strategy, ) from langchain.chat_models import init_chat_model from langchain.tools.tool_node import ToolCallWithContext, _ToolNode @@ -347,29 +348,6 @@ def _get_can_jump_to(middleware: AgentMiddleware[Any, Any], hook_name: str) -> l return [] -def _supports_provider_strategy(model: str | BaseChatModel) -> bool: - """Check if a model supports provider-specific structured output. - - Args: - model: Model name string or `BaseChatModel` instance. - - Returns: - `True` if the model supports provider-specific structured output, `False` otherwise. - """ - model_name: str | None = None - if isinstance(model, str): - model_name = model - elif isinstance(model, BaseChatModel): - model_name = getattr(model, "model_name", None) - - return ( - "grok" in model_name.lower() - or any(part in model_name for part in ["gpt-5", "gpt-4.1", "gpt-oss", "o3-pro", "o3-mini"]) - if model_name - else False - ) - - def _handle_structured_output_error( exception: Exception, response_format: ResponseFormat, diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index 2b54ddeab34b8..4cad8fa4f2d68 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable + from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import AIMessage # Supported schema types: Pydantic models, dataclasses, TypedDict, JSON schema dicts @@ -31,6 +32,30 @@ SchemaKind = Literal["pydantic", "dataclass", "typeddict", "json_schema"] +def _supports_provider_strategy(model: str | Any) -> bool: + """Check if a model supports provider-specific structured output. + + Args: + model: Model name string or `BaseChatModel` instance. + + Returns: + `True` if the model supports provider-specific structured output, `False` otherwise. + """ + model_name: str | None = None + if isinstance(model, str): + model_name = model + else: + # Try to get model_name attribute from model instance + model_name = getattr(model, "model_name", None) + + return ( + "grok" in model_name.lower() + or any(part in model_name for part in ["gpt-5", "gpt-4.1", "gpt-oss", "o3-pro", "o3-mini"]) + if model_name + else False + ) + + class StructuredOutputError(Exception): """Base class for structured output errors.""" @@ -329,11 +354,10 @@ def to_model_kwargs(self, model: Any | None = None) -> dict[str, Any]: # Both providers require strict=True for structured output kwargs: dict[str, Any] = {"response_format": response_format} - if model is not None and hasattr(model, "_get_ls_params"): - ls_params = model._get_ls_params() - provider = ls_params.get("ls_provider", "").lower() - if provider in ("openai", "xai"): - kwargs["strict"] = True + # Use _supports_provider_strategy to determine if we should set strict=True + # This checks model name patterns for OpenAI and Grok models + if model is not None and _supports_provider_strategy(model): + kwargs["strict"] = True return kwargs diff --git a/libs/langchain_v1/tests/unit_tests/agents/model.py b/libs/langchain_v1/tests/unit_tests/agents/model.py index ac2a7a47f125b..17f493fbbe915 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/model.py +++ b/libs/langchain_v1/tests/unit_tests/agents/model.py @@ -31,6 +31,7 @@ class FakeToolCallingModel(BaseChatModel, Generic[StructuredResponseT]): index: int = 0 tool_style: Literal["openai", "anthropic"] = "openai" ls_provider: str = "openai" + model_name: str = "fake-model" def _generate( self, @@ -54,6 +55,7 @@ def _generate( tool_calls = [] if is_native and not tool_calls: + content_obj = {} if isinstance(self.structured_response, BaseModel): content_obj = self.structured_response.model_dump() elif is_dataclass(self.structured_response): diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index 8f26749f0199e..a48427d69fff5 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -619,7 +619,7 @@ def test_pydantic_model(self) -> None: ] model = FakeToolCallingModel[WeatherBaseModel]( - tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_PYDANTIC + tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_PYDANTIC, model_name="gpt-4.1" ) agent = create_agent( @@ -637,7 +637,7 @@ def test_dataclass(self) -> None: ] model = FakeToolCallingModel[WeatherDataclass]( - tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DATACLASS + tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DATACLASS, model_name="gpt-4.1" ) agent = create_agent( @@ -657,7 +657,7 @@ def test_typed_dict(self) -> None: ] model = FakeToolCallingModel[WeatherTypedDict]( - tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DICT + tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DICT, model_name="gpt-4.1" ) agent = create_agent( @@ -675,7 +675,7 @@ def test_json_schema(self) -> None: ] model = FakeToolCallingModel[dict]( - tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DICT + tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DICT, model_name="gpt-4.1" ) agent = create_agent( @@ -697,13 +697,13 @@ def test_middleware_model_swap_provider_to_tool_strategy(self) -> None: on the middleware-modified model (not the original), ensuring the correct strategy is selected based on the final model's capabilities. """ - from unittest.mock import patch from langchain.agents.middleware.types import AgentMiddleware, ModelRequest from langchain_core.language_models.fake_chat_models import GenericFakeChatModel - # Custom model that we'll use to test whether the tool strategy is applied - # correctly at runtime. + # Custom model that we'll use to test whether the provider strategy is applied + # correctly at runtime. Use a model_name that supports provider strategy. class CustomModel(GenericFakeChatModel): + model_name: str = "gpt-4.1" tool_bindings: list[Any] = [] def bind_tools( @@ -715,12 +715,6 @@ def bind_tools( self.tool_bindings.append(tools) return self - def _get_ls_params(self, **kwargs: Any): - """Return OpenAI as provider to pass ProviderStrategy validation.""" - from langchain_core.language_models.base import LangSmithParams - - return LangSmithParams(ls_provider="openai", ls_model_type="chat") - model = CustomModel( messages=iter( [ @@ -742,14 +736,6 @@ def wrap_model_call( request.model = model return handler(request) - # Track which model is checked for provider strategy support - calls = [] - - def mock_supports_provider_strategy(model) -> bool: - """Track which model is checked and return True for ProviderStrategy.""" - calls.append(model) - return True - # Use raw Pydantic model (not wrapped in ToolStrategy or ProviderStrategy) # This should auto-detect strategy based on model capabilities agent = create_agent( @@ -760,14 +746,7 @@ def mock_supports_provider_strategy(model) -> bool: middleware=[ModelSwappingMiddleware()], ) - with patch( - "langchain.agents.factory._supports_provider_strategy", - side_effect=mock_supports_provider_strategy, - ): - response = agent.invoke({"messages": [HumanMessage("What's the weather?")]}) - - # Verify strategy resolution was deferred: check was called once during _get_bound_model - assert len(calls) == 1 + response = agent.invoke({"messages": [HumanMessage("What's the weather?")]}) # Verify successful parsing of JSON as structured output via ProviderStrategy assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC @@ -811,17 +790,14 @@ def test_union_of_types() -> None: def test_provider_strategy_strict_only_for_openai() -> None: """Test that strict=True is set for OpenAI and Grok models in ProviderStrategy.""" from langchain.agents.structured_output import ProviderStrategy - from langchain_core.language_models.base import LangSmithParams - # Create a mock OpenAI model + # Create a mock OpenAI model with model_name class MockOpenAIModel: - def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: - return LangSmithParams(ls_provider="openai", ls_model_type="chat") + model_name: str = "gpt-4.1" - # Create a mock Grok/X.AI model + # Create a mock Grok/X.AI model with model_name class MockGrokModel: - def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: - return LangSmithParams(ls_provider="xai", ls_model_type="chat") + model_name: str = "grok-beta" provider_strategy = ProviderStrategy(WeatherBaseModel) @@ -843,25 +819,14 @@ def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: def test_provider_strategy_validation() -> None: """Test that ProviderStrategy validates provider support at agent invocation time.""" from langchain.agents.structured_output import ProviderStrategy - from langchain_core.language_models.base import LangSmithParams # Create a mock model from an unsupported provider (e.g., Anthropic) + # Use a model_name that doesn't match any supported patterns class MockAnthropicModel(FakeToolCallingModel): - def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: - return LangSmithParams(ls_provider="anthropic", ls_model_type="chat") - - # Create a mock model without _get_ls_params - class MockModelNoLSParams(FakeToolCallingModel): - def _get_ls_params(self, **kwargs: Any): - msg = "This model doesn't support _get_ls_params" - raise AttributeError(msg) + model_name: str = "claude-3-5-sonnet-20241022" # Test unsupported provider: should raise ValueError when invoking agent anthropic_model = MockAnthropicModel(tool_calls=[[]]) agent = create_agent(anthropic_model, [], response_format=ProviderStrategy(WeatherBaseModel)) - with pytest.raises(ValueError, match="does not support provider 'anthropic'"): + with pytest.raises(ValueError, match="ProviderStrategy does not support this model"): agent.invoke({"messages": [HumanMessage("What's the weather?")]}) - - # Test model without proper _get_ls_params: still works if model has the method - # (validation checks hasattr and calls it) - # We can't easily test the "no _get_ls_params" case without breaking BaseChatModel From 6efea75d1cef52f5eb44f609c0a3c4ee85e96bf3 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 13:24:02 -0400 Subject: [PATCH 05/10] linting --- libs/langchain_v1/langchain/agents/factory.py | 2 +- libs/langchain_v1/langchain/agents/structured_output.py | 8 +------- .../tests/unit_tests/agents/test_response_format.py | 8 ++++++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index f6e80426f7ab6..a44bf809f4d97 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -13,7 +13,6 @@ get_type_hints, ) -from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import AIMessage, AnyMessage, SystemMessage, ToolMessage from langchain_core.tools import BaseTool from langgraph._internal._runnable import RunnableCallable @@ -50,6 +49,7 @@ if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Sequence + from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.runnables import Runnable from langgraph.cache.base import BaseCache from langgraph.graph.state import CompiledStateGraph diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index 4cad8fa4f2d68..9eec0bc0b52c7 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -23,7 +23,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable - from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import AIMessage # Supported schema types: Pydantic models, dataclasses, TypedDict, JSON schema dicts @@ -41,12 +40,7 @@ def _supports_provider_strategy(model: str | Any) -> bool: Returns: `True` if the model supports provider-specific structured output, `False` otherwise. """ - model_name: str | None = None - if isinstance(model, str): - model_name = model - else: - # Try to get model_name attribute from model instance - model_name = getattr(model, "model_name", None) + model_name: str | None = model if isinstance(model, str) else getattr(model, "model_name", None) return ( "grok" in model_name.lower() diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index a48427d69fff5..d44fecb7b5e78 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -619,7 +619,9 @@ def test_pydantic_model(self) -> None: ] model = FakeToolCallingModel[WeatherBaseModel]( - tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_PYDANTIC, model_name="gpt-4.1" + tool_calls=tool_calls, + structured_response=EXPECTED_WEATHER_PYDANTIC, + model_name="gpt-4.1", ) agent = create_agent( @@ -637,7 +639,9 @@ def test_dataclass(self) -> None: ] model = FakeToolCallingModel[WeatherDataclass]( - tool_calls=tool_calls, structured_response=EXPECTED_WEATHER_DATACLASS, model_name="gpt-4.1" + tool_calls=tool_calls, + structured_response=EXPECTED_WEATHER_DATACLASS, + model_name="gpt-4.1", ) agent = create_agent( From e4ab6896a719623294c09864a23f187b322558a0 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 13:33:46 -0400 Subject: [PATCH 06/10] another pass --- libs/langchain_v1/langchain/agents/factory.py | 44 +++++++++--------- .../langchain/agents/structured_output.py | 15 ++----- .../unit_tests/agents/test_response_format.py | 45 ------------------- 3 files changed, 28 insertions(+), 76 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index a44bf809f4d97..987b3be89b8fe 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -910,31 +910,36 @@ def _get_bound_model(request: ModelRequest) -> tuple[Runnable, ResponseFormat | # Determine effective response format (auto-detect if needed) effective_response_format: ResponseFormat | None + model_name: str = cast( + "str", + ( + request.model + if isinstance(request.model, str) + else getattr(request.model, "model_name", "") + ), + ) if isinstance(request.response_format, AutoStrategy): # User provided raw schema via AutoStrategy - auto-detect best strategy based on model - if _supports_provider_strategy(request.model): + if _supports_provider_strategy(model_name): # Model supports provider strategy - use it effective_response_format = ProviderStrategy(schema=request.response_format.schema) else: # Model doesn't support provider strategy - use ToolStrategy effective_response_format = ToolStrategy(schema=request.response_format.schema) + elif isinstance(request.response_format, ProviderStrategy): + if not _supports_provider_strategy(model_name): + msg = ( + f"Cannot use ProviderStrategy with {model_name}. " + "Supported models: OpenAI (gpt-5, gpt-4.1, gpt-oss, o3-pro, o3-mini), " + "X.AI (Grok). " + "Consider using a raw schema (which auto-selects the best strategy) or " + "explicitly use `ToolStrategy` for unsupported providers." + ) + raise ValueError(msg) + effective_response_format = request.response_format else: - # User explicitly specified a strategy - preserve it effective_response_format = request.response_format - # Validate ProviderStrategy is used with supported model - if isinstance( - effective_response_format, ProviderStrategy - ) and not _supports_provider_strategy(request.model): - msg = ( - "ProviderStrategy does not support this model. " - "Supported providers: OpenAI (gpt-5, gpt-4.1, gpt-oss, o3-pro, o3-mini), " - "X.AI (Grok). " - "Consider using a raw schema (which auto-selects the best strategy) or " - "explicitly use ToolStrategy for unsupported providers." - ) - raise ValueError(msg) - # Build final tools list including structured output tools # request.tools now only contains BaseTool instances (converted from callables) # and dicts (built-ins) @@ -947,11 +952,10 @@ def _get_bound_model(request: ModelRequest) -> tuple[Runnable, ResponseFormat | # Bind model based on effective response format if isinstance(effective_response_format, ProviderStrategy): # Use provider-specific structured output - kwargs = effective_response_format.to_model_kwargs(model=request.model) - return ( - request.model.bind_tools(final_tools, **kwargs, **request.model_settings), - effective_response_format, - ) + kwargs = effective_response_format.to_model_kwargs() + return request.model.bind_tools( + final_tools, **kwargs, **request.model_settings + ), effective_response_format if isinstance(effective_response_format, ToolStrategy): # Current implementation requires that tools used for structured output diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index 9eec0bc0b52c7..4d05c961e94a3 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -31,17 +31,15 @@ SchemaKind = Literal["pydantic", "dataclass", "typeddict", "json_schema"] -def _supports_provider_strategy(model: str | Any) -> bool: +def _supports_provider_strategy(model_name: str) -> bool: """Check if a model supports provider-specific structured output. Args: - model: Model name string or `BaseChatModel` instance. + model_name: Model name string. Returns: `True` if the model supports provider-specific structured output, `False` otherwise. """ - model_name: str | None = model if isinstance(model, str) else getattr(model, "model_name", None) - return ( "grok" in model_name.lower() or any(part in model_name for part in ["gpt-5", "gpt-4.1", "gpt-oss", "o3-pro", "o3-mini"]) @@ -322,7 +320,7 @@ def __init__( self.schema = schema self.schema_spec = _SchemaSpec(schema) - def to_model_kwargs(self, model: Any | None = None) -> dict[str, Any]: + def to_model_kwargs(self) -> dict[str, Any]: """Convert to kwargs to bind to a model to force structured output. Args: @@ -346,12 +344,7 @@ def to_model_kwargs(self, model: Any | None = None) -> dict[str, Any]: # Set strict=True for OpenAI and X.AI (Grok) models # Both providers require strict=True for structured output - kwargs: dict[str, Any] = {"response_format": response_format} - - # Use _supports_provider_strategy to determine if we should set strict=True - # This checks model name patterns for OpenAI and Grok models - if model is not None and _supports_provider_strategy(model): - kwargs["strict"] = True + kwargs: dict[str, Any] = {"response_format": response_format, "strict": True} return kwargs diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index d44fecb7b5e78..57e7bb1c42635 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -789,48 +789,3 @@ def test_union_of_types() -> None: assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC assert len(response["messages"]) == 5 - - -def test_provider_strategy_strict_only_for_openai() -> None: - """Test that strict=True is set for OpenAI and Grok models in ProviderStrategy.""" - from langchain.agents.structured_output import ProviderStrategy - - # Create a mock OpenAI model with model_name - class MockOpenAIModel: - model_name: str = "gpt-4.1" - - # Create a mock Grok/X.AI model with model_name - class MockGrokModel: - model_name: str = "grok-beta" - - provider_strategy = ProviderStrategy(WeatherBaseModel) - - # Test OpenAI model: should include strict=True - openai_model = MockOpenAIModel() - openai_kwargs = provider_strategy.to_model_kwargs(model=openai_model) - assert "strict" in openai_kwargs - assert openai_kwargs["strict"] is True - assert "response_format" in openai_kwargs - - # Test Grok model: should include strict=True (Grok requires strict) - grok_model = MockGrokModel() - grok_kwargs = provider_strategy.to_model_kwargs(model=grok_model) - assert "strict" in grok_kwargs - assert grok_kwargs["strict"] is True - assert "response_format" in grok_kwargs - - -def test_provider_strategy_validation() -> None: - """Test that ProviderStrategy validates provider support at agent invocation time.""" - from langchain.agents.structured_output import ProviderStrategy - - # Create a mock model from an unsupported provider (e.g., Anthropic) - # Use a model_name that doesn't match any supported patterns - class MockAnthropicModel(FakeToolCallingModel): - model_name: str = "claude-3-5-sonnet-20241022" - - # Test unsupported provider: should raise ValueError when invoking agent - anthropic_model = MockAnthropicModel(tool_calls=[[]]) - agent = create_agent(anthropic_model, [], response_format=ProviderStrategy(WeatherBaseModel)) - with pytest.raises(ValueError, match="ProviderStrategy does not support this model"): - agent.invoke({"messages": [HumanMessage("What's the weather?")]}) From d8d94e6c95f4474793c22f8c531816d8943d3e1f Mon Sep 17 00:00:00 2001 From: Sydney Runkle <54324534+sydney-runkle@users.noreply.github.com> Date: Thu, 16 Oct 2025 13:38:45 -0400 Subject: [PATCH 07/10] Apply suggestions from code review --- libs/langchain_v1/tests/unit_tests/agents/model.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libs/langchain_v1/tests/unit_tests/agents/model.py b/libs/langchain_v1/tests/unit_tests/agents/model.py index 17f493fbbe915..c56a81bcb38cb 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/model.py +++ b/libs/langchain_v1/tests/unit_tests/agents/model.py @@ -30,7 +30,6 @@ class FakeToolCallingModel(BaseChatModel, Generic[StructuredResponseT]): structured_response: StructuredResponseT | None = None index: int = 0 tool_style: Literal["openai", "anthropic"] = "openai" - ls_provider: str = "openai" model_name: str = "fake-model" def _generate( @@ -55,7 +54,6 @@ def _generate( tool_calls = [] if is_native and not tool_calls: - content_obj = {} if isinstance(self.structured_response, BaseModel): content_obj = self.structured_response.model_dump() elif is_dataclass(self.structured_response): @@ -77,10 +75,6 @@ def _generate( def _llm_type(self) -> str: return "fake-tool-call-model" - def _get_ls_params(self, **kwargs: Any) -> LangSmithParams: - """Get LangSmith parameters for this model.""" - return LangSmithParams(ls_provider=self.ls_provider, ls_model_type="chat") - def bind_tools( self, tools: Sequence[Union[dict[str, Any], type[BaseModel], Callable, BaseTool]], From c8b116206772b2c99ddc65bc68f5f5f24a0a58bc Mon Sep 17 00:00:00 2001 From: Sydney Runkle <54324534+sydney-runkle@users.noreply.github.com> Date: Thu, 16 Oct 2025 13:40:11 -0400 Subject: [PATCH 08/10] Apply suggestions from code review --- libs/langchain_v1/langchain/agents/structured_output.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index 4d05c961e94a3..33e807c2bdbf7 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -342,11 +342,7 @@ def to_model_kwargs(self) -> dict[str, Any]: }, } - # Set strict=True for OpenAI and X.AI (Grok) models - # Both providers require strict=True for structured output - kwargs: dict[str, Any] = {"response_format": response_format, "strict": True} - - return kwargs + return {"response_format", response_format, "strict": True} @dataclass From e18a21a8b0396f56cbfa11e6468d4b6bdd221e82 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 14:24:54 -0400 Subject: [PATCH 09/10] tests' --- .../unit_tests/agents/test_response_format.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index 57e7bb1c42635..0f8fa6d0122df 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -632,6 +632,78 @@ def test_pydantic_model(self) -> None: assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC assert len(response["messages"]) == 4 + def test_unsupported_model_raises_error(self) -> None: + """Test that ProviderStrategy raises ValueError for unsupported models.""" + tool_calls = [ + [{"args": {}, "id": "1", "name": "get_weather"}], + ] + + # Use a model name that doesn't support provider strategy + model = FakeToolCallingModel[WeatherBaseModel]( + tool_calls=tool_calls, + structured_response=EXPECTED_WEATHER_PYDANTIC, + model_name="claude-3-5-sonnet", + ) + + agent = create_agent( + model, [get_weather], response_format=ProviderStrategy(WeatherBaseModel) + ) + + with pytest.raises( + ValueError, + match=( + r"Cannot use ProviderStrategy with claude-3-5-sonnet\. " + r"Supported models: OpenAI \(gpt-5, gpt-4\.1, gpt-oss, o3-pro, o3-mini\), " + r"X\.AI \(Grok\)\. " + r"Consider using a raw schema \(which auto-selects the best strategy\) or " + r"explicitly use `ToolStrategy` for unsupported providers\." + ), + ): + agent.invoke({"messages": [HumanMessage("What's the weather?")]}) + + def test_supported_openai_models(self) -> None: + """Test that ProviderStrategy works with all supported OpenAI model variants.""" + supported_models = ["gpt-5", "gpt-4.1", "gpt-oss", "o3-pro", "o3-mini"] + + for model_name in supported_models: + tool_calls = [ + [{"args": {}, "id": "1", "name": "get_weather"}], + ] + + model = FakeToolCallingModel[WeatherBaseModel]( + tool_calls=tool_calls, + structured_response=EXPECTED_WEATHER_PYDANTIC, + model_name=model_name, + ) + + agent = create_agent( + model, [get_weather], response_format=ProviderStrategy(WeatherBaseModel) + ) + response = agent.invoke({"messages": [HumanMessage("What's the weather?")]}) + + assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC + assert len(response["messages"]) == 4 + + def test_supported_grok_model(self) -> None: + """Test that ProviderStrategy works with Grok models.""" + tool_calls = [ + [{"args": {}, "id": "1", "name": "get_weather"}], + ] + + model = FakeToolCallingModel[WeatherBaseModel]( + tool_calls=tool_calls, + structured_response=EXPECTED_WEATHER_PYDANTIC, + model_name="grok-beta", + ) + + agent = create_agent( + model, [get_weather], response_format=ProviderStrategy(WeatherBaseModel) + ) + response = agent.invoke({"messages": [HumanMessage("What's the weather?")]}) + + assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC + assert len(response["messages"]) == 4 + def test_dataclass(self) -> None: """Test response_format as ProviderStrategy with dataclass.""" tool_calls = [ From c16969d8ac2310ff591062b6c867dd96a677fba9 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 16 Oct 2025 15:15:51 -0400 Subject: [PATCH 10/10] typo --- libs/langchain_v1/langchain/agents/structured_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index 33e807c2bdbf7..02d5f44a84d8f 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -342,7 +342,7 @@ def to_model_kwargs(self) -> dict[str, Any]: }, } - return {"response_format", response_format, "strict": True} + return {"response_format": response_format, "strict": True} @dataclass