From 9da06e6e94fe59d87f235e2218358586de896ea5 Mon Sep 17 00:00:00 2001 From: ccurme Date: Thu, 6 Feb 2025 13:09:06 -0500 Subject: [PATCH] standard-tests[patch]: use `has_structured_output` property to engage structured output tests (#29635) Motivation: dedicated structured output features are becoming more common, such that integrations can support structured output without supporting tool calling. Here we make two changes: 1. Update the `has_structured_output` method to default to True if a model supports tool calling (in addition to defaulting to True if `with_structured_output` is overridden). 2. Update structured output tests to engage if `has_structured_output` is True. --- .../integration_tests/chat_models.py | 60 +++++++++++-------- .../langchain_tests/unit_tests/chat_models.py | 8 +-- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index f7371c6bcf7..0b0c6a54046 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -1202,20 +1202,23 @@ class ChatModelIntegrationTests(ChatModelTests): """Test to verify structured output is generated both on invoke and stream. This test is optional and should be skipped if the model does not support - tool calling (see Configuration below). + structured output (see Configuration below). .. dropdown:: Configuration - To disable tool calling tests, set ``has_tool_calling`` to False in your - test class: + To disable structured output tests, set ``has_structured_output`` to False + in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property - def has_tool_calling(self) -> bool: + def has_structured_output(self) -> bool: return False + By default, ``has_structured_output`` is True if a model overrides the + ``with_structured_output`` or ``bind_tools`` methods. + .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method @@ -1225,8 +1228,8 @@ class ChatModelIntegrationTests(ChatModelTests): See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output """ # noqa: E501 - if not self.has_tool_calling: - pytest.skip("Test requires tool calling.") + if not self.has_structured_output: + pytest.skip("Test requires structured output.") schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type] chat = model.with_structured_output(schema, **self.structured_output_kwargs) @@ -1277,20 +1280,23 @@ class ChatModelIntegrationTests(ChatModelTests): """Test to verify structured output is generated both on invoke and stream. This test is optional and should be skipped if the model does not support - tool calling (see Configuration below). + structured output (see Configuration below). .. dropdown:: Configuration - To disable tool calling tests, set ``has_tool_calling`` to False in your - test class: + To disable structured output tests, set ``has_structured_output`` to False + in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property - def has_tool_calling(self) -> bool: + def has_structured_output(self) -> bool: return False + By default, ``has_structured_output`` is True if a model overrides the + ``with_structured_output`` or ``bind_tools`` methods. + .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method @@ -1300,8 +1306,8 @@ class ChatModelIntegrationTests(ChatModelTests): See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output """ # noqa: E501 - if not self.has_tool_calling: - pytest.skip("Test requires tool calling.") + if not self.has_structured_output: + pytest.skip("Test requires structured output.") schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type] @@ -1352,20 +1358,23 @@ class ChatModelIntegrationTests(ChatModelTests): pydantic.v1.BaseModel is available in the pydantic 2 package. This test is optional and should be skipped if the model does not support - tool calling (see Configuration below). + structured output (see Configuration below). .. dropdown:: Configuration - To disable tool calling tests, set ``has_tool_calling`` to False in your - test class: + To disable structured output tests, set ``has_structured_output`` to False + in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property - def has_tool_calling(self) -> bool: + def has_structured_output(self) -> bool: return False + By default, ``has_structured_output`` is True if a model overrides the + ``with_structured_output`` or ``bind_tools`` methods. + .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method @@ -1375,8 +1384,8 @@ class ChatModelIntegrationTests(ChatModelTests): See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output """ - if not self.has_tool_calling: - pytest.skip("Test requires tool calling.") + if not self.has_structured_output: + pytest.skip("Test requires structured output.") class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel """Joke to tell user.""" @@ -1410,20 +1419,23 @@ class ChatModelIntegrationTests(ChatModelTests): parameters. This test is optional and should be skipped if the model does not support - tool calling (see Configuration below). + structured output (see Configuration below). .. dropdown:: Configuration - To disable tool calling tests, set ``has_tool_calling`` to False in your - test class: + To disable structured output tests, set ``has_structured_output`` to False + in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property - def has_tool_calling(self) -> bool: + def has_structured_output(self) -> bool: return False + By default, ``has_structured_output`` is True if a model overrides the + ``with_structured_output`` or ``bind_tools`` methods. + .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method @@ -1433,8 +1445,8 @@ class ChatModelIntegrationTests(ChatModelTests): See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output """ - if not self.has_tool_calling: - pytest.skip("Test requires tool calling.") + if not self.has_structured_output: + pytest.skip("Test requires structured output.") # Pydantic class Joke(BaseModel): diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py index e2b58d2f050..65dea9079b8 100644 --- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py @@ -133,7 +133,7 @@ class ChatModelTests(BaseStandardTests): return ( self.chat_model_class.with_structured_output is not BaseChatModel.with_structured_output - ) + ) or self.has_tool_calling @property def structured_output_kwargs(self) -> dict: @@ -293,9 +293,9 @@ class ChatModelUnitTests(ChatModelTests): Boolean property indicating whether the chat model supports structured output. - By default, this is determined by whether the chat model's - ``with_structured_output`` method is overridden. If the base implementation is - intended to be used, this method should be overridden. + By default, this is determined by whether the chat model overrides the + ``with_structured_output`` or ``bind_tools`` methods. If the base + implementations are intended to be used, this method should be overridden. See: https://python.langchain.com/docs/concepts/structured_outputs/