github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-21 14:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

standard-tests[patch]: use has_structured_output property to engage structured output tests (#29635)

Browse Source 
Motivation: dedicated structured output features are becoming more
common, such that integrations can support structured output without
supporting tool calling.

Here we make two changes:

1. Update the `has_structured_output` method to default to True if a
model supports tool calling (in addition to defaulting to True if
`with_structured_output` is overridden).
2. Update structured output tests to engage if `has_structured_output`
is True.
This commit is contained in:
ccurme 2025-02-06 13:09:06 -05:00 committed by GitHub
parent db8201d4da
commit 9da06e6e94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 40 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
libs/standard-tests/langchain_tests/integration_tests/chat_models.py
View File

@ -1202,20 +1202,23 @@ class ChatModelIntegrationTests(ChatModelTests):
"""Test to verify structured output is generated both on invoke and stream. """Test to verify structured output is generated both on invoke and stream.
This test is optional and should be skipped if the model does not support This test is optional and should be skipped if the model does not support
tool calling (see Configuration below). structured output (see Configuration below).
.. dropdown:: Configuration .. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your To disable structured output tests, set ``has_structured_output`` to False
test class: in your test class:
.. code-block:: python .. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests): class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property @property
def has_tool_calling(self) -> bool: def has_structured_output(self) -> bool:
return False return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting .. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method If this test fails, ensure that the model's ``bind_tools`` method
@ -1225,8 +1228,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" # noqa: E501 """ # noqa: E501
if not self.has_tool_calling: if not self.has_structured_output:
pytest.skip("Test requires tool calling.") pytest.skip("Test requires structured output.")
schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type] schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
chat = model.with_structured_output(schema, **self.structured_output_kwargs) chat = model.with_structured_output(schema, **self.structured_output_kwargs)
@ -1277,20 +1280,23 @@ class ChatModelIntegrationTests(ChatModelTests):
"""Test to verify structured output is generated both on invoke and stream. """Test to verify structured output is generated both on invoke and stream.
This test is optional and should be skipped if the model does not support This test is optional and should be skipped if the model does not support
tool calling (see Configuration below). structured output (see Configuration below).
.. dropdown:: Configuration .. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your To disable structured output tests, set ``has_structured_output`` to False
test class: in your test class:
.. code-block:: python .. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests): class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property @property
def has_tool_calling(self) -> bool: def has_structured_output(self) -> bool:
return False return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting .. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method If this test fails, ensure that the model's ``bind_tools`` method
@ -1300,8 +1306,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" # noqa: E501 """ # noqa: E501
if not self.has_tool_calling: if not self.has_structured_output:
pytest.skip("Test requires tool calling.") pytest.skip("Test requires structured output.")
schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type] schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
@ -1352,20 +1358,23 @@ class ChatModelIntegrationTests(ChatModelTests):
pydantic.v1.BaseModel is available in the pydantic 2 package. pydantic.v1.BaseModel is available in the pydantic 2 package.
This test is optional and should be skipped if the model does not support This test is optional and should be skipped if the model does not support
tool calling (see Configuration below). structured output (see Configuration below).
.. dropdown:: Configuration .. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your To disable structured output tests, set ``has_structured_output`` to False
test class: in your test class:
.. code-block:: python .. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests): class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property @property
def has_tool_calling(self) -> bool: def has_structured_output(self) -> bool:
return False return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting .. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method If this test fails, ensure that the model's ``bind_tools`` method
@ -1375,8 +1384,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" """
if not self.has_tool_calling: if not self.has_structured_output:
pytest.skip("Test requires tool calling.") pytest.skip("Test requires structured output.")
class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel
"""Joke to tell user.""" """Joke to tell user."""
@ -1410,20 +1419,23 @@ class ChatModelIntegrationTests(ChatModelTests):
parameters. parameters.
This test is optional and should be skipped if the model does not support This test is optional and should be skipped if the model does not support
tool calling (see Configuration below). structured output (see Configuration below).
.. dropdown:: Configuration .. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your To disable structured output tests, set ``has_structured_output`` to False
test class: in your test class:
.. code-block:: python .. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests): class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property @property
def has_tool_calling(self) -> bool: def has_structured_output(self) -> bool:
return False return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting .. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method If this test fails, ensure that the model's ``bind_tools`` method
@ -1433,8 +1445,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" """
if not self.has_tool_calling: if not self.has_structured_output:
pytest.skip("Test requires tool calling.") pytest.skip("Test requires structured output.")
# Pydantic # Pydantic
class Joke(BaseModel): class Joke(BaseModel):

8
libs/standard-tests/langchain_tests/unit_tests/chat_models.py
View File

@ -133,7 +133,7 @@ class ChatModelTests(BaseStandardTests):
return ( return (
self.chat_model_class.with_structured_output self.chat_model_class.with_structured_output
is not BaseChatModel.with_structured_output is not BaseChatModel.with_structured_output
) ) or self.has_tool_calling
@property @property
def structured_output_kwargs(self) -> dict: def structured_output_kwargs(self) -> dict:
@ -293,9 +293,9 @@ class ChatModelUnitTests(ChatModelTests):
Boolean property indicating whether the chat model supports structured Boolean property indicating whether the chat model supports structured
output. output.
By default, this is determined by whether the chat model's By default, this is determined by whether the chat model overrides the
``with_structured_output`` method is overridden. If the base implementation is ``with_structured_output`` or ``bind_tools`` methods. If the base
intended to be used, this method should be overridden. implementations are intended to be used, this method should be overridden.
See: https://python.langchain.com/docs/concepts/structured_outputs/ See: https://python.langchain.com/docs/concepts/structured_outputs/