github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-21 14:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

standard-tests[patch]: use has_structured_output property to engage structured output tests (#29635)

Browse Source 
Motivation: dedicated structured output features are becoming more
common, such that integrations can support structured output without
supporting tool calling.

Here we make two changes:

1. Update the `has_structured_output` method to default to True if a
model supports tool calling (in addition to defaulting to True if
`with_structured_output` is overridden).
2. Update structured output tests to engage if `has_structured_output`
is True.
This commit is contained in:
ccurme 2025-02-06 13:09:06 -05:00 committed by GitHub
parent db8201d4da
commit 9da06e6e94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 40 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
libs/standard-tests/langchain_tests/integration_tests/chat_models.py
View File

@ -1202,20 +1202,23 @@ class ChatModelIntegrationTests(ChatModelTests):
"""Test to verify structured output is generated both on invoke and stream.
This test is optional and should be skipped if the model does not support
tool calling (see Configuration below).
structured output (see Configuration below).
.. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your
test class:
To disable structured output tests, set ``has_structured_output`` to False
in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
def has_tool_calling(self) -> bool:
def has_structured_output(self) -> bool:
return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@ -1225,8 +1228,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
if not self.has_structured_output:
pytest.skip("Test requires structured output.")
schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
chat = model.with_structured_output(schema, **self.structured_output_kwargs)
@ -1277,20 +1280,23 @@ class ChatModelIntegrationTests(ChatModelTests):
"""Test to verify structured output is generated both on invoke and stream.
This test is optional and should be skipped if the model does not support
tool calling (see Configuration below).
structured output (see Configuration below).
.. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your
test class:
To disable structured output tests, set ``has_structured_output`` to False
in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
def has_tool_calling(self) -> bool:
def has_structured_output(self) -> bool:
return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@ -1300,8 +1306,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
if not self.has_structured_output:
pytest.skip("Test requires structured output.")
schema, validation_function = _get_joke_class(schema_type) # type: ignore[arg-type]
@ -1352,20 +1358,23 @@ class ChatModelIntegrationTests(ChatModelTests):
pydantic.v1.BaseModel is available in the pydantic 2 package.
This test is optional and should be skipped if the model does not support
tool calling (see Configuration below).
structured output (see Configuration below).
.. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your
test class:
To disable structured output tests, set ``has_structured_output`` to False
in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
def has_tool_calling(self) -> bool:
def has_structured_output(self) -> bool:
return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@ -1375,8 +1384,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
"""
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
if not self.has_structured_output:
pytest.skip("Test requires structured output.")
class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel
"""Joke to tell user."""
@ -1410,20 +1419,23 @@ class ChatModelIntegrationTests(ChatModelTests):
parameters.
This test is optional and should be skipped if the model does not support
tool calling (see Configuration below).
structured output (see Configuration below).
.. dropdown:: Configuration
To disable tool calling tests, set ``has_tool_calling`` to False in your
test class:
To disable structured output tests, set ``has_structured_output`` to False
in your test class:
.. code-block:: python
class TestMyChatModelIntegration(ChatModelIntegrationTests):
@property
def has_tool_calling(self) -> bool:
def has_structured_output(self) -> bool:
return False
By default, ``has_structured_output`` is True if a model overrides the
``with_structured_output`` or ``bind_tools`` methods.
.. dropdown:: Troubleshooting
If this test fails, ensure that the model's ``bind_tools`` method
@ -1433,8 +1445,8 @@ class ChatModelIntegrationTests(ChatModelTests):
See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output
"""
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
if not self.has_structured_output:
pytest.skip("Test requires structured output.")
# Pydantic
class Joke(BaseModel):

8
libs/standard-tests/langchain_tests/unit_tests/chat_models.py
View File

@ -133,7 +133,7 @@ class ChatModelTests(BaseStandardTests):
return (
self.chat_model_class.with_structured_output
is not BaseChatModel.with_structured_output
)
) or self.has_tool_calling
@property
def structured_output_kwargs(self) -> dict:
@ -293,9 +293,9 @@ class ChatModelUnitTests(ChatModelTests):
Boolean property indicating whether the chat model supports structured
output.
By default, this is determined by whether the chat model's
``with_structured_output`` method is overridden. If the base implementation is
intended to be used, this method should be overridden.
By default, this is determined by whether the chat model overrides the
``with_structured_output`` or ``bind_tools`` methods. If the base
implementations are intended to be used, this method should be overridden.
See: https://python.langchain.com/docs/concepts/structured_outputs/