github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-23 15:19:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

tests[patch]: update API ref for chat models (#28594)

Browse Source
This commit is contained in:
ccurme 2024-12-06 19:00:14 -05:00 committed by GitHub
parent 0eb7ab65f1
commit 80a88f8f04
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 52 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/docs/contributing/how_to/integrations/standard_tests.ipynb
View File

@ -230,7 +230,9 @@
"\n",
"Tests for \"optional\" capabilities are controlled via a set of properties that can be overridden on the test model subclass.\n",
"\n",
"You can see the entire list of properties in the API reference [here](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelTests.html). These properties are shared by both unit and integration tests.\n",
"You can see the entire list of properties in the API references for\n",
"[unit tests](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelUnitTests.html)\n",
"and [integration tests](https://python.langchain.com/api_reference/standard_tests/integration_tests/langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.html).\n",
"\n",
"For example, to enable integration tests for image inputs, we can implement\n",
"\n",

27
libs/standard-tests/langchain_tests/integration_tests/chat_models.py
View File

@ -145,6 +145,14 @@ class ChatModelIntegrationTests(ChatModelTests):
By default, this is determined by whether the chat model's `bind_tools` method
is overridden. It typically does not need to be overridden on the test class.
Example override:
.. code-block:: python
@property
def has_tool_calling(self) -> bool:
return True
.. dropdown:: tool_choice_value
Value to use for tool choice when used in tests.
@ -308,7 +316,7 @@ class ChatModelIntegrationTests(ChatModelTests):
@property
def standard_chat_model_params(self) -> dict:
""":meta private:"""
""":private:"""
return {}
def test_invoke(self, model: BaseChatModel) -> None:
@ -1323,8 +1331,8 @@ class ChatModelIntegrationTests(ChatModelTests):
.. code-block:: python
@pytest.mark.xfail(reason=("Not implemented."))
def test_tool_message_histories_string_content(self, model: BaseChatModel) -> None:
super().test_tool_message_histories_string_content(model)
def test_tool_message_histories_string_content(self, *args: Any) -> None:
super().test_tool_message_histories_string_content(*args)
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
@ -1407,8 +1415,8 @@ class ChatModelIntegrationTests(ChatModelTests):
.. code-block:: python
@pytest.mark.xfail(reason=("Not implemented."))
def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None:
super().test_tool_message_histories_list_content(model)
def test_tool_message_histories_list_content(self, *args: Any) -> None:
super().test_tool_message_histories_list_content(*args)
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
@ -1489,8 +1497,8 @@ class ChatModelIntegrationTests(ChatModelTests):
.. code-block:: python
@pytest.mark.xfail(reason=("Not implemented."))
def test_structured_few_shot_examples(self, model: BaseChatModel) -> None:
super().test_structured_few_shot_examples(model)
def test_structured_few_shot_examples(self, *args: Any) -> None:
super().test_structured_few_shot_examples(*args)
""" # noqa: E501
if not self.has_tool_calling:
pytest.skip("Test requires tool calling.")
@ -1852,16 +1860,21 @@ class ChatModelIntegrationTests(ChatModelTests):
assert len(result.content) > 0
def invoke_with_audio_input(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()
def invoke_with_audio_output(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()
def invoke_with_reasoning_output(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()
def invoke_with_cache_read_input(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()
def invoke_with_cache_creation_input(self, *, stream: bool = False) -> AIMessage:
""":private:"""
raise NotImplementedError()

56
libs/standard-tests/langchain_tests/unit_tests/chat_models.py
View File

@ -77,17 +77,17 @@ class ChatModelTests(BaseStandardTests):
@property
@abstractmethod
def chat_model_class(self) -> Type[BaseChatModel]:
"""The chat model class to test, e.g., `ChatParrotLink`."""
"""The chat model class to test, e.g., ``ChatParrotLink``."""
...
@property
def chat_model_params(self) -> dict:
"""Initialization parameters for the chat mobdel."""
"""Initialization parameters for the chat model."""
return {}
@property
def standard_chat_model_params(self) -> dict:
""":meta private:"""
""":private:"""
return {
"temperature": 0,
"max_tokens": 100,
@ -98,8 +98,7 @@ class ChatModelTests(BaseStandardTests):
@pytest.fixture
def model(self) -> BaseChatModel:
"""Fixture that returns an instance of the chat model. Should not be
overridden."""
""":private:"""
return self.chat_model_class(
**{**self.standard_chat_model_params, **self.chat_model_params}
)
@ -115,18 +114,17 @@ class ChatModelTests(BaseStandardTests):
@property
def has_tool_calling(self) -> bool:
"""Boolean property indicating whether the model supports tool calling."""
"""(bool) whether the model supports tool calling."""
return self.chat_model_class.bind_tools is not BaseChatModel.bind_tools
@property
def tool_choice_value(self) -> Optional[str]:
"""Value to use for tool choice when used in tests."""
"""(None or str) to use for tool choice when used in tests."""
return None
@property
def has_structured_output(self) -> bool:
"""Boolean property indicating whether the chat model supports structured
output."""
"""(bool) whether the chat model supports structured output."""
return (
self.chat_model_class.with_structured_output
is not BaseChatModel.with_structured_output
@ -134,32 +132,31 @@ class ChatModelTests(BaseStandardTests):
@property
def supports_image_inputs(self) -> bool:
"""Boolean property indicating whether the chat model supports image inputs.
Defaults to ``False``."""
"""(bool) whether the chat model supports image inputs, defaults to
``False``."""
return False
@property
def supports_video_inputs(self) -> bool:
"""Boolean property indicating whether the chat model supports image inputs.
Defaults to ``False``. No current tests are written for this feature."""
"""(bool) whether the chat model supports video inputs, efaults to ``False``.
No current tests are written for this feature."""
return False
@property
def returns_usage_metadata(self) -> bool:
"""Boolean property indicating whether the chat model returns usage metadata
on invoke and streaming responses."""
"""(bool) whether the chat model returns usage metadata on invoke and streaming
responses."""
return True
@property
def supports_anthropic_inputs(self) -> bool:
"""Boolean property indicating whether the chat model supports Anthropic-style
inputs."""
"""(bool) whether the chat model supports Anthropic-style inputs."""
return False
@property
def supports_image_tool_message(self) -> bool:
"""Boolean property indicating whether the chat model supports ToolMessages
that include image content."""
"""(bool) whether the chat model supports ToolMessages that include image
content."""
return False
@property
@ -177,9 +174,8 @@ class ChatModelTests(BaseStandardTests):
]
],
]:
"""Property controlling what usage metadata details are emitted in both invoke
and stream. Only needs to be overridden if these details are returned by the
model."""
"""(dict) what usage metadata details are emitted in invoke and stream. Only
needs to be overridden if these details are returned by the model."""
return {"invoke": [], "stream": []}
@ -250,6 +246,14 @@ class ChatModelUnitTests(ChatModelTests):
By default, this is determined by whether the chat model's `bind_tools` method
is overridden. It typically does not need to be overridden on the test class.
Example override:
.. code-block:: python
@property
def has_tool_calling(self) -> bool:
return True
.. dropdown:: tool_choice_value
Value to use for tool choice when used in tests.
@ -445,17 +449,15 @@ class ChatModelUnitTests(ChatModelTests):
@property
def standard_chat_model_params(self) -> dict:
""":meta private:"""
""":private:"""
params = super().standard_chat_model_params
params["api_key"] = "test"
return params
@property
def init_from_env_params(self) -> Tuple[dict, dict, dict]:
"""This property is used in unit tests to test initialization from environment
variables. It should return a tuple of three dictionaries that specify the
environment variables, additional initialization args, and expected instance
attributes to check."""
"""(tuple) environment variables, additional initialization args, and expected
instance attributes for testing initialization from environment variables."""
return {}, {}, {}
def test_init(self) -> None: