From 80a88f8f045408c11b5605c254bdc23c1536daef Mon Sep 17 00:00:00 2001 From: ccurme Date: Fri, 6 Dec 2024 19:00:14 -0500 Subject: [PATCH] tests[patch]: update API ref for chat models (#28594) --- .../how_to/integrations/standard_tests.ipynb | 4 +- .../integration_tests/chat_models.py | 27 ++++++--- .../langchain_tests/unit_tests/chat_models.py | 56 ++++++++++--------- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/docs/docs/contributing/how_to/integrations/standard_tests.ipynb b/docs/docs/contributing/how_to/integrations/standard_tests.ipynb index 46b20e24fa8..c567393d95b 100644 --- a/docs/docs/contributing/how_to/integrations/standard_tests.ipynb +++ b/docs/docs/contributing/how_to/integrations/standard_tests.ipynb @@ -230,7 +230,9 @@ "\n", "Tests for \"optional\" capabilities are controlled via a set of properties that can be overridden on the test model subclass.\n", "\n", - "You can see the entire list of properties in the API reference [here](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelTests.html). These properties are shared by both unit and integration tests.\n", + "You can see the entire list of properties in the API references for\n", + "[unit tests](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelUnitTests.html)\n", + "and [integration tests](https://python.langchain.com/api_reference/standard_tests/integration_tests/langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.html).\n", "\n", "For example, to enable integration tests for image inputs, we can implement\n", "\n", diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index 0dd5334cc48..7a787563b26 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -145,6 +145,14 @@ class ChatModelIntegrationTests(ChatModelTests): By default, this is determined by whether the chat model's `bind_tools` method is overridden. It typically does not need to be overridden on the test class. + Example override: + + .. code-block:: python + + @property + def has_tool_calling(self) -> bool: + return True + .. dropdown:: tool_choice_value Value to use for tool choice when used in tests. @@ -308,7 +316,7 @@ class ChatModelIntegrationTests(ChatModelTests): @property def standard_chat_model_params(self) -> dict: - """:meta private:""" + """:private:""" return {} def test_invoke(self, model: BaseChatModel) -> None: @@ -1323,8 +1331,8 @@ class ChatModelIntegrationTests(ChatModelTests): .. code-block:: python @pytest.mark.xfail(reason=("Not implemented.")) - def test_tool_message_histories_string_content(self, model: BaseChatModel) -> None: - super().test_tool_message_histories_string_content(model) + def test_tool_message_histories_string_content(self, *args: Any) -> None: + super().test_tool_message_histories_string_content(*args) """ # noqa: E501 if not self.has_tool_calling: pytest.skip("Test requires tool calling.") @@ -1407,8 +1415,8 @@ class ChatModelIntegrationTests(ChatModelTests): .. code-block:: python @pytest.mark.xfail(reason=("Not implemented.")) - def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None: - super().test_tool_message_histories_list_content(model) + def test_tool_message_histories_list_content(self, *args: Any) -> None: + super().test_tool_message_histories_list_content(*args) """ # noqa: E501 if not self.has_tool_calling: pytest.skip("Test requires tool calling.") @@ -1489,8 +1497,8 @@ class ChatModelIntegrationTests(ChatModelTests): .. code-block:: python @pytest.mark.xfail(reason=("Not implemented.")) - def test_structured_few_shot_examples(self, model: BaseChatModel) -> None: - super().test_structured_few_shot_examples(model) + def test_structured_few_shot_examples(self, *args: Any) -> None: + super().test_structured_few_shot_examples(*args) """ # noqa: E501 if not self.has_tool_calling: pytest.skip("Test requires tool calling.") @@ -1852,16 +1860,21 @@ class ChatModelIntegrationTests(ChatModelTests): assert len(result.content) > 0 def invoke_with_audio_input(self, *, stream: bool = False) -> AIMessage: + """:private:""" raise NotImplementedError() def invoke_with_audio_output(self, *, stream: bool = False) -> AIMessage: + """:private:""" raise NotImplementedError() def invoke_with_reasoning_output(self, *, stream: bool = False) -> AIMessage: + """:private:""" raise NotImplementedError() def invoke_with_cache_read_input(self, *, stream: bool = False) -> AIMessage: + """:private:""" raise NotImplementedError() def invoke_with_cache_creation_input(self, *, stream: bool = False) -> AIMessage: + """:private:""" raise NotImplementedError() diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py index 19225c1f2a6..f698f658d08 100644 --- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py @@ -77,17 +77,17 @@ class ChatModelTests(BaseStandardTests): @property @abstractmethod def chat_model_class(self) -> Type[BaseChatModel]: - """The chat model class to test, e.g., `ChatParrotLink`.""" + """The chat model class to test, e.g., ``ChatParrotLink``.""" ... @property def chat_model_params(self) -> dict: - """Initialization parameters for the chat mobdel.""" + """Initialization parameters for the chat model.""" return {} @property def standard_chat_model_params(self) -> dict: - """:meta private:""" + """:private:""" return { "temperature": 0, "max_tokens": 100, @@ -98,8 +98,7 @@ class ChatModelTests(BaseStandardTests): @pytest.fixture def model(self) -> BaseChatModel: - """Fixture that returns an instance of the chat model. Should not be - overridden.""" + """:private:""" return self.chat_model_class( **{**self.standard_chat_model_params, **self.chat_model_params} ) @@ -115,18 +114,17 @@ class ChatModelTests(BaseStandardTests): @property def has_tool_calling(self) -> bool: - """Boolean property indicating whether the model supports tool calling.""" + """(bool) whether the model supports tool calling.""" return self.chat_model_class.bind_tools is not BaseChatModel.bind_tools @property def tool_choice_value(self) -> Optional[str]: - """Value to use for tool choice when used in tests.""" + """(None or str) to use for tool choice when used in tests.""" return None @property def has_structured_output(self) -> bool: - """Boolean property indicating whether the chat model supports structured - output.""" + """(bool) whether the chat model supports structured output.""" return ( self.chat_model_class.with_structured_output is not BaseChatModel.with_structured_output @@ -134,32 +132,31 @@ class ChatModelTests(BaseStandardTests): @property def supports_image_inputs(self) -> bool: - """Boolean property indicating whether the chat model supports image inputs. - Defaults to ``False``.""" + """(bool) whether the chat model supports image inputs, defaults to + ``False``.""" return False @property def supports_video_inputs(self) -> bool: - """Boolean property indicating whether the chat model supports image inputs. - Defaults to ``False``. No current tests are written for this feature.""" + """(bool) whether the chat model supports video inputs, efaults to ``False``. + No current tests are written for this feature.""" return False @property def returns_usage_metadata(self) -> bool: - """Boolean property indicating whether the chat model returns usage metadata - on invoke and streaming responses.""" + """(bool) whether the chat model returns usage metadata on invoke and streaming + responses.""" return True @property def supports_anthropic_inputs(self) -> bool: - """Boolean property indicating whether the chat model supports Anthropic-style - inputs.""" + """(bool) whether the chat model supports Anthropic-style inputs.""" return False @property def supports_image_tool_message(self) -> bool: - """Boolean property indicating whether the chat model supports ToolMessages - that include image content.""" + """(bool) whether the chat model supports ToolMessages that include image + content.""" return False @property @@ -177,9 +174,8 @@ class ChatModelTests(BaseStandardTests): ] ], ]: - """Property controlling what usage metadata details are emitted in both invoke - and stream. Only needs to be overridden if these details are returned by the - model.""" + """(dict) what usage metadata details are emitted in invoke and stream. Only + needs to be overridden if these details are returned by the model.""" return {"invoke": [], "stream": []} @@ -250,6 +246,14 @@ class ChatModelUnitTests(ChatModelTests): By default, this is determined by whether the chat model's `bind_tools` method is overridden. It typically does not need to be overridden on the test class. + Example override: + + .. code-block:: python + + @property + def has_tool_calling(self) -> bool: + return True + .. dropdown:: tool_choice_value Value to use for tool choice when used in tests. @@ -445,17 +449,15 @@ class ChatModelUnitTests(ChatModelTests): @property def standard_chat_model_params(self) -> dict: - """:meta private:""" + """:private:""" params = super().standard_chat_model_params params["api_key"] = "test" return params @property def init_from_env_params(self) -> Tuple[dict, dict, dict]: - """This property is used in unit tests to test initialization from environment - variables. It should return a tuple of three dictionaries that specify the - environment variables, additional initialization args, and expected instance - attributes to check.""" + """(tuple) environment variables, additional initialization args, and expected + instance attributes for testing initialization from environment variables.""" return {}, {}, {} def test_init(self) -> None: