multiple: enforce standards on tool_choice (#30372)

- Test if models support forcing tool calls via `tool_choice`. If they
do, they should support
  - `"any"` to specify any tool
  - the tool name as a string to force calling a particular tool
- Add `tool_choice` to signature of `BaseChatModel.bind_tools` in core
- Deprecate `tool_choice_value` in standard tests in favor of a boolean
`has_tool_choice`

Will follow up with PRs in external repos (tested in AWS and Google
already).
This commit is contained in:
ccurme
2025-03-20 13:48:59 -04:00
committed by GitHub
parent b86cd8270c
commit de3960d285
10 changed files with 190 additions and 56 deletions

View File

@@ -2,6 +2,7 @@
:autodoc-options: autoproperty
"""
import inspect
import os
from abc import abstractmethod
from typing import Any, Dict, List, Literal, Optional, Tuple, Type
@@ -127,6 +128,14 @@ class ChatModelTests(BaseStandardTests):
"""(None or str) to use for tool choice when used in tests."""
return None
@property
def has_tool_choice(self) -> bool:
"""(bool) whether the model supports tool calling."""
bind_tools_params = inspect.signature(
self.chat_model_class.bind_tools
).parameters
return "tool_choice" in bind_tools_params
@property
def has_structured_output(self) -> bool:
"""(bool) whether the chat model supports structured output."""
@@ -273,12 +282,11 @@ class ChatModelUnitTests(ChatModelTests):
Value to use for tool choice when used in tests.
Some tests for tool calling features attempt to force tool calling via a
`tool_choice` parameter. A common value for this parameter is "any". Defaults
to `None`.
Note: if the value is set to "tool_name", the name of the tool used in each
test will be set as the value for `tool_choice`.
.. warning:: Deprecated since version 0.3.15:
This property will be removed in version 0.3.20. If a model does not
support forcing tool calling, override the ``has_tool_choice`` property to
return ``False``. Otherwise, models should accept values of ``"any"`` or
the name of a tool in ``tool_choice``.
Example:
@@ -288,6 +296,26 @@ class ChatModelUnitTests(ChatModelTests):
def tool_choice_value(self) -> Optional[str]:
return "any"
.. dropdown:: has_tool_choice
Boolean property indicating whether the chat model supports forcing tool
calling via a ``tool_choice`` parameter.
By default, this is determined by whether the parameter is included in the
signature for the corresponding ``bind_tools`` method.
If ``True``, the minimum requirement for this feature is that
``tool_choice="any"`` will force a tool call, and ``tool_choice=<tool name>``
will force a call to a specific tool.
Example override:
.. code-block:: python
@property
def has_tool_choice(self) -> bool:
return False
.. dropdown:: has_structured_output
Boolean property indicating whether the chat model supports structured