mirror of
https://github.com/hwchase17/langchain.git
synced 2026-06-09 10:17:00 +00:00
feat(anthropic): auto append relevant beta headers for computer use (#34117)
in addition to documenting it https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool
This commit is contained in:
@@ -138,6 +138,8 @@ _TOOL_TYPE_TO_BETA: dict[str, str] = {
|
||||
"code_execution_20250522": "code-execution-2025-05-22",
|
||||
"code_execution_20250825": "code-execution-2025-08-25",
|
||||
"memory_20250818": "context-management-2025-06-27",
|
||||
"computer_20250124": "computer-use-2025-01-24",
|
||||
"computer_20251124": "computer-use-2025-11-24",
|
||||
"tool_search_tool_regex_20251119": "advanced-tool-use-2025-11-20",
|
||||
"tool_search_tool_bm25_20251119": "advanced-tool-use-2025-11-20",
|
||||
}
|
||||
@@ -2531,6 +2533,60 @@ class ChatAnthropic(BaseChatModel):
|
||||
)
|
||||
```
|
||||
|
||||
??? example "Computer use tool"
|
||||
|
||||
Claude supports computer use capabilities, allowing it to interact with
|
||||
desktop environments through screenshots, mouse control, and keyboard input.
|
||||
|
||||
!!! warning "Execution environment required"
|
||||
|
||||
LangChain handles the API integration, but **you must provide**:
|
||||
|
||||
- A sandboxed computing environment (Docker, VM, etc.)
|
||||
- A virtual display (e.g., Xvfb)
|
||||
- Code to execute tool calls (screenshot, clicks, typing)
|
||||
- An agent loop to pass results back to Claude
|
||||
|
||||
Anthropic provides a [reference implementation](https://github.com/anthropics/anthropic-quickstarts/tree/main/computer-use-demo).
|
||||
|
||||
!!! note
|
||||
|
||||
Computer use requires:
|
||||
|
||||
- Claude Opus 4.5, Claude 4, or Claude Sonnet 3.7
|
||||
- A sandboxed computing environment with virtual display
|
||||
|
||||
See the [Claude docs](https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool)
|
||||
for setup instructions, model capability, and best practices.
|
||||
|
||||
```python
|
||||
from langchain_anthropic import ChatAnthropic
|
||||
|
||||
model = ChatAnthropic(model="claude-sonnet-4-5-20250929")
|
||||
|
||||
# LangChain handles the API call and tool binding
|
||||
computer_tool = {
|
||||
"type": "computer_20250124",
|
||||
"name": "computer",
|
||||
"display_width_px": 1024,
|
||||
"display_height_px": 768,
|
||||
"display_number": 1,
|
||||
}
|
||||
|
||||
model_with_computer = model.bind_tools([computer_tool])
|
||||
response = model_with_computer.invoke("Take a screenshot to see what's on the screen")
|
||||
|
||||
# response.tool_calls contains the action Claude wants to perform
|
||||
# You must execute this action in your environment and pass the result back
|
||||
```
|
||||
|
||||
!!! note "Automatic beta header"
|
||||
|
||||
The required beta header is automatically appended based on the tool
|
||||
version. For `computer_20250124` and `computer_20251124`, the respective
|
||||
`computer-use-2025-01-24` and `computer-use-2025-11-24` beta header is
|
||||
added automatically.
|
||||
|
||||
??? example "Strict tool use"
|
||||
|
||||
Strict tool use guarantees that tool names and arguments are validated
|
||||
|
||||
@@ -118,6 +118,23 @@ _PROFILES: dict[str, dict[str, Any]] = {
|
||||
"image_tool_message": True,
|
||||
"structured_output": False,
|
||||
},
|
||||
"claude-opus-4-5": {
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"image_inputs": True,
|
||||
"audio_inputs": False,
|
||||
"video_inputs": False,
|
||||
"image_outputs": False,
|
||||
"audio_outputs": False,
|
||||
"video_outputs": False,
|
||||
"reasoning_output": True,
|
||||
"tool_calling": True,
|
||||
"image_url_inputs": True,
|
||||
"pdf_inputs": True,
|
||||
"pdf_tool_message": True,
|
||||
"image_tool_message": True,
|
||||
"structured_output": False,
|
||||
},
|
||||
"claude-3-opus-20240229": {
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 4096,
|
||||
|
||||
@@ -582,6 +582,47 @@ def test_builtin_tools_text_editor() -> None:
|
||||
assert content_blocks[1]["name"] == "str_replace_based_edit_tool"
|
||||
|
||||
|
||||
def test_builtin_tools_computer_use() -> None:
|
||||
"""Test computer use tool integration.
|
||||
|
||||
Beta header should be automatically appended based on tool type.
|
||||
|
||||
This test only verifies tool call generation.
|
||||
"""
|
||||
llm = ChatAnthropic(
|
||||
model="claude-sonnet-4-5-20250929", # type: ignore[call-arg]
|
||||
)
|
||||
tool = {
|
||||
"type": "computer_20250124",
|
||||
"name": "computer",
|
||||
"display_width_px": 1024,
|
||||
"display_height_px": 768,
|
||||
"display_number": 1,
|
||||
}
|
||||
llm_with_tools = llm.bind_tools([tool])
|
||||
response = llm_with_tools.invoke(
|
||||
"Can you take a screenshot to see what's on the screen?",
|
||||
)
|
||||
assert isinstance(response, AIMessage)
|
||||
assert response.tool_calls
|
||||
|
||||
content_blocks = response.content_blocks
|
||||
assert len(content_blocks) >= 2
|
||||
assert content_blocks[0]["type"] == "text"
|
||||
assert content_blocks[0]["text"]
|
||||
|
||||
# Check that we have a tool_call for computer use
|
||||
tool_call_blocks = [b for b in content_blocks if b["type"] == "tool_call"]
|
||||
assert len(tool_call_blocks) >= 1
|
||||
assert tool_call_blocks[0]["name"] == "computer"
|
||||
|
||||
# Verify tool call has expected action (screenshot in this case)
|
||||
tool_call = response.tool_calls[0]
|
||||
assert tool_call["name"] == "computer"
|
||||
assert "action" in tool_call["args"]
|
||||
assert tool_call["args"]["action"] == "screenshot"
|
||||
|
||||
|
||||
class GenerateUsername(BaseModel):
|
||||
"""Get a username based on someone's name and hair color."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user