From 26833f2ebc66f1d4e3e5748762bdbed37fe40468 Mon Sep 17 00:00:00 2001 From: ccurme Date: Fri, 22 Aug 2025 17:06:53 -0300 Subject: [PATCH] feat(anthropic): v1 support (#32623) --- .../langchain_core/language_models/_utils.py | 2 +- .../language_models/chat_models.py | 2 +- libs/core/langchain_core/messages/ai.py | 10 +- libs/core/langchain_core/messages/base.py | 43 +- .../messages/block_translators/__init__.py | 66 +-- .../block_translators/amazon/__init__.py | 1 - .../messages/block_translators/anthropic.py | 414 ++++++++++++++++- .../block_translators/{amazon => }/bedrock.py | 26 +- .../{amazon => }/bedrock_converse.py | 28 +- .../messages/block_translators/chroma.py | 27 -- .../block_translators/google/__init__.py | 1 - .../{google/genai.py => google_genai.py} | 22 +- .../vertexai.py => google_vertexai.py} | 24 +- .../messages/block_translators/groq.py | 22 +- .../{langchain.py => langchain_v0.py} | 43 +- .../messages/block_translators/ollama.py | 22 +- .../messages/block_translators/openai.py | 51 +- libs/core/langchain_core/messages/content.py | 8 - libs/core/langchain_core/utils/_merge.py | 34 +- .../language_models/chat_models/test_base.py | 8 +- .../block_translators/test_anthropic.py | 439 ++++++++++++++++++ .../block_translators/test_langchain_v0.py | 79 ++++ .../messages/block_translators/test_openai.py | 92 ++-- .../block_translators/test_registration.py | 29 ++ .../core/tests/unit_tests/messages/test_ai.py | 93 ++++ .../anthropic/langchain_anthropic/_compat.py | 245 ++++++++++ .../langchain_anthropic/chat_models.py | 95 +++- .../tests/cassettes/test_agent_loop.yaml.gz | Bin 0 -> 2028 bytes .../test_agent_loop_streaming.yaml.gz | Bin 0 -> 3179 bytes .../tests/cassettes/test_citations.yaml.gz | Bin 0 -> 3388 bytes .../integration_tests/test_chat_models.py | 278 +++++++++-- .../__snapshots__/test_standard.ambr | 1 + .../tests/unit_tests/test_chat_models.py | 181 +++++++- 33 files changed, 2123 insertions(+), 263 deletions(-) delete mode 100644 libs/core/langchain_core/messages/block_translators/amazon/__init__.py rename libs/core/langchain_core/messages/block_translators/{amazon => }/bedrock.py (55%) rename libs/core/langchain_core/messages/block_translators/{amazon => }/bedrock_converse.py (54%) delete mode 100644 libs/core/langchain_core/messages/block_translators/chroma.py delete mode 100644 libs/core/langchain_core/messages/block_translators/google/__init__.py rename libs/core/langchain_core/messages/block_translators/{google/genai.py => google_genai.py} (60%) rename libs/core/langchain_core/messages/block_translators/{google/vertexai.py => google_vertexai.py} (59%) rename libs/core/langchain_core/messages/block_translators/{langchain.py => langchain_v0.py} (89%) create mode 100644 libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py create mode 100644 libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py create mode 100644 libs/core/tests/unit_tests/messages/block_translators/test_registration.py create mode 100644 libs/partners/anthropic/langchain_anthropic/_compat.py create mode 100644 libs/partners/anthropic/tests/cassettes/test_agent_loop.yaml.gz create mode 100644 libs/partners/anthropic/tests/cassettes/test_agent_loop_streaming.yaml.gz create mode 100644 libs/partners/anthropic/tests/cassettes/test_citations.yaml.gz diff --git a/libs/core/langchain_core/language_models/_utils.py b/libs/core/langchain_core/language_models/_utils.py index 94680674e3a..cb80fedb3dd 100644 --- a/libs/core/langchain_core/language_models/_utils.py +++ b/libs/core/langchain_core/language_models/_utils.py @@ -212,7 +212,7 @@ def _normalize_messages( } """ - from langchain_core.messages.block_translators.langchain import ( + from langchain_core.messages.block_translators.langchain_v0 import ( _convert_legacy_v0_content_block_to_v1, _convert_openai_format_to_data_block, ) diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py index cfd648c2d0c..33331e512eb 100644 --- a/libs/core/langchain_core/language_models/chat_models.py +++ b/libs/core/langchain_core/language_models/chat_models.py @@ -124,7 +124,7 @@ def _format_for_tracing(messages: list[BaseMessage]) -> list[BaseMessage]: if ( block.get("type") == "image" and is_data_content_block(block) - and block.get("source_type") != "id" + and not ("file_id" in block or block.get("source_type") == "id") ): if message_to_trace is message: # Shallow copy diff --git a/libs/core/langchain_core/messages/ai.py b/libs/core/langchain_core/messages/ai.py index 8fd48c5027e..31be4dbca4e 100644 --- a/libs/core/langchain_core/messages/ai.py +++ b/libs/core/langchain_core/messages/ai.py @@ -231,7 +231,10 @@ class AIMessage(BaseMessage): translator = get_translator(model_provider) if translator: - return translator["translate_content"](self) + try: + return translator["translate_content_chunk"](self) + except NotImplementedError: + pass # Otherwise, use best-effort parsing blocks = super().content_blocks @@ -380,7 +383,10 @@ class AIMessageChunk(AIMessage, BaseMessageChunk): translator = get_translator(model_provider) if translator: - return translator["translate_content_chunk"](self) + try: + return translator["translate_content_chunk"](self) + except NotImplementedError: + pass # Otherwise, use best-effort parsing blocks = super().content_blocks diff --git a/libs/core/langchain_core/messages/base.py b/libs/core/langchain_core/messages/base.py index 3452740b46e..89008c8c429 100644 --- a/libs/core/langchain_core/messages/base.py +++ b/libs/core/langchain_core/messages/base.py @@ -7,13 +7,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union, cast, overload from pydantic import ConfigDict, Field from langchain_core.load.serializable import Serializable -from langchain_core.messages.block_translators.langchain import ( - _convert_legacy_v0_content_block_to_v1, - _convert_v0_multimodal_input_to_v1, -) -from langchain_core.messages.block_translators.openai import ( - _convert_to_v1_from_chat_completions_input, -) +from langchain_core.messages import content as types from langchain_core.utils import get_bolded_text from langchain_core.utils._merge import merge_dicts, merge_lists from langchain_core.utils.interactive_env import is_interactive_env @@ -21,7 +15,6 @@ from langchain_core.utils.interactive_env import is_interactive_env if TYPE_CHECKING: from collections.abc import Sequence - from langchain_core.messages import content as types from langchain_core.prompts.chat import ChatPromptTemplate @@ -129,6 +122,15 @@ class BaseMessage(Serializable): """ from langchain_core.messages import content as types + from langchain_core.messages.block_translators.anthropic import ( + _convert_to_v1_from_anthropic_input, + ) + from langchain_core.messages.block_translators.langchain_v0 import ( + _convert_v0_multimodal_input_to_v1, + ) + from langchain_core.messages.block_translators.openai import ( + _convert_to_v1_from_chat_completions_input, + ) blocks: list[types.ContentBlock] = [] @@ -143,26 +145,19 @@ class BaseMessage(Serializable): blocks.append({"type": "text", "text": item}) elif isinstance(item, dict): item_type = item.get("type") - # Try to convert potential v0 format first - converted_block = _convert_legacy_v0_content_block_to_v1(item) - if converted_block is not item: # Conversion happened - blocks.append(cast("types.ContentBlock", converted_block)) - elif item_type is None or item_type not in types.KNOWN_BLOCK_TYPES: - blocks.append( - cast( - "types.ContentBlock", - {"type": "non_standard", "value": item}, - ) - ) + if item_type not in types.KNOWN_BLOCK_TYPES: + blocks.append({"type": "non_standard", "value": item}) else: blocks.append(cast("types.ContentBlock", item)) # Subsequent passes: attempt to unpack non-standard blocks - blocks = _convert_v0_multimodal_input_to_v1(blocks) - # blocks = _convert_to_v1_from_anthropic_input(blocks) - # ... - - return _convert_to_v1_from_chat_completions_input(blocks) + for parsing_step in [ + _convert_v0_multimodal_input_to_v1, + _convert_to_v1_from_chat_completions_input, + _convert_to_v1_from_anthropic_input, + ]: + blocks = parsing_step(blocks) + return blocks def text(self) -> str: """Get the text content of the message. diff --git a/libs/core/langchain_core/messages/block_translators/__init__.py b/libs/core/langchain_core/messages/block_translators/__init__.py index ff58558713d..bb9673a7c37 100644 --- a/libs/core/langchain_core/messages/block_translators/__init__.py +++ b/libs/core/langchain_core/messages/block_translators/__init__.py @@ -45,37 +45,45 @@ def get_translator( return PROVIDER_TRANSLATORS.get(provider) -def _auto_register_translators() -> None: - """Automatically register all available block translators.""" - import contextlib - import importlib - import pkgutil - from pathlib import Path +def _register_translators() -> None: + """Register all translators in langchain-core. - package_path = Path(__file__).parent + A unit test ensures all modules in ``block_translators`` are represented here. - # Discover all sub-modules - for module_info in pkgutil.iter_modules([str(package_path)]): - module_name = module_info.name + For translators implemented outside langchain-core, they can be registered by + calling ``register_translator`` from within the integration package. + """ + from langchain_core.messages.block_translators.anthropic import ( + _register_anthropic_translator, + ) + from langchain_core.messages.block_translators.bedrock import ( + _register_bedrock_translator, + ) + from langchain_core.messages.block_translators.bedrock_converse import ( + _register_bedrock_converse_translator, + ) + from langchain_core.messages.block_translators.google_genai import ( + _register_google_genai_translator, + ) + from langchain_core.messages.block_translators.google_vertexai import ( + _register_google_vertexai_translator, + ) + from langchain_core.messages.block_translators.groq import _register_groq_translator + from langchain_core.messages.block_translators.ollama import ( + _register_ollama_translator, + ) + from langchain_core.messages.block_translators.openai import ( + _register_openai_translator, + ) - # Skip the __init__ module and any private modules - if module_name.startswith("_"): - continue - - if module_info.ispkg: - # For subpackages, discover their submodules - subpackage_path = package_path / module_name - for submodule_info in pkgutil.iter_modules([str(subpackage_path)]): - submodule_name = submodule_info.name - if not submodule_name.startswith("_"): - with contextlib.suppress(ImportError, AttributeError): - importlib.import_module( - f".{module_name}.{submodule_name}", package=__name__ - ) - else: - # Import top-level translator modules - with contextlib.suppress(ImportError, AttributeError): - importlib.import_module(f".{module_name}", package=__name__) + _register_bedrock_translator() + _register_bedrock_converse_translator() + _register_anthropic_translator() + _register_google_genai_translator() + _register_google_vertexai_translator() + _register_groq_translator() + _register_ollama_translator() + _register_openai_translator() -_auto_register_translators() +_register_translators() diff --git a/libs/core/langchain_core/messages/block_translators/amazon/__init__.py b/libs/core/langchain_core/messages/block_translators/amazon/__init__.py deleted file mode 100644 index 1fbfad4912d..00000000000 --- a/libs/core/langchain_core/messages/block_translators/amazon/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Derivations of standard content blocks from Amazon content.""" diff --git a/libs/core/langchain_core/messages/block_translators/anthropic.py b/libs/core/langchain_core/messages/block_translators/anthropic.py index 469b3812a57..8f0b3919fa4 100644 --- a/libs/core/langchain_core/messages/block_translators/anthropic.py +++ b/libs/core/langchain_core/messages/block_translators/anthropic.py @@ -1,17 +1,423 @@ """Derivations of standard content blocks from Anthropic content.""" +import json +from collections.abc import Iterable +from typing import Any, cast + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +def _populate_extras( + standard_block: types.ContentBlock, block: dict[str, Any], known_fields: set[str] +) -> types.ContentBlock: + """Mutate a block, populating extras.""" + if standard_block.get("type") == "non_standard": + return standard_block + + for key, value in block.items(): + if key not in known_fields: + if "extras" not in block: + # Below type-ignores are because mypy thinks a non-standard block can + # get here, although we exclude them above. + standard_block["extras"] = {} # type: ignore[typeddict-unknown-key] + standard_block["extras"][key] = value # type: ignore[typeddict-item] + + return standard_block + + +def _convert_to_v1_from_anthropic_input( + content: list[types.ContentBlock], +) -> list[types.ContentBlock]: + """Attempt to unpack non-standard blocks.""" + + def _iter_blocks() -> Iterable[types.ContentBlock]: + blocks: list[dict[str, Any]] = [ + cast("dict[str, Any]", block) + if block.get("type") != "non_standard" + else block["value"] # type: ignore[typeddict-item] # this is only non-standard blocks + for block in content + ] + for block in blocks: + block_type = block.get("type") + + if ( + block_type == "document" + and "source" in block + and "type" in block["source"] + ): + if block["source"]["type"] == "base64": + file_block: types.FileContentBlock = { + "type": "file", + "base64": block["source"]["data"], + "mime_type": block["source"]["media_type"], + } + _populate_extras(file_block, block, {"type", "source"}) + yield file_block + + elif block["source"]["type"] == "url": + file_block = { + "type": "file", + "url": block["source"]["url"], + } + _populate_extras(file_block, block, {"type", "source"}) + yield file_block + + elif block["source"]["type"] == "file": + file_block = { + "type": "file", + "id": block["source"]["file_id"], + } + _populate_extras(file_block, block, {"type", "source"}) + yield file_block + + elif block["source"]["type"] == "text": + plain_text_block: types.PlainTextContentBlock = { + "type": "text-plain", + "text": block["source"]["data"], + "mime_type": block.get("media_type", "text/plain"), + } + _populate_extras(plain_text_block, block, {"type", "source"}) + yield plain_text_block + + else: + yield {"type": "non_standard", "value": block} + + elif ( + block_type == "image" + and "source" in block + and "type" in block["source"] + ): + if block["source"]["type"] == "base64": + image_block: types.ImageContentBlock = { + "type": "image", + "base64": block["source"]["data"], + "mime_type": block["source"]["media_type"], + } + _populate_extras(image_block, block, {"type", "source"}) + yield image_block + + elif block["source"]["type"] == "url": + image_block = { + "type": "image", + "url": block["source"]["url"], + } + _populate_extras(image_block, block, {"type", "source"}) + yield image_block + + elif block["source"]["type"] == "file": + image_block = { + "type": "image", + "id": block["source"]["file_id"], + } + _populate_extras(image_block, block, {"type", "source"}) + yield image_block + + else: + yield {"type": "non_standard", "value": block} + + elif block_type in types.KNOWN_BLOCK_TYPES: + yield cast("types.ContentBlock", block) + + else: + yield {"type": "non_standard", "value": block} + + return list(_iter_blocks()) + + +def _convert_citation_to_v1(citation: dict[str, Any]) -> types.Annotation: + citation_type = citation.get("type") + + if citation_type == "web_search_result_location": + url_citation: types.Citation = { + "type": "citation", + "cited_text": citation["cited_text"], + "url": citation["url"], + } + if title := citation.get("title"): + url_citation["title"] = title + known_fields = {"type", "cited_text", "url", "title", "index", "extras"} + for key, value in citation.items(): + if key not in known_fields: + if "extras" not in url_citation: + url_citation["extras"] = {} + url_citation["extras"][key] = value + + return url_citation + + if citation_type in ( + "char_location", + "content_block_location", + "page_location", + "search_result_location", + ): + document_citation: types.Citation = { + "type": "citation", + "cited_text": citation["cited_text"], + } + if "document_title" in citation: + document_citation["title"] = citation["document_title"] + elif title := citation.get("title"): + document_citation["title"] = title + else: + pass + known_fields = { + "type", + "cited_text", + "document_title", + "title", + "index", + "extras", + } + for key, value in citation.items(): + if key not in known_fields: + if "extras" not in document_citation: + document_citation["extras"] = {} + document_citation["extras"][key] = value + + return document_citation + + return { + "type": "non_standard_annotation", + "value": citation, + } + + +def _convert_to_v1_from_anthropic(message: AIMessage) -> list[types.ContentBlock]: + """Convert Anthropic message content to v1 format.""" + if isinstance(message.content, str): + message.content = [{"type": "text", "text": message.content}] + + def _iter_blocks() -> Iterable[types.ContentBlock]: + for block in message.content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + + if block_type == "text": + if citations := block.get("citations"): + text_block: types.TextContentBlock = { + "type": "text", + "text": block.get("text", ""), + "annotations": [_convert_citation_to_v1(a) for a in citations], + } + else: + text_block = {"type": "text", "text": block["text"]} + if "index" in block: + text_block["index"] = block["index"] + yield text_block + + elif block_type == "thinking": + reasoning_block: types.ReasoningContentBlock = { + "type": "reasoning", + "reasoning": block.get("thinking", ""), + } + if "index" in block: + reasoning_block["index"] = block["index"] + known_fields = {"type", "thinking", "index", "extras"} + for key in block: + if key not in known_fields: + if "extras" not in reasoning_block: + reasoning_block["extras"] = {} + reasoning_block["extras"][key] = block[key] + yield reasoning_block + + elif block_type == "tool_use": + if ( + isinstance(message, AIMessageChunk) + and len(message.tool_call_chunks) == 1 + ): + tool_call_chunk: types.ToolCallChunk = ( + message.tool_call_chunks[0].copy() # type: ignore[assignment] + ) + if "type" not in tool_call_chunk: + tool_call_chunk["type"] = "tool_call_chunk" + yield tool_call_chunk + elif ( + not isinstance(message, AIMessageChunk) + and len(message.tool_calls) == 1 + ): + tool_call_block = message.tool_calls[0] + if "index" in block: + tool_call_block["index"] = block["index"] + yield tool_call_block + else: + tool_call_block = { + "type": "tool_call", + "name": block.get("name", ""), + "args": block.get("input", {}), + "id": block.get("id", ""), + } + yield tool_call_block + + elif ( + block_type == "input_json_delta" + and isinstance(message, AIMessageChunk) + and len(message.tool_call_chunks) == 1 + ): + tool_call_chunk = ( + message.tool_call_chunks[0].copy() # type: ignore[assignment] + ) + if "type" not in tool_call_chunk: + tool_call_chunk["type"] = "tool_call_chunk" + yield tool_call_chunk + + elif block_type == "server_tool_use": + if block.get("name") == "web_search": + web_search_call: types.WebSearchCall = {"type": "web_search_call"} + + if query := block.get("input", {}).get("query"): + web_search_call["query"] = query + + elif block.get("input") == {} and "partial_json" in block: + try: + input_ = json.loads(block["partial_json"]) + if isinstance(input_, dict) and "query" in input_: + web_search_call["query"] = input_["query"] + except json.JSONDecodeError: + pass + + if "id" in block: + web_search_call["id"] = block["id"] + if "index" in block: + web_search_call["index"] = block["index"] + known_fields = {"type", "name", "input", "id", "index"} + for key, value in block.items(): + if key not in known_fields: + if "extras" not in web_search_call: + web_search_call["extras"] = {} + web_search_call["extras"][key] = value + yield web_search_call + + elif block.get("name") == "code_execution": + code_interpreter_call: types.CodeInterpreterCall = { + "type": "code_interpreter_call" + } + + if code := block.get("input", {}).get("code"): + code_interpreter_call["code"] = code + + elif block.get("input") == {} and "partial_json" in block: + try: + input_ = json.loads(block["partial_json"]) + if isinstance(input_, dict) and "code" in input_: + code_interpreter_call["code"] = input_["code"] + except json.JSONDecodeError: + pass + + if "id" in block: + code_interpreter_call["id"] = block["id"] + if "index" in block: + code_interpreter_call["index"] = block["index"] + known_fields = {"type", "name", "input", "id", "index"} + for key, value in block.items(): + if key not in known_fields: + if "extras" not in code_interpreter_call: + code_interpreter_call["extras"] = {} + code_interpreter_call["extras"][key] = value + yield code_interpreter_call + + else: + new_block: types.NonStandardContentBlock = { + "type": "non_standard", + "value": block, + } + if "index" in new_block["value"]: + new_block["index"] = new_block["value"].pop("index") + yield new_block + + elif block_type == "web_search_tool_result": + web_search_result: types.WebSearchResult = {"type": "web_search_result"} + if "tool_use_id" in block: + web_search_result["id"] = block["tool_use_id"] + if "index" in block: + web_search_result["index"] = block["index"] + + if web_search_result_content := block.get("content", []): + if "extras" not in web_search_result: + web_search_result["extras"] = {} + urls = [] + extra_content = [] + for result_content in web_search_result_content: + if isinstance(result_content, dict): + if "url" in result_content: + urls.append(result_content["url"]) + extra_content.append(result_content) + web_search_result["extras"]["content"] = extra_content + if urls: + web_search_result["urls"] = urls + yield web_search_result + + elif block_type == "code_execution_tool_result": + code_interpreter_result: types.CodeInterpreterResult = { + "type": "code_interpreter_result", + "output": [], + } + if "tool_use_id" in block: + code_interpreter_result["id"] = block["tool_use_id"] + if "index" in block: + code_interpreter_result["index"] = block["index"] + + code_interpreter_output: types.CodeInterpreterOutput = { + "type": "code_interpreter_output" + } + + code_execution_content = block.get("content", {}) + if code_execution_content.get("type") == "code_execution_result": + if "return_code" in code_execution_content: + code_interpreter_output["return_code"] = code_execution_content[ + "return_code" + ] + if "stdout" in code_execution_content: + code_interpreter_output["stdout"] = code_execution_content[ + "stdout" + ] + if stderr := code_execution_content.get("stderr"): + code_interpreter_output["stderr"] = stderr + if ( + output := code_interpreter_output.get("content") + ) and isinstance(output, list): + if "extras" not in code_interpreter_result: + code_interpreter_result["extras"] = {} + code_interpreter_result["extras"]["content"] = output + for output_block in output: + if "file_id" in output_block: + if "file_ids" not in code_interpreter_output: + code_interpreter_output["file_ids"] = [] + code_interpreter_output["file_ids"].append( + output_block["file_id"] + ) + code_interpreter_result["output"].append(code_interpreter_output) + + elif ( + code_execution_content.get("type") + == "code_execution_tool_result_error" + ): + if "extras" not in code_interpreter_result: + code_interpreter_result["extras"] = {} + code_interpreter_result["extras"]["error_code"] = ( + code_execution_content.get("error_code") + ) + + yield code_interpreter_result + + else: + new_block = {"type": "non_standard", "value": block} + if "index" in new_block["value"]: + new_block["index"] = new_block["value"].pop("index") + yield new_block + + return list(_iter_blocks()) + + def translate_content(message: AIMessage) -> list[types.ContentBlock]: - """Derive standard content blocks from a message with Anthropic content.""" - raise NotImplementedError + """Derive standard content blocks from a message with OpenAI content.""" + return _convert_to_v1_from_anthropic(message) def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: - """Derive standard content blocks from a message chunk with Anthropic content.""" - raise NotImplementedError + """Derive standard content blocks from a message chunk with OpenAI content.""" + return _convert_to_v1_from_anthropic(message) def _register_anthropic_translator() -> None: diff --git a/libs/core/langchain_core/messages/block_translators/amazon/bedrock.py b/libs/core/langchain_core/messages/block_translators/bedrock.py similarity index 55% rename from libs/core/langchain_core/messages/block_translators/amazon/bedrock.py rename to libs/core/langchain_core/messages/block_translators/bedrock.py index 76467152b10..796d45336b1 100644 --- a/libs/core/langchain_core/messages/block_translators/amazon/bedrock.py +++ b/libs/core/langchain_core/messages/block_translators/bedrock.py @@ -1,16 +1,34 @@ """Derivations of standard content blocks from Amazon (Bedrock) content.""" +import warnings + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +WARNED = False -def translate_content(message: AIMessage) -> list[types.ContentBlock]: + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message with Bedrock content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a chunk with Bedrock content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError @@ -21,9 +39,7 @@ def _register_bedrock_translator() -> None: """ from langchain_core.messages.block_translators import register_translator - register_translator( - "amazon_bedrock_chat", translate_content, translate_content_chunk - ) + register_translator("bedrock", translate_content, translate_content_chunk) _register_bedrock_translator() diff --git a/libs/core/langchain_core/messages/block_translators/amazon/bedrock_converse.py b/libs/core/langchain_core/messages/block_translators/bedrock_converse.py similarity index 54% rename from libs/core/langchain_core/messages/block_translators/amazon/bedrock_converse.py rename to libs/core/langchain_core/messages/block_translators/bedrock_converse.py index 5882ef2583b..6249c9107a9 100644 --- a/libs/core/langchain_core/messages/block_translators/amazon/bedrock_converse.py +++ b/libs/core/langchain_core/messages/block_translators/bedrock_converse.py @@ -1,16 +1,36 @@ """Derivations of standard content blocks from Amazon (Bedrock Converse) content.""" +import warnings + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +WARNED = False -def translate_content(message: AIMessage) -> list[types.ContentBlock]: + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message with Bedrock Converse content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock " + "Converse." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a chunk with Bedrock Converse content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Bedrock " + "Converse." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError @@ -21,9 +41,7 @@ def _register_bedrock_converse_translator() -> None: """ from langchain_core.messages.block_translators import register_translator - register_translator( - "amazon_bedrock_converse_chat", translate_content, translate_content_chunk - ) + register_translator("bedrock_converse", translate_content, translate_content_chunk) _register_bedrock_converse_translator() diff --git a/libs/core/langchain_core/messages/block_translators/chroma.py b/libs/core/langchain_core/messages/block_translators/chroma.py deleted file mode 100644 index 652aa8d0e1b..00000000000 --- a/libs/core/langchain_core/messages/block_translators/chroma.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Derivations of standard content blocks from Chroma content.""" - -from langchain_core.messages import AIMessage, AIMessageChunk -from langchain_core.messages import content as types - - -def translate_content(message: AIMessage) -> list[types.ContentBlock]: - """Derive standard content blocks from a message with Chroma content.""" - raise NotImplementedError - - -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: - """Derive standard content blocks from a message chunk with Chroma content.""" - raise NotImplementedError - - -def _register_chroma_translator() -> None: - """Register the Chroma translator with the central registry. - - Run automatically when the module is imported. - """ - from langchain_core.messages.block_translators import register_translator - - register_translator("chroma", translate_content, translate_content_chunk) - - -_register_chroma_translator() diff --git a/libs/core/langchain_core/messages/block_translators/google/__init__.py b/libs/core/langchain_core/messages/block_translators/google/__init__.py deleted file mode 100644 index 0c3f0698aa2..00000000000 --- a/libs/core/langchain_core/messages/block_translators/google/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Derivations of standard content blocks from Google content.""" diff --git a/libs/core/langchain_core/messages/block_translators/google/genai.py b/libs/core/langchain_core/messages/block_translators/google_genai.py similarity index 60% rename from libs/core/langchain_core/messages/block_translators/google/genai.py rename to libs/core/langchain_core/messages/block_translators/google_genai.py index b9761f94bc4..bd4de65c3b0 100644 --- a/libs/core/langchain_core/messages/block_translators/google/genai.py +++ b/libs/core/langchain_core/messages/block_translators/google_genai.py @@ -1,16 +1,34 @@ """Derivations of standard content blocks from Google (GenAI) content.""" +import warnings + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +WARNED = False -def translate_content(message: AIMessage) -> list[types.ContentBlock]: + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message with Google (GenAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google GenAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a chunk with Google (GenAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google GenAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError diff --git a/libs/core/langchain_core/messages/block_translators/google/vertexai.py b/libs/core/langchain_core/messages/block_translators/google_vertexai.py similarity index 59% rename from libs/core/langchain_core/messages/block_translators/google/vertexai.py rename to libs/core/langchain_core/messages/block_translators/google_vertexai.py index ae51fd4065d..e49ee384058 100644 --- a/libs/core/langchain_core/messages/block_translators/google/vertexai.py +++ b/libs/core/langchain_core/messages/block_translators/google_vertexai.py @@ -1,16 +1,36 @@ """Derivations of standard content blocks from Google (VertexAI) content.""" +import warnings + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +WARNED = False -def translate_content(message: AIMessage) -> list[types.ContentBlock]: + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message with Google (VertexAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google " + "VertexAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a chunk with Google (VertexAI) content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Google " + "VertexAI." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError diff --git a/libs/core/langchain_core/messages/block_translators/groq.py b/libs/core/langchain_core/messages/block_translators/groq.py index 4b01dfb017f..6a96b1775f4 100644 --- a/libs/core/langchain_core/messages/block_translators/groq.py +++ b/libs/core/langchain_core/messages/block_translators/groq.py @@ -1,16 +1,34 @@ """Derivations of standard content blocks from Groq content.""" +import warnings + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +WARNED = False -def translate_content(message: AIMessage) -> list[types.ContentBlock]: + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message with Groq content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Groq." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message chunk with Groq content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Groq." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError diff --git a/libs/core/langchain_core/messages/block_translators/langchain.py b/libs/core/langchain_core/messages/block_translators/langchain_v0.py similarity index 89% rename from libs/core/langchain_core/messages/block_translators/langchain.py rename to libs/core/langchain_core/messages/block_translators/langchain_v0.py index 4b5e4479835..5fde4c0fcb0 100644 --- a/libs/core/langchain_core/messages/block_translators/langchain.py +++ b/libs/core/langchain_core/messages/block_translators/langchain_v0.py @@ -1,4 +1,4 @@ -"""Derivations of standard content blocks from LangChain content.""" +"""Derivations of standard content blocks from LangChain v0 multimodal content.""" from typing import Any, Union, cast @@ -21,26 +21,20 @@ def _convert_v0_multimodal_input_to_v1( Updated list with v0 blocks converted to v1 format. """ converted_blocks = [] - for block in blocks: - if ( - isinstance(block, dict) - and block.get("type") == "non_standard" - and "value" in block - and isinstance(block["value"], dict) # type: ignore[typeddict-item] - ): - # We know this is a NonStandardContentBlock, so we can safely access value - value = cast("Any", block)["value"] - # Check if this looks like v0 format - if ( - value.get("type") in {"image", "audio", "file"} - and "source_type" in value - ): - converted_block = _convert_legacy_v0_content_block_to_v1(value) - converted_blocks.append(cast("types.ContentBlock", converted_block)) - else: - converted_blocks.append(block) + unpacked_blocks: list[dict[str, Any]] = [ + cast("dict[str, Any]", block) + if block.get("type") != "non_standard" + else block["value"] # type: ignore[typeddict-item] # this is only non-standard blocks + for block in blocks + ] + for block in unpacked_blocks: + if block.get("type") in {"image", "audio", "file"} and "source_type" in block: + converted_block = _convert_legacy_v0_content_block_to_v1(block) + converted_blocks.append(cast("types.ContentBlock", converted_block)) + elif block.get("type") in types.KNOWN_BLOCK_TYPES: + converted_blocks.append(cast("types.ContentBlock", block)) else: - converted_blocks.append(block) + converted_blocks.append({"type": "non_standard", "value": block}) return converted_blocks @@ -213,7 +207,7 @@ def _convert_openai_format_to_data_block( return types.create_image_block( # Even though this is labeled as `url`, it can be base64-encoded - base64=block["image_url"]["url"], + base64=parsed["data"], mime_type=parsed["mime_type"], **all_extras, ) @@ -278,9 +272,7 @@ def _convert_openai_format_to_data_block( ) # base64-style file block - if (block["type"] == "file") and ( - parsed := _parse_data_uri(block["file"]["file_data"]) - ): + if block["type"] == "file": known_keys = {"type", "file"} extras = _extract_extras(block, known_keys) @@ -291,11 +283,10 @@ def _convert_openai_format_to_data_block( for key, value in file_extras.items(): all_extras[f"file_{key}"] = value - mime_type = parsed["mime_type"] filename = block["file"].get("filename") return types.create_file_block( base64=block["file"]["file_data"], - mime_type=mime_type, + mime_type="application/pdf", filename=filename, **all_extras, ) diff --git a/libs/core/langchain_core/messages/block_translators/ollama.py b/libs/core/langchain_core/messages/block_translators/ollama.py index a0f41ab7634..736ecfe0651 100644 --- a/libs/core/langchain_core/messages/block_translators/ollama.py +++ b/libs/core/langchain_core/messages/block_translators/ollama.py @@ -1,16 +1,34 @@ """Derivations of standard content blocks from Ollama content.""" +import warnings + from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types +WARNED = False -def translate_content(message: AIMessage) -> list[types.ContentBlock]: + +def translate_content(message: AIMessage) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message with Ollama content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Ollama." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError -def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: +def translate_content_chunk(message: AIMessageChunk) -> list[types.ContentBlock]: # noqa: ARG001 """Derive standard content blocks from a message chunk with Ollama content.""" + global WARNED # noqa: PLW0603 + if not WARNED: + warning_message = ( + "Content block standardization is not yet fully supported for Ollama." + ) + warnings.warn(warning_message, stacklevel=2) + WARNED = True raise NotImplementedError diff --git a/libs/core/langchain_core/messages/block_translators/openai.py b/libs/core/langchain_core/messages/block_translators/openai.py index 029757563b4..b11e64558aa 100644 --- a/libs/core/langchain_core/messages/block_translators/openai.py +++ b/libs/core/langchain_core/messages/block_translators/openai.py @@ -9,7 +9,7 @@ from langchain_core.language_models._utils import ( _is_openai_data_block, ) from langchain_core.messages import content as types -from langchain_core.messages.block_translators.langchain import ( +from langchain_core.messages.block_translators.langchain_v0 import ( _convert_openai_format_to_data_block, ) @@ -52,34 +52,31 @@ def _convert_to_v1_from_chat_completions_input( from langchain_core.messages import content as types converted_blocks = [] - for block in blocks: - if ( - isinstance(block, dict) - and block.get("type") == "non_standard" - and "value" in block - and isinstance(block["value"], dict) # type: ignore[typeddict-item] - ): - # We know this is a NonStandardContentBlock, so we can safely access value - value = cast("Any", block)["value"] - # Check if this looks like OpenAI format - if value.get("type") in { - "image_url", - "input_audio", - "file", - } and _is_openai_data_block(value): - converted_block = _convert_openai_format_to_data_block(value) - # If conversion succeeded, use it; otherwise keep as non_standard - if ( - isinstance(converted_block, dict) - and converted_block.get("type") in types.KNOWN_BLOCK_TYPES - ): - converted_blocks.append(cast("types.ContentBlock", converted_block)) - else: - converted_blocks.append(block) + unpacked_blocks: list[dict[str, Any]] = [ + cast("dict[str, Any]", block) + if block.get("type") != "non_standard" + else block["value"] # type: ignore[typeddict-item] # this is only non-standard blocks + for block in blocks + ] + for block in unpacked_blocks: + if block.get("type") in { + "image_url", + "input_audio", + "file", + } and _is_openai_data_block(block): + converted_block = _convert_openai_format_to_data_block(block) + # If conversion succeeded, use it; otherwise keep as non_standard + if ( + isinstance(converted_block, dict) + and converted_block.get("type") in types.KNOWN_BLOCK_TYPES + ): + converted_blocks.append(cast("types.ContentBlock", converted_block)) else: - converted_blocks.append(block) + converted_blocks.append({"type": "non_standard", "value": block}) + elif block.get("type") in types.KNOWN_BLOCK_TYPES: + converted_blocks.append(cast("types.ContentBlock", block)) else: - converted_blocks.append(block) + converted_blocks.append({"type": "non_standard", "value": block}) return converted_blocks diff --git a/libs/core/langchain_core/messages/content.py b/libs/core/langchain_core/messages/content.py index 83287fb06c8..845c3b481ce 100644 --- a/libs/core/langchain_core/messages/content.py +++ b/libs/core/langchain_core/messages/content.py @@ -503,12 +503,6 @@ class CodeInterpreterOutput(TypedDict): file_ids: NotRequired[list[str]] """List of file IDs generated by the code interpreter.""" - index: NotRequired[Union[int, str]] - """Index of block in aggregate response. Used during streaming.""" - - extras: NotRequired[dict[str, Any]] - """Provider-specific metadata.""" - class CodeInterpreterResult(TypedDict): """Result of a code interpreter tool call.""" @@ -886,7 +880,6 @@ ToolContentBlock = Union[ ToolCall, ToolCallChunk, CodeInterpreterCall, - CodeInterpreterOutput, CodeInterpreterResult, WebSearchCall, WebSearchResult, @@ -918,7 +911,6 @@ KNOWN_BLOCK_TYPES = { "video", # Server-side tool calls "code_interpreter_call", - "code_interpreter_output", "code_interpreter_result", "web_search_call", "web_search_result", diff --git a/libs/core/langchain_core/utils/_merge.py b/libs/core/langchain_core/utils/_merge.py index c32b09e2e66..7b8465e8d02 100644 --- a/libs/core/langchain_core/utils/_merge.py +++ b/libs/core/langchain_core/utils/_merge.py @@ -116,11 +116,35 @@ def merge_lists(left: Optional[list], *others: Optional[list]) -> Optional[list] if to_merge: # TODO: Remove this once merge_dict is updated with special # handling for 'type'. - new_e = ( - {k: v for k, v in e.items() if k != "type"} - if "type" in e - else e - ) + if (left_type := merged[to_merge[0]].get("type")) and ( + e.get("type") == "non_standard" and "value" in e + ): + if left_type != "non_standard": + # standard + non_standard + new_e: dict[str, Any] = { + "extras": { + k: v + for k, v in e["value"].items() + if k != "type" + } + } + else: + # non_standard + non_standard + new_e = { + "value": { + k: v + for k, v in e["value"].items() + if k != "type" + } + } + if "index" in e: + new_e["index"] = e["index"] + else: + new_e = ( + {k: v for k, v in e.items() if k != "type"} + if "type" in e + else e + ) merged[to_merge[0]] = merge_dicts(merged[to_merge[0]], new_e) else: merged.append(e) diff --git a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py index 848fb75091a..22d8bc7907f 100644 --- a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py +++ b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py @@ -621,14 +621,14 @@ def test_extend_support_to_openai_multimodal_formats() -> None: "type": "input_audio", "input_audio": { "format": "wav", - "data": "data:audio/wav;base64,", + "data": "", }, }, { # file-base64 "type": "file", "file": { "filename": "draconomicon.pdf", - "file_data": "data:application/pdf;base64,", + "file_data": "", }, }, { # file-id @@ -643,12 +643,12 @@ def test_extend_support_to_openai_multimodal_formats() -> None: {"type": "text", "text": "Hello"}, # TextContentBlock { # AudioContentBlock "type": "audio", - "base64": "data:audio/wav;base64,", + "base64": "", "mime_type": "audio/wav", }, { # FileContentBlock "type": "file", - "base64": "data:application/pdf;base64,", + "base64": "", "mime_type": "application/pdf", "extras": {"filename": "draconomicon.pdf"}, }, diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py b/libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py new file mode 100644 index 00000000000..e0f65657b99 --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_anthropic.py @@ -0,0 +1,439 @@ +from typing import Optional + +from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage +from langchain_core.messages import content as types + + +def test_convert_to_v1_from_anthropic() -> None: + message = AIMessage( + [ + {"type": "thinking", "thinking": "foo", "signature": "foo_signature"}, + {"type": "text", "text": "Let's call a tool."}, + { + "type": "tool_use", + "id": "abc_123", + "name": "get_weather", + "input": {"location": "San Francisco"}, + }, + { + "type": "text", + "text": "It's sunny.", + "citations": [ + { + "type": "search_result_location", + "cited_text": "The weather is sunny.", + "source": "source_123", + "title": "Document Title", + "search_result_index": 1, + "start_block_index": 0, + "end_block_index": 2, + }, + {"bar": "baz"}, + ], + }, + { + "type": "server_tool_use", + "name": "web_search", + "input": {"query": "web search query"}, + "id": "srvtoolu_abc123", + }, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_abc123", + "content": [ + { + "type": "web_search_result", + "title": "Page Title 1", + "url": "", + "page_age": "January 1, 2025", + "encrypted_content": "", + }, + { + "type": "web_search_result", + "title": "Page Title 2", + "url": "", + "page_age": "January 2, 2025", + "encrypted_content": "", + }, + ], + }, + { + "type": "server_tool_use", + "id": "srvtoolu_def456", + "name": "code_execution", + "input": {"code": "import numpy as np..."}, + }, + { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_def456", + "content": { + "type": "code_execution_result", + "stdout": "Mean: 5.5\nStandard deviation...", + "stderr": "", + "return_code": 0, + }, + }, + {"type": "something_else", "foo": "bar"}, + ], + response_metadata={"model_provider": "anthropic"}, + ) + expected_content: list[types.ContentBlock] = [ + { + "type": "reasoning", + "reasoning": "foo", + "extras": {"signature": "foo_signature"}, + }, + {"type": "text", "text": "Let's call a tool."}, + { + "type": "tool_call", + "id": "abc_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + { + "type": "text", + "text": "It's sunny.", + "annotations": [ + { + "type": "citation", + "title": "Document Title", + "cited_text": "The weather is sunny.", + "extras": { + "source": "source_123", + "search_result_index": 1, + "start_block_index": 0, + "end_block_index": 2, + }, + }, + {"type": "non_standard_annotation", "value": {"bar": "baz"}}, + ], + }, + { + "type": "web_search_call", + "id": "srvtoolu_abc123", + "query": "web search query", + }, + { + "type": "web_search_result", + "id": "srvtoolu_abc123", + "urls": ["", ""], + "extras": { + "content": [ + { + "type": "web_search_result", + "title": "Page Title 1", + "url": "", + "page_age": "January 1, 2025", + "encrypted_content": "", + }, + { + "type": "web_search_result", + "title": "Page Title 2", + "url": "", + "page_age": "January 2, 2025", + "encrypted_content": "", + }, + ] + }, + }, + { + "type": "code_interpreter_call", + "id": "srvtoolu_def456", + "code": "import numpy as np...", + }, + { + "type": "code_interpreter_result", + "id": "srvtoolu_def456", + "output": [ + { + "type": "code_interpreter_output", + "return_code": 0, + "stdout": "Mean: 5.5\nStandard deviation...", + } + ], + }, + { + "type": "non_standard", + "value": {"type": "something_else", "foo": "bar"}, + }, + ] + assert message.content_blocks == expected_content + + # Check no mutation + assert message.content != expected_content + + +def test_convert_to_v1_from_anthropic_chunk() -> None: + chunks = [ + AIMessageChunk( + content=[{"text": "Looking ", "type": "text", "index": 0}], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[{"text": "now.", "type": "text", "index": 0}], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + { + "type": "tool_use", + "name": "get_weather", + "input": {}, + "id": "toolu_abc123", + "index": 1, + } + ], + tool_call_chunks=[ + { + "type": "tool_call_chunk", + "name": "get_weather", + "args": "", + "id": "toolu_abc123", + "index": 1, + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[{"type": "input_json_delta", "partial_json": "", "index": 1}], + tool_call_chunks=[ + { + "name": None, + "args": "", + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + {"type": "input_json_delta", "partial_json": '{"loca', "index": 1} + ], + tool_call_chunks=[ + { + "name": None, + "args": '{"loca', + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + {"type": "input_json_delta", "partial_json": 'tion": "San ', "index": 1} + ], + tool_call_chunks=[ + { + "name": None, + "args": 'tion": "San ', + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + AIMessageChunk( + content=[ + {"type": "input_json_delta", "partial_json": 'Francisco"}', "index": 1} + ], + tool_call_chunks=[ + { + "name": None, + "args": 'Francisco"}', + "id": None, + "index": 1, + "type": "tool_call_chunk", + } + ], + response_metadata={"model_provider": "anthropic"}, + ), + ] + expected_contents: list[types.ContentBlock] = [ + {"type": "text", "text": "Looking ", "index": 0}, + {"type": "text", "text": "now.", "index": 0}, + { + "type": "tool_call_chunk", + "name": "get_weather", + "args": "", + "id": "toolu_abc123", + "index": 1, + }, + {"name": None, "args": "", "id": None, "index": 1, "type": "tool_call_chunk"}, + { + "name": None, + "args": '{"loca', + "id": None, + "index": 1, + "type": "tool_call_chunk", + }, + { + "name": None, + "args": 'tion": "San ', + "id": None, + "index": 1, + "type": "tool_call_chunk", + }, + { + "name": None, + "args": 'Francisco"}', + "id": None, + "index": 1, + "type": "tool_call_chunk", + }, + ] + for chunk, expected in zip(chunks, expected_contents): + assert chunk.content_blocks == [expected] + + full: Optional[AIMessageChunk] = None + for chunk in chunks: + full = chunk if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + + expected_content = [ + {"type": "text", "text": "Looking now.", "index": 0}, + { + "type": "tool_use", + "name": "get_weather", + "partial_json": '{"location": "San Francisco"}', + "input": {}, + "id": "toolu_abc123", + "index": 1, + }, + ] + assert full.content == expected_content + + expected_content_blocks = [ + {"type": "text", "text": "Looking now.", "index": 0}, + { + "type": "tool_call_chunk", + "name": "get_weather", + "args": '{"location": "San Francisco"}', + "id": "toolu_abc123", + "index": 1, + }, + ] + assert full.content_blocks == expected_content_blocks + + +def test_convert_to_v1_from_anthropic_input() -> None: + message = HumanMessage( + [ + {"type": "text", "text": "foo"}, + { + "type": "document", + "source": { + "type": "base64", + "data": "", + "media_type": "application/pdf", + }, + }, + { + "type": "document", + "source": { + "type": "url", + "url": "", + }, + }, + { + "type": "document", + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "The grass is green"}, + {"type": "text", "text": "The sky is blue"}, + ], + }, + "citations": {"enabled": True}, + }, + { + "type": "document", + "source": { + "type": "text", + "data": "", + "media_type": "text/plain", + }, + }, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "", + }, + }, + { + "type": "image", + "source": { + "type": "file", + "file_id": "", + }, + }, + { + "type": "document", + "source": {"type": "file", "file_id": ""}, + }, + ] + ) + + expected: list[types.ContentBlock] = [ + {"type": "text", "text": "foo"}, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + }, + { + "type": "file", + "url": "", + }, + { + "type": "non_standard", + "value": { + "type": "document", + "source": { + "type": "content", + "content": [ + {"type": "text", "text": "The grass is green"}, + {"type": "text", "text": "The sky is blue"}, + ], + }, + "citations": {"enabled": True}, + }, + }, + { + "type": "text-plain", + "text": "", + "mime_type": "text/plain", + }, + { + "type": "image", + "base64": "", + "mime_type": "image/jpeg", + }, + { + "type": "image", + "url": "", + }, + { + "type": "image", + "id": "", + }, + { + "type": "file", + "id": "", + }, + ] + + assert message.content_blocks == expected diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py b/libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py new file mode 100644 index 00000000000..c586f134075 --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_langchain_v0.py @@ -0,0 +1,79 @@ +from langchain_core.messages import HumanMessage +from langchain_core.messages import content as types +from tests.unit_tests.language_models.chat_models.test_base import ( + _content_blocks_equal_ignore_id, +) + + +def test_convert_to_v1_from_openai_input() -> None: + message = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, + { + "type": "image", + "source_type": "url", + "url": "https://example.com/image.png", + }, + { + "type": "image", + "source_type": "base64", + "data": "", + "mime_type": "image/png", + }, + { + "type": "file", + "source_type": "url", + "url": "", + }, + { + "type": "file", + "source_type": "base64", + "data": "", + "mime_type": "application/pdf", + }, + { + "type": "audio", + "source_type": "base64", + "data": "", + "mime_type": "audio/mpeg", + }, + { + "type": "file", + "source_type": "id", + "id": "", + }, + ] + ) + + expected: list[types.ContentBlock] = [ + {"type": "text", "text": "Hello"}, + { + "type": "image", + "url": "https://example.com/image.png", + }, + { + "type": "image", + "base64": "", + "mime_type": "image/png", + }, + { + "type": "file", + "url": "", + }, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + }, + { + "type": "audio", + "base64": "", + "mime_type": "audio/mpeg", + }, + { + "type": "file", + "file_id": "", + }, + ] + + assert _content_blocks_equal_ignore_id(message.content_blocks, expected) diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_openai.py b/libs/core/tests/unit_tests/messages/block_translators/test_openai.py index 3602d9eb08d..2ed2086ea44 100644 --- a/libs/core/tests/unit_tests/messages/block_translators/test_openai.py +++ b/libs/core/tests/unit_tests/messages/block_translators/test_openai.py @@ -1,41 +1,12 @@ from typing import Optional -from langchain_core.language_models.fake_chat_models import ParrotFakeChatModel -from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage from langchain_core.messages import content as types from tests.unit_tests.language_models.chat_models.test_base import ( _content_blocks_equal_ignore_id, ) -def test_v0_to_v1_content_blocks() -> None: - llm = ParrotFakeChatModel() - messages = [ - { - "role": "user", - # v0 format - "content": [ - { - "type": "image", - "source_type": "url", - "url": "https://example.com/image.png", - } - ], - } - ] - response = llm.invoke(messages) - assert len(response.content_blocks) == 1 - expected_content_blocks = [ - { - "type": "image", - "url": "https://example.com/image.png", - } - ] - assert _content_blocks_equal_ignore_id( - response.content_blocks, expected_content_blocks - ) - - def test_convert_to_v1_from_responses() -> None: message = AIMessage( [ @@ -261,3 +232,64 @@ def test_convert_to_v1_from_responses_chunk() -> None: }, ] assert full.content_blocks == expected_content_blocks + + +def test_convert_to_v1_from_openai_input() -> None: + message = HumanMessage( + content=[ + {"type": "text", "text": "Hello"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + }, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."}, + }, + { + "type": "input_audio", + "input_audio": { + "format": "wav", + "data": "", + }, + }, + { + "type": "file", + "file": { + "filename": "draconomicon.pdf", + "file_data": "", + }, + }, + { + "type": "file", + "file": {"file_id": ""}, + }, + ] + ) + + expected: list[types.ContentBlock] = [ + {"type": "text", "text": "Hello"}, + { + "type": "image", + "url": "https://example.com/image.png", + }, + { + "type": "image", + "base64": "/9j/4AAQSkZJRg...", + "mime_type": "image/jpeg", + }, + { + "type": "audio", + "base64": "", + "mime_type": "audio/wav", + }, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + "extras": {"filename": "draconomicon.pdf"}, + }, + {"type": "file", "file_id": ""}, + ] + + assert _content_blocks_equal_ignore_id(message.content_blocks, expected) diff --git a/libs/core/tests/unit_tests/messages/block_translators/test_registration.py b/libs/core/tests/unit_tests/messages/block_translators/test_registration.py new file mode 100644 index 00000000000..74c16d30a24 --- /dev/null +++ b/libs/core/tests/unit_tests/messages/block_translators/test_registration.py @@ -0,0 +1,29 @@ +import pkgutil +from pathlib import Path + +import pytest + +from langchain_core.messages.block_translators import PROVIDER_TRANSLATORS + + +def test_all_providers_registered() -> None: + """Test that all block translators implemented in langchain-core are registered. + + If this test fails, it is likely that a block translator is implemented but not + registered on import. Check that the provider is included in + ``langchain_core.messages.block_translators.__init__._register_translators``. + """ + package_path = ( + Path(__file__).parents[4] / "langchain_core" / "messages" / "block_translators" + ) + + for module_info in pkgutil.iter_modules([str(package_path)]): + module_name = module_info.name + + # Skip the __init__ module, any private modules, and ``langchain_v0``, which is + # only used to parse v0 multimodal inputs. + if module_name.startswith("_") or module_name == "langchain_v0": + continue + + if module_name not in PROVIDER_TRANSLATORS: + pytest.fail(f"Block translator not registered: {module_name}") diff --git a/libs/core/tests/unit_tests/messages/test_ai.py b/libs/core/tests/unit_tests/messages/test_ai.py index 67b0a2dc968..4f623c0910c 100644 --- a/libs/core/tests/unit_tests/messages/test_ai.py +++ b/libs/core/tests/unit_tests/messages/test_ai.py @@ -1,3 +1,7 @@ +from typing import Union, cast + +import pytest + from langchain_core.load import dumpd, load from langchain_core.messages import AIMessage, AIMessageChunk from langchain_core.messages import content as types @@ -310,3 +314,92 @@ def test_content_blocks() -> None: } ] assert message.content == "" + + # Non-standard + standard_content_1: list[types.ContentBlock] = [ + {"type": "non_standard", "index": 0, "value": {"foo": "bar "}} + ] + standard_content_2: list[types.ContentBlock] = [ + {"type": "non_standard", "index": 0, "value": {"foo": "baz"}} + ] + chunk_1 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_1) + ) + chunk_2 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_2) + ) + merged_chunk = chunk_1 + chunk_2 + assert merged_chunk.content == [ + {"type": "non_standard", "index": 0, "value": {"foo": "bar baz"}}, + ] + + # Test non-standard + non-standard + chunk_1 = AIMessageChunk( + content=[ + { + "type": "non_standard", + "index": 0, + "value": {"type": "non_standard_tool", "foo": "bar"}, + } + ] + ) + chunk_2 = AIMessageChunk( + content=[ + { + "type": "non_standard", + "index": 0, + "value": {"type": "input_json_delta", "partial_json": "a"}, + } + ] + ) + chunk_3 = AIMessageChunk( + content=[ + { + "type": "non_standard", + "index": 0, + "value": {"type": "input_json_delta", "partial_json": "b"}, + } + ] + ) + merged_chunk = chunk_1 + chunk_2 + chunk_3 + assert merged_chunk.content == [ + { + "type": "non_standard", + "index": 0, + "value": {"type": "non_standard_tool", "foo": "bar", "partial_json": "ab"}, + } + ] + + # Test standard + non-standard with same index + standard_content_1 = [ + {"type": "web_search_call", "id": "ws_123", "query": "web query", "index": 0} + ] + standard_content_2 = [{"type": "non_standard", "value": {"foo": "bar"}, "index": 0}] + chunk_1 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_1) + ) + chunk_2 = AIMessageChunk( + content=cast("Union[str, list[Union[str, dict]]]", standard_content_2) + ) + merged_chunk = chunk_1 + chunk_2 + assert merged_chunk.content == [ + { + "type": "web_search_call", + "id": "ws_123", + "query": "web query", + "index": 0, + "extras": {"foo": "bar"}, + } + ] + + +def test_provider_warns() -> None: + # Test that major providers warn if content block standardization is not yet + # implemented. + # This test should be removed when all major providers support content block + # standardization. + message = AIMessage("Hello.", response_metadata={"model_provider": "groq"}) + with pytest.warns(match="not yet fully supported for Groq"): + content_blocks = message.content_blocks + + assert content_blocks == [{"type": "text", "text": "Hello."}] diff --git a/libs/partners/anthropic/langchain_anthropic/_compat.py b/libs/partners/anthropic/langchain_anthropic/_compat.py new file mode 100644 index 00000000000..3b904162324 --- /dev/null +++ b/libs/partners/anthropic/langchain_anthropic/_compat.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import json +from typing import Any, Optional, cast + +from langchain_core.messages import content as types + + +def _convert_annotation_from_v1(annotation: types.Annotation) -> dict[str, Any]: + """Right-inverse of _convert_citation_to_v1.""" + if annotation["type"] == "non_standard_annotation": + return annotation["value"] + + if annotation["type"] == "citation": + if "url" in annotation: + # web_search_result_location + out: dict[str, Any] = {} + if cited_text := annotation.get("cited_text"): + out["cited_text"] = cited_text + if "encrypted_index" in annotation.get("extras", {}): + out["encrypted_index"] = annotation["extras"]["encrypted_index"] + if "title" in annotation: + out["title"] = annotation["title"] + out["type"] = "web_search_result_location" + if "url" in annotation: + out["url"] = annotation["url"] + + for key, value in annotation.get("extras", {}).items(): + if key not in out: + out[key] = value + + return out + + if "start_char_index" in annotation.get("extras", {}): + # char_location + out = {"type": "char_location"} + for field in ["cited_text"]: + if value := annotation.get(field): + out[field] = value + if title := annotation.get("title"): + out["document_title"] = title + + for key, value in annotation.get("extras", {}).items(): + out[key] = value + + return out + + if "search_result_index" in annotation.get("extras", {}): + # search_result_location + out = {"type": "search_result_location"} + for field in ["cited_text", "title"]: + if value := annotation.get(field): + out[field] = value + + for key, value in annotation.get("extras", {}).items(): + out[key] = value + + return out + + if "start_block_index" in annotation.get("extras", {}): + # content_block_location + out = {} + if cited_text := annotation.get("cited_text"): + out["cited_text"] = cited_text + if "document_index" in annotation.get("extras", {}): + out["document_index"] = annotation["extras"]["document_index"] + if "title" in annotation: + out["document_title"] = annotation["title"] + + for key, value in annotation.get("extras", {}).items(): + if key not in out: + out[key] = value + + out["type"] = "content_block_location" + return out + + if "start_page_number" in annotation.get("extras", {}): + # page_location + out = {"type": "page_location"} + for field in ["cited_text"]: + if value := annotation.get(field): + out[field] = value + if title := annotation.get("title"): + out["document_title"] = title + + for key, value in annotation.get("extras", {}).items(): + out[key] = value + + return out + + return cast(dict[str, Any], annotation) + + return cast(dict[str, Any], annotation) + + +def _convert_from_v1_to_anthropic( + content: list[types.ContentBlock], + tool_calls: list[types.ToolCall], + model_provider: Optional[str], +) -> list[dict[str, Any]]: + new_content: list = [] + for block in content: + if block["type"] == "text": + if model_provider == "anthropic" and "annotations" in block: + new_block: dict[str, Any] = {"type": "text"} + new_block["citations"] = [ + _convert_annotation_from_v1(a) for a in block["annotations"] + ] + if "text" in block: + new_block["text"] = block["text"] + else: + new_block = {"text": block.get("text", ""), "type": "text"} + new_content.append(new_block) + + elif block["type"] == "tool_call": + new_content.append( + { + "type": "tool_use", + "name": block.get("name", ""), + "input": block.get("args", {}), + "id": block.get("id", ""), + } + ) + + elif block["type"] == "tool_call_chunk": + if isinstance(block["args"], str): + try: + input_ = json.loads(block["args"] or "{}") + except json.JSONDecodeError: + input_ = {} + else: + input_ = block.get("args") or {} + new_content.append( + { + "type": "tool_use", + "name": block.get("name", ""), + "input": input_, + "id": block.get("id", ""), + } + ) + + elif block["type"] == "reasoning" and model_provider == "anthropic": + new_block = {} + if "reasoning" in block: + new_block["thinking"] = block["reasoning"] + new_block["type"] = "thinking" + if signature := block.get("extras", {}).get("signature"): + new_block["signature"] = signature + + new_content.append(new_block) + + elif block["type"] == "web_search_call" and model_provider == "anthropic": + new_block = {} + if "id" in block: + new_block["id"] = block["id"] + + if (query := block.get("query")) and "input" not in block: + new_block["input"] = {"query": query} + elif input_ := block.get("extras", {}).get("input"): + new_block["input"] = input_ + elif partial_json := block.get("extras", {}).get("partial_json"): + new_block["input"] = {} + new_block["partial_json"] = partial_json + else: + pass + new_block["name"] = "web_search" + new_block["type"] = "server_tool_use" + new_content.append(new_block) + + elif block["type"] == "web_search_result" and model_provider == "anthropic": + new_block = {} + if "content" in block.get("extras", {}): + new_block["content"] = block["extras"]["content"] + if "id" in block: + new_block["tool_use_id"] = block["id"] + new_block["type"] = "web_search_tool_result" + new_content.append(new_block) + + elif block["type"] == "code_interpreter_call" and model_provider == "anthropic": + new_block = {} + if "id" in block: + new_block["id"] = block["id"] + if (code := block.get("code")) and "input" not in block: + new_block["input"] = {"code": code} + elif input_ := block.get("extras", {}).get("input"): + new_block["input"] = input_ + elif partial_json := block.get("extras", {}).get("partial_json"): + new_block["input"] = {} + new_block["partial_json"] = partial_json + else: + pass + new_block["name"] = "code_execution" + new_block["type"] = "server_tool_use" + new_content.append(new_block) + + elif ( + block["type"] == "code_interpreter_result" and model_provider == "anthropic" + ): + new_block = {} + if (output := block.get("output", [])) and len(output) == 1: + code_interpreter_output = output[0] + code_execution_content = {} + if "content" in block.get("extras", {}): + code_execution_content["content"] = block["extras"]["content"] + elif (file_ids := block.get("file_ids")) and isinstance(file_ids, list): + code_execution_content["content"] = [ + {"file_id": file_id, "type": "code_execution_output"} + for file_id in file_ids + ] + else: + code_execution_content["content"] = [] + if "return_code" in code_interpreter_output: + code_execution_content["return_code"] = code_interpreter_output[ + "return_code" + ] + code_execution_content["stderr"] = code_interpreter_output.get( + "stderr", "" + ) + if "stdout" in code_interpreter_output: + code_execution_content["stdout"] = code_interpreter_output["stdout"] + code_execution_content["type"] = "code_execution_result" + new_block["content"] = code_execution_content + elif "error_code" in block.get("extras", {}): + code_execution_content = { + "error_code": block["extras"]["error_code"], + "type": "code_execution_tool_result_error", + } + new_block["content"] = code_execution_content + else: + pass + if "id" in block: + new_block["tool_use_id"] = block["id"] + new_block["type"] = "code_execution_tool_result" + new_content.append(new_block) + + elif ( + block["type"] == "non_standard" + and "value" in block + and model_provider == "anthropic" + ): + new_content.append(block["value"]) + else: + new_content.append(block) + + return new_content diff --git a/libs/partners/anthropic/langchain_anthropic/chat_models.py b/libs/partners/anthropic/langchain_anthropic/chat_models.py index 641b630ddb3..b038ba09ff5 100644 --- a/libs/partners/anthropic/langchain_anthropic/chat_models.py +++ b/libs/partners/anthropic/langchain_anthropic/chat_models.py @@ -33,6 +33,7 @@ from langchain_core.messages import ( ToolMessage, is_data_content_block, ) +from langchain_core.messages import content as types from langchain_core.messages.ai import InputTokenDetails, UsageMetadata from langchain_core.messages.tool import tool_call_chunk as create_tool_call_chunk from langchain_core.output_parsers import JsonOutputKeyToolsParser, PydanticToolsParser @@ -51,6 +52,7 @@ from langchain_anthropic._client_utils import ( _get_default_async_httpx_client, _get_default_httpx_client, ) +from langchain_anthropic._compat import _convert_from_v1_to_anthropic from langchain_anthropic.output_parsers import extract_tool_calls _message_type_lookups = { @@ -212,7 +214,7 @@ def _merge_messages( def _format_data_content_block(block: dict) -> dict: """Format standard data content block to format expected by Anthropic.""" if block["type"] == "image": - if block["source_type"] == "url": + if "url" in block: if block["url"].startswith("data:"): # Data URI formatted_block = { @@ -224,16 +226,24 @@ def _format_data_content_block(block: dict) -> dict: "type": "image", "source": {"type": "url", "url": block["url"]}, } - elif block["source_type"] == "base64": + elif "base64" in block or block.get("source_type") == "base64": formatted_block = { "type": "image", "source": { "type": "base64", "media_type": block["mime_type"], - "data": block["data"], + "data": block.get("base64") or block.get("data", ""), }, } - elif block["source_type"] == "id": + elif "file_id" in block: + formatted_block = { + "type": "image", + "source": { + "type": "file", + "file_id": block["file_id"], + }, + } + elif block.get("source_type") == "id": formatted_block = { "type": "image", "source": { @@ -243,7 +253,7 @@ def _format_data_content_block(block: dict) -> dict: } else: msg = ( - "Anthropic only supports 'url' and 'base64' source_type for image " + "Anthropic only supports 'url', 'base64', or 'id' keys for image " "content blocks." ) raise ValueError( @@ -251,7 +261,7 @@ def _format_data_content_block(block: dict) -> dict: ) elif block["type"] == "file": - if block["source_type"] == "url": + if "url" in block: formatted_block = { "type": "document", "source": { @@ -259,16 +269,16 @@ def _format_data_content_block(block: dict) -> dict: "url": block["url"], }, } - elif block["source_type"] == "base64": + elif "base64" in block or block.get("source_type") == "base64": formatted_block = { "type": "document", "source": { "type": "base64", "media_type": block.get("mime_type") or "application/pdf", - "data": block["data"], + "data": block.get("base64") or block.get("data", ""), }, } - elif block["source_type"] == "text": + elif block.get("source_type") == "text": formatted_block = { "type": "document", "source": { @@ -277,7 +287,15 @@ def _format_data_content_block(block: dict) -> dict: "data": block["text"], }, } - elif block["source_type"] == "id": + elif "file_id" in block: + formatted_block = { + "type": "document", + "source": { + "type": "file", + "file_id": block["file_id"], + }, + } + elif block.get("source_type") == "id": formatted_block = { "type": "document", "source": { @@ -285,6 +303,22 @@ def _format_data_content_block(block: dict) -> dict: "file_id": block["id"], }, } + else: + msg = ( + "Anthropic only supports 'url', 'base64', or 'id' keys for file " + "content blocks." + ) + raise ValueError(msg) + + elif block["type"] == "text-plain": + formatted_block = { + "type": "document", + "source": { + "type": "text", + "media_type": block.get("mime_type") or "text/plain", + "data": block["text"], + }, + } else: msg = f"Block of type {block['type']} is not supported." @@ -294,7 +328,10 @@ def _format_data_content_block(block: dict) -> dict: for key in ["cache_control", "citations", "title", "context"]: if key in block: formatted_block[key] = block[key] + elif (metadata := block.get("extras")) and key in metadata: + formatted_block[key] = metadata[key] elif (metadata := block.get("metadata")) and key in metadata: + # Backward compat formatted_block[key] = metadata[key] return formatted_block @@ -741,13 +778,11 @@ class ChatAnthropic(BaseChatModel): }, { "type": "image", - "source_type": "base64", - "data": image_data, + "base64": image_data, "mime_type": "image/jpeg", }, { "type": "image", - "source_type": "url", "url": image_url, }, ], @@ -781,7 +816,6 @@ class ChatAnthropic(BaseChatModel): }, { "type": "image", - "source_type": "id", "id": "file_abc123...", }, ], @@ -810,9 +844,8 @@ class ChatAnthropic(BaseChatModel): "Summarize this document.", { "type": "file", - "source_type": "base64", "mime_type": "application/pdf", - "data": data, + "base64": data, }, ] ) @@ -846,7 +879,6 @@ class ChatAnthropic(BaseChatModel): }, { "type": "file", - "source_type": "id", "id": "file_abc123...", }, ], @@ -1462,6 +1494,23 @@ class ChatAnthropic(BaseChatModel): **kwargs: dict, ) -> dict: messages = self._convert_input(input_).to_messages() + + for idx, message in enumerate(messages): + # Translate v1 content + if ( + isinstance(message, AIMessage) + and message.response_metadata.get("output_version") == "v1" + ): + messages[idx] = message.model_copy( + update={ + "content": _convert_from_v1_to_anthropic( + cast(list[types.ContentBlock], message.content), + message.tool_calls, + message.response_metadata.get("model_provider"), + ) + } + ) + system, formatted_messages = _format_messages(messages) # If cache_control is provided in kwargs, add it to last message @@ -1626,6 +1675,7 @@ class ChatAnthropic(BaseChatModel): llm_output = { k: v for k, v in data_dict.items() if k not in ("content", "role", "type") } + response_metadata = {"model_provider": "anthropic"} if "model" in llm_output and "model_name" not in llm_output: llm_output["model_name"] = llm_output["model"] if ( @@ -1633,15 +1683,18 @@ class ChatAnthropic(BaseChatModel): and content[0]["type"] == "text" and not content[0].get("citations") ): - msg = AIMessage(content=content[0]["text"]) + msg = AIMessage( + content=content[0]["text"], response_metadata=response_metadata + ) elif any(block["type"] == "tool_use" for block in content): tool_calls = extract_tool_calls(content) msg = AIMessage( content=content, tool_calls=tool_calls, + response_metadata=response_metadata, ) else: - msg = AIMessage(content=content) + msg = AIMessage(content=content, response_metadata=response_metadata) msg.usage_metadata = _create_usage_metadata(data.usage) return ChatResult( generations=[ChatGeneration(message=msg)], @@ -2363,7 +2416,7 @@ def _make_message_chunk_from_anthropic_event( elif event.type == "message_delta" and stream_usage: usage_metadata = _create_usage_metadata(event.usage) message_chunk = AIMessageChunk( - content="", + content="" if coerce_content_to_string else [], usage_metadata=usage_metadata, response_metadata={ "stop_reason": event.delta.stop_reason, @@ -2375,6 +2428,8 @@ def _make_message_chunk_from_anthropic_event( else: pass + if message_chunk: + message_chunk.response_metadata["model_provider"] = "anthropic" return message_chunk, block_start_event diff --git a/libs/partners/anthropic/tests/cassettes/test_agent_loop.yaml.gz b/libs/partners/anthropic/tests/cassettes/test_agent_loop.yaml.gz new file mode 100644 index 0000000000000000000000000000000000000000..d53dffb02da7f2011eb20a1282a0470f96fe817b GIT binary patch literal 2028 zcmV7^UY;X<+e9@aN z8*GqluP=aI=f|Ix41`I@ZO+SXRq~WX-K|!)++Vkf=CbP<3<&HeX$B9I}Y`j;6%-XM4fxi^~y@eaX`%+OXl`| zpx#aA-;<>+lm->INViL7%#-#pHyAn=xt~ueb0%hP7T!${R8kX3?HD@X1mc}T&KuB>5` zJdm#W-{FYtx}%!{rldyASXs_?-H|c1_n|`?mRhOkIcWxZ(1nkU*)*xIu7z<%1Us zNvn+^YCNaE_Mt`vO;>&*M-lH6qcl>hk2`O=jZ&dxn?|lm*rpJ&EjJiwiio8J^RV_D zkuM_27F@a8^$@E(NL4W$7~jCiS$%_e5;KO-=$L9XH|R;E^`!5mF*H@St3k=;xMMJ~ zmy^4p2*#-I#)d!K`%}kskd&v3*0S+v>=~&T9^=`0*c}7B>>%MuN*(HU&AefmeXRS| zHjws9DD`dV?1iVY#)b&o;8>olux!^sx~Z8E_l$?ZUiyp=Nk^>E{=S>c?HcPSkA`tJ z*K*YpQR>rhZff=plT~+ZSbVUk0G-BL3^74eE5VrJx?;%gVjx*0U>;BtuxM}4Ola@t zi^|uv+?N+splgsMlY893g~tjoN2)&z+K&T=hag3O#m75ou-Mjk#FAyMfuWT*;|bcq zW#1upEclMya)X?qbief^fUTp{9aB$Nqzy-qVTzNb3kgfd1iN+)aon)G`hy-3%Ff`U z^46m{@qqLUw$K{1o($m_9l)H7N64M7n8(M9%Y)8 zx{#2PHL?|k*XDR|NCm&*)QT~kGLSw)*g)%9x8BXAk}!D0%nQq$8*|2uO9FR(b$MYH z6;O~aF3k%+Eti0j;Srgaq}-mB1zA=0=Kf`YN1A5E%1@KlXUFIz;TzHgxJ(k7RJ|n4 zYhYrwiBbRt``6E@77>NA`qDw(p_q literal 0 HcmV?d00001 diff --git a/libs/partners/anthropic/tests/cassettes/test_agent_loop_streaming.yaml.gz b/libs/partners/anthropic/tests/cassettes/test_agent_loop_streaming.yaml.gz new file mode 100644 index 0000000000000000000000000000000000000000..8e76e86628fef510449e73e3c8cdaef7343961f1 GIT binary patch literal 3179 zcmV-x43zU9iwFRS&ZKAp|Lt2@Z{o-nexF~Fc^auDrfn0_O{94U1RK-Ax!B5~w;p`*#*6w*JkmBuyUwXx! zg)=jqY7uacfO{?`C{m7Nb~}-gcLt;kjEBVsLWxdsAc?ooUHm&?$sG?9MAVHYpKDSpU<0xZL~~W9NlhMjY8Q03cX?pK z_xLipr&EJ}Rf!|m0Ey3U2!dX6O$Pl*?F=2_fpxYI91ESMD$$@bLWP6G)Bpe9K4y_{d0?z?YOI_q&8zyZ-1!>2Qa@&FUQCv1=F zY|^tg-#)A_PKVEQU+R&sD$U2Qle8~APu3$+4TZo=d7ldb3QSD!z*@j1q;Gc#5ZBI3 zM$8a5$YspFx7@f4MzE>ji)&L_5!JIBqre4r?h%9L&^DT4rU&y39onfuoNCet>rAKG zi79?x4kxBEp0ws7le7fFRJbwsEc(um!pbD0P&U?&LAE_`0RR{FLG|ebhykdO2(?n3 z;yoQb+~)v;#kndF>zDB))fhU`nv+y#=ncj~G+?$7ECOPs;Ddw_h#rDI#8DX~x=*KA zWZ(1vgLmi#cL7e9rp!-ENCbx^9Uug{?Bsg016=apMneqPI@6wkl_Q_$05U9^Vsu>VlZe3;$Tz5Mgkq_b* zM~5PGDn51(MGo2i1m{n1{siY6z&W;(C=3#7*VN!=Bx#JI;}Ek^E>aPmJs=1IpmYEf zrU)mEVDv+Ugr$r_yF?URl1t*+py)sq+iR1dbI7)+Gr@ODl7KTP%M?jAH7FC{px3ej zb!{{|nilk$LSKED^to{8bHa!iyx;`AO)MoV6Sp9Vng&Svm;(iB2>??$l*ifJ$9Cg&Nl=s~sNRlXyUGD^V(44h@p1i3r~-9ieU4cDB2V2YkW{+=q$fw#%1 zt!bIZD^=4H)h8vF5@dF`@~H90@mw?gnSTTysSp%ux@>cAxCM|9#>| z*B2>NkQF3f-Ojc~lPO$`#9#sfNDy702hiv}u>A^@`P@Pe0DuF2s`#kuOG`hlI1yB7 zyk~Vwy%ionw=u|R`L1={lrrKyc5fhS^^q^LP+M-eEl8G3*K&)>fj;k?%WdgI(Bi{! z_mJE2^L_kAobl7jok}4)Xcr`Rs_wMAcI25p2>l$|ZxleHNLmrfC<0UQ4oLN_Qek2V zOD<~!Ey@xo1Z?nxC)}~)&v8|&Uzz1)gxC$|Z|Tj+CORtIPeB`kzOxvgr+S7ZHjViAGGvXUbxpnp1 z2M%=11gCUgki66cr*vO(dBg<9G98s+z5e^ z_Pu<1SPWhW3U&FkAbBk)6!|n)Wi&Gf9e-RLM`JshKn35m-0raFCJ3u?Tk*>SmE)m= zCACQwP2ebpc+(@wDxa9K2I_OA@-%R5vZu`+yd_g2~op`Og5G4gZ7Zi8{Hh~ zy^L|xJJvf**--zB`tm9X0}UV;&(VwTaMlNxKXjrC0k+PRrK`UOSHh7>uF$PaMQhad zzbyur$eA<=dT)H>9tr+U?jjC{5E&TcDL zx66*ry5r1ZZ*{E7+ncPTu`zJrRg1okgJ$cBw3_1Y(0s#3n<%zY)pb5-^eEDONpdRR z9yO)uwq@4sQB#IXB?TWuO}T2@nd8j(-wt(dC|3_8DJ_16KyM{s?GZs6kkATA&c$&I zZ5QBDX&$4yA0Y>N+ir1BTmE1cELMfA23#>!MKjAHii*viDRW-29$Of6dyU=}0TyPJ zDXSiciH>HGfrvT6F7MaOE7wC3O;d|8>I@BcabM}RaaimEvNEhi#S7=47hW)T5`Sz+ zyMWIQKQBAop-k9E^>?CR*`KeY3Z=d#DstuFzN||(_$TGDZvW8L499A8!yt`Aul~et z>viO7yv7=%HCEk$XDzM&La^e671w?>jUht|!*{PY$#6YOeS1 zUe5O6{Y&_ci#P8sWO;k-a&9YEFJbg5&0ek2cXeq@8wL*JB-PR^sXbFjdd&CHCeVOK zp}32}IBg`Bk;QJB*PdRu+GG7P(+U=0-pC@LXK6lAi5IS6;mu~vje_4|9d5!Air7@E z#pd$y?)Aqnwy~sNm7tTW%|TD{oQWo@sp@$XYeGM$f#>|@8_UtUezo}9Z7kW#Hm*;b-*vLyMFww1sBHbX?O*VT0HA7#>6dGpxIYW4uOCJ*<9{(FsXzq7NrQJ2JQ z=sH2ahI>hWTn6No%aFSMxYA~yT81c1F*lIOm{n^xME(Ew#8(YY*QV6UEaBc3%56vY zO$mPOn4+rxV}f6>RXfywxwtFL0H8oZP_FCd*yl>HeJkPmt=gdk2kXUCj-)eQtO^|m zNpOnexATP`w{pBq8<*_?71Ut@q^Z!MvJCH7!fu1PogrTDG{**hH#kknxh5T%z<-=& z)7v)?K8~vcP!N{TgRC<=DaEhtM0l6rPquqisnvpy5Ompad=0T&`=ZDPSt$3-g()qM z<2cx3wqt^h#e?AAMp99m+f$8VK R^!FH;{tqr?1U$k%004{e5`h2! literal 0 HcmV?d00001 diff --git a/libs/partners/anthropic/tests/cassettes/test_citations.yaml.gz b/libs/partners/anthropic/tests/cassettes/test_citations.yaml.gz new file mode 100644 index 0000000000000000000000000000000000000000..c704ad451e382cecff40e25c5a842eb552961b45 GIT binary patch literal 3388 zcmV-C4a4#uiwFP^uB2!J|Lt2@Z{tQ1e$TIPo&pS@3`yDE$Uq(}*`%nk=!lYdz44Mn ziPwtML1BOVRyWx^BnMI68bl#wD{U7PIU3iRTQcn${s0N@jHZ;R%-H$#Bgg8x8?4L| z)`l+8f<`xiv7ylQF3%cNljt21sZtpHKSv#*oB(kiqhSRJ>>v*TG{y*&K&IUK?N zu9nWr!z_-&=;ix&2=MPYmVY+`@B5nm?txZtPIOO_KYNmdU71ZW(PTCz)=dU~-V^6N zao!W>J#qd&B~Gj~F_4eUT=S}XX_3he>~C*RQmT)}{aTx*3sf)&D- zHV{2Vmql0Wv=;^=(1r|)-$K)Hw=3)t5|GU5I{zU#tWafKN>#)VqP0Vj0!!z*0?OG| zDhL)!+|1=UD?ygyEyK?md=x1ukReXeBrHEIG-c|1&R@at;#zG=F@k~86d?%61>wq) zw-G=Df*?*HP-~TmC&jvV@He7`Ltm|g_iTq=ya(!e)h(FPXhLCXEdwjiRdxlXr` z{LJFqwc&vOhXqPf*tnI5h43%@EV)Sdqgx7qvzbDW7JQ0;s{i~HDJih`;j)J}6jkJB zO%^Ao@ybW3g=N20w_QXW0tA7==h*P2$ml>Yz}kIxr3&W(N(4KwDk=FXgaEiUSkoVF zYR+*H93-Ej6HyQL! zqyO0S$02LGLPu@7fiVuYBUjyhjxBZ|yQ09TPPraHa@?5=>!WPS`@MCh(nzy6unQrC zzGRlsl(YRzfBu>XZ||G_yhuoyO2Tf=P=*fFO;oI*8ntnvEU~F)g?o(oPOp`zL}{?5 zBwtwUNa>LlP;Fh}%QN(9uc#C?p;N8y@CmByOYey`xJ_QGkCf>nHJ(k&GE@WN%TiH5 z(LBpN{Nq%ux^dHRsA(pimk6j1&Zirfc~M?!ik%><(=wi^O!?YnmHb>0C%!yV%MnIR z{|GCdY16}?0W#6rC{4jN z72r;uP-b>?b^sC84h_Feqa7Md>>|HsDnI9AM!^1L=Q<}%RKQ~fd$ie&rHJKb2KjBS z%0i$?Ei_bEAfK9D2@y+j#&t%*gbP9UQ0*gbS2U#^XaH$of0442;Bri3Ta4j%jq*hWM(!)Mf5Y!3K`} zF)r^TRNm_Z-t@Ts4~F&<>PGD!jh4C}O`zhx6J}Ey0ks0e#bU)!Scwy{L4^VZ??L+_Ik7T z*I#4zGP~Kj(&T*E_12b_USsG@p1s+nXMJf-8m4Z}Y(%l*RrGQJU9S_#oN?@UAQEMQ zHRx!D!7^?{wh7cMPHLssj{Z{518sjDB#l)F`fS|;Ez7lI+L#p9tjM;M?ZE;OhDzx{ zM*3bJK7U+B_B2JqREXyw%{GOZ#z>R2`VD?SZlu&vFfEt*oZv~Vw4uc(jXER-n>5mZ){Bf}Dzwt>vc|o(6(&UFeti7dzdbZe>XjEOzy`|U^ zE05INTZ%2IfJfBaQfvv0$~TI~N3Q8>p$b3(vgeu(;+8mE3!thC-lto9=XF~CcQOcm z5FirH!yOJnPT{hKo9Fl>D!mF1JfNHq)!I>AF_P+~mUpQinaYkQF-XOy)2?B#V0~Ad zbv(p+-=FzzbaKo?K*UKAh$E7>*A1kLR99;IvzvY3N!Wn!pBAlY{VXLI9aaG7)UD6wy})izL0u zvKrvP=eCqONL3LXnabn+O6ofVn7t`%`uiGiEqpecu06+P>j~I~)#-Z~E%|{w-RYlb z@eghc2#B2X-TtL*J~=&=6Hwpn_m!fIodQ+PHgY##TE0p?t$=EgM;!n0^T8m7EiJw1 zhVOMsaCt$7N#WkwoC6oxc==54osf%aQZt=!q0eP1FM)J+j;l4PeBZa1FOwQSR)#iw zC7J#ABA(nFarl{ISBF*eZy_+w^#n%ipC06w#|p3sKd?2)O131rKM@MaeR7 z?{_TsJC^$$%TIa7^0Qx{9KM!`cKS$o(z#hM@Fm(FS)TpQ?$o*ZXiEJ5_G_8Grl7}q z`6GEFj!B|JPI`a>CcUra6Zz$SYE+aEsO^x7jQ8Mlps>SjO%`MqfrR5t_4 z!*ehSy6DA6-S8!E%C@Jk9?9Q4AoVGT7Yi4=_3@V;$c|7-C z+U!#bGN&G7gjQbhvgq8CA~GJj;$_h}fbGr2E_qpW?nzNGTs`#^_$qJ}O<66*E&x}u z4=&Q19KqNv;39nbo%L1OU4bQ_%FUjtA>eF*vh(RX@@_i;+=V7HJ^<8Eeng)@=_J2* z1^lJzD&nA)O<6M;*V+3CPqLJb%;lk z97xVZehUeB%r3NuGMm71Xmfhtzbt)pCG5MZ#gCQ8b_Fi8d;hOLwmY~MS9kp{KDHxP ze&POq|3aUr{KEZzu|s|%6mKg`MoKKtox=ve None: @@ -65,6 +66,9 @@ def test_stream() -> None: assert chunks_with_model_name == 1 # check token usage is populated assert isinstance(full, AIMessageChunk) + assert len(full.content_blocks) == 1 + assert full.content_blocks[0]["type"] == "text" + assert full.content_blocks[0]["text"] assert full.usage_metadata is not None assert full.usage_metadata["input_tokens"] > 0 assert full.usage_metadata["output_tokens"] > 0 @@ -105,6 +109,9 @@ async def test_astream() -> None: ) # check token usage is populated assert isinstance(full, AIMessageChunk) + assert len(full.content_blocks) == 1 + assert full.content_blocks[0]["type"] == "text" + assert full.content_blocks[0]["text"] assert full.usage_metadata is not None assert full.usage_metadata["input_tokens"] > 0 assert full.usage_metadata["output_tokens"] > 0 @@ -421,6 +428,14 @@ def test_tool_use() -> None: assert isinstance(tool_call["args"], dict) assert "location" in tool_call["args"] + content_blocks = response.content_blocks + assert len(content_blocks) == 2 + assert content_blocks[0]["type"] == "text" + assert content_blocks[0]["text"] + assert content_blocks[1]["type"] == "tool_call" + assert content_blocks[1]["name"] == "get_weather" + assert content_blocks[1]["args"] == tool_call["args"] + # Test streaming llm = ChatAnthropic( model="claude-3-7-sonnet-20250219", # type: ignore[call-arg] @@ -440,6 +455,8 @@ def test_tool_use() -> None: first = False else: gathered = gathered + chunk # type: ignore[assignment] + for block in chunk.content_blocks: + assert block["type"] in ("text", "tool_call_chunk") assert len(chunks) > 1 assert isinstance(gathered.content, list) assert len(gathered.content) == 2 @@ -461,6 +478,14 @@ def test_tool_use() -> None: assert "location" in tool_call["args"] assert tool_call["id"] is not None + content_blocks = gathered.content_blocks + assert len(content_blocks) == 2 + assert content_blocks[0]["type"] == "text" + assert content_blocks[0]["text"] + assert content_blocks[1]["type"] == "tool_call_chunk" + assert content_blocks[1]["name"] == "get_weather" + assert content_blocks[1]["args"] + # Testing token-efficient tools # https://docs.anthropic.com/en/docs/build-with-claude/tool-use/token-efficient-tool-use assert gathered.usage_metadata @@ -500,6 +525,13 @@ def test_builtin_tools() -> None: assert isinstance(response, AIMessage) assert response.tool_calls + content_blocks = response.content_blocks + assert len(content_blocks) == 2 + assert content_blocks[0]["type"] == "text" + assert content_blocks[0]["text"] + assert content_blocks[1]["type"] == "tool_call" + assert content_blocks[1]["name"] == "str_replace_editor" + class GenerateUsername(BaseModel): """Get a username based on someone's name and hair color.""" @@ -682,8 +714,74 @@ def test_pdf_document_input() -> None: assert len(result.content) > 0 -def test_citations() -> None: - llm = ChatAnthropic(model="claude-3-5-haiku-latest") # type: ignore[call-arg] +@pytest.mark.default_cassette("test_agent_loop.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_agent_loop(output_version: Literal["v0", "v1"]) -> None: + @tool + def get_weather(location: str) -> str: + """Get the weather for a location.""" + return "It's sunny." + + llm = ChatAnthropic(model="claude-3-5-haiku-latest", output_version=output_version) # type: ignore[call-arg] + llm_with_tools = llm.bind_tools([get_weather]) + input_message = HumanMessage("What is the weather in San Francisco, CA?") + tool_call_message = llm_with_tools.invoke([input_message]) + assert isinstance(tool_call_message, AIMessage) + tool_calls = tool_call_message.tool_calls + assert len(tool_calls) == 1 + tool_call = tool_calls[0] + tool_message = get_weather.invoke(tool_call) + assert isinstance(tool_message, ToolMessage) + response = llm_with_tools.invoke( + [ + input_message, + tool_call_message, + tool_message, + ] + ) + assert isinstance(response, AIMessage) + + +@pytest.mark.default_cassette("test_agent_loop_streaming.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_agent_loop_streaming(output_version: Literal["v0", "v1"]) -> None: + @tool + def get_weather(location: str) -> str: + """Get the weather for a location.""" + return "It's sunny." + + llm = ChatAnthropic( + model="claude-3-5-haiku-latest", + streaming=True, + output_version=output_version, # type: ignore[call-arg] + ) + llm_with_tools = llm.bind_tools([get_weather]) + input_message = HumanMessage("What is the weather in San Francisco, CA?") + tool_call_message = llm_with_tools.invoke([input_message]) + assert isinstance(tool_call_message, AIMessage) + + tool_calls = tool_call_message.tool_calls + assert len(tool_calls) == 1 + tool_call = tool_calls[0] + tool_message = get_weather.invoke(tool_call) + assert isinstance(tool_message, ToolMessage) + response = llm_with_tools.invoke( + [ + input_message, + tool_call_message, + tool_message, + ] + ) + assert isinstance(response, AIMessage) + + +@pytest.mark.default_cassette("test_citations.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_citations(output_version: Literal["v0", "v1"]) -> None: + llm = ChatAnthropic(model="claude-3-5-haiku-latest", output_version=output_version) # type: ignore[call-arg] messages = [ { "role": "user", @@ -706,7 +804,10 @@ def test_citations() -> None: response = llm.invoke(messages) assert isinstance(response, AIMessage) assert isinstance(response.content, list) - assert any("citations" in block for block in response.content) + if output_version == "v1": + assert any("annotations" in block for block in response.content) + else: + assert any("citations" in block for block in response.content) # Test streaming full: Optional[BaseMessageChunk] = None @@ -714,8 +815,11 @@ def test_citations() -> None: full = cast(BaseMessageChunk, chunk) if full is None else full + chunk assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) - assert any("citations" in block for block in full.content) assert not any("citation" in block for block in full.content) + if output_version == "v1": + assert any("annotations" in block for block in full.content) + else: + assert any("citations" in block for block in full.content) # Test pass back in next_message = { @@ -762,25 +866,26 @@ def test_thinking() -> None: _ = llm.invoke([input_message, full, next_message]) +@pytest.mark.default_cassette("test_thinking.yaml.gz") @pytest.mark.vcr -def test_redacted_thinking() -> None: +def test_thinking_v1() -> None: llm = ChatAnthropic( model="claude-3-7-sonnet-latest", # type: ignore[call-arg] max_tokens=5_000, # type: ignore[call-arg] thinking={"type": "enabled", "budget_tokens": 2_000}, + output_version="v1", ) - query = "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB" # noqa: E501 - input_message = {"role": "user", "content": query} + input_message = {"role": "user", "content": "Hello"} response = llm.invoke([input_message]) - has_reasoning = False + assert any("reasoning" in block for block in response.content) for block in response.content: assert isinstance(block, dict) - if block["type"] == "redacted_thinking": - has_reasoning = True - assert set(block.keys()) == {"type", "data"} - assert block["data"] and isinstance(block["data"], str) - assert has_reasoning + if block["type"] == "reasoning": + assert set(block.keys()) == {"type", "reasoning", "extras"} + assert block["reasoning"] and isinstance(block["reasoning"], str) + signature = block["extras"]["signature"] + assert signature and isinstance(signature, str) # Test streaming full: Optional[BaseMessageChunk] = None @@ -788,14 +893,76 @@ def test_redacted_thinking() -> None: full = cast(BaseMessageChunk, chunk) if full is None else full + chunk assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) - stream_has_reasoning = False + assert any("reasoning" in block for block in full.content) + for block in full.content: + assert isinstance(block, dict) + if block["type"] == "reasoning": + assert set(block.keys()) == {"type", "reasoning", "extras", "index"} + assert block["reasoning"] and isinstance(block["reasoning"], str) + signature = block["extras"]["signature"] + assert signature and isinstance(signature, str) + + # Test pass back in + next_message = {"role": "user", "content": "How are you?"} + _ = llm.invoke([input_message, full, next_message]) + + +@pytest.mark.default_cassette("test_redacted_thinking.yaml.gz") +@pytest.mark.vcr +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_redacted_thinking(output_version: Literal["v0", "v1"]) -> None: + llm = ChatAnthropic( + model="claude-3-7-sonnet-latest", # type: ignore[call-arg] + max_tokens=5_000, # type: ignore[call-arg] + thinking={"type": "enabled", "budget_tokens": 2_000}, + output_version=output_version, + ) + query = "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB" # noqa: E501 + input_message = {"role": "user", "content": query} + + response = llm.invoke([input_message]) + value = None + for block in response.content: + assert isinstance(block, dict) + if block["type"] == "redacted_thinking": + value = block + elif ( + block["type"] == "non_standard" + and block["value"]["type"] == "redacted_thinking" + ): + value = block["value"] + else: + pass + if value: + assert set(value.keys()) == {"type", "data"} + assert value["data"] and isinstance(value["data"], str) + assert value is not None + + # Test streaming + full: Optional[BaseMessageChunk] = None + for chunk in llm.stream([input_message]): + full = cast(BaseMessageChunk, chunk) if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + assert isinstance(full.content, list) + value = None for block in full.content: assert isinstance(block, dict) if block["type"] == "redacted_thinking": - stream_has_reasoning = True - assert set(block.keys()) == {"type", "data", "index"} - assert block["data"] and isinstance(block["data"], str) - assert stream_has_reasoning + value = block + assert set(value.keys()) == {"type", "data", "index"} + assert "index" in block + elif ( + block["type"] == "non_standard" + and block["value"]["type"] == "redacted_thinking" + ): + value = block["value"] + assert set(value.keys()) == {"type", "data"} + assert "index" in block + else: + pass + if value: + assert value["data"] and isinstance(value["data"], str) + assert value is not None # Test pass back in next_message = {"role": "user", "content": "What?"} @@ -899,11 +1066,14 @@ def test_image_tool_calling() -> None: llm.bind_tools([color_picker]).invoke(messages) +@pytest.mark.default_cassette("test_web_search.yaml.gz") @pytest.mark.vcr -def test_web_search() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_web_search(output_version: Literal["v0", "v1"]) -> None: llm = ChatAnthropic( model="claude-3-5-sonnet-latest", # type: ignore[call-arg] max_tokens=1024, + output_version=output_version, ) tool = {"type": "web_search_20250305", "name": "web_search", "max_uses": 1} @@ -921,7 +1091,10 @@ def test_web_search() -> None: response = llm_with_tools.invoke([input_message]) assert all(isinstance(block, dict) for block in response.content) block_types = {block["type"] for block in response.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + else: + assert block_types == {"text", "web_search_call", "web_search_result"} # Test streaming full: Optional[BaseMessageChunk] = None @@ -931,7 +1104,10 @@ def test_web_search() -> None: assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) block_types = {block["type"] for block in full.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "web_search_tool_result"} + else: + assert block_types == {"text", "web_search_call", "web_search_result"} # Test we can pass back in next_message = { @@ -943,12 +1119,15 @@ def test_web_search() -> None: ) +@pytest.mark.default_cassette("test_code_execution.yaml.gz") @pytest.mark.vcr -def test_code_execution() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_code_execution(output_version: Literal["v0", "v1"]) -> None: llm = ChatAnthropic( model="claude-sonnet-4-20250514", # type: ignore[call-arg] betas=["code-execution-2025-05-22"], max_tokens=10_000, # type: ignore[call-arg] + output_version=output_version, ) tool = {"type": "code_execution_20250522", "name": "code_execution"} @@ -969,7 +1148,14 @@ def test_code_execution() -> None: response = llm_with_tools.invoke([input_message]) assert all(isinstance(block, dict) for block in response.content) block_types = {block["type"] for block in response.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + else: + assert block_types == { + "text", + "code_interpreter_call", + "code_interpreter_result", + } # Test streaming full: Optional[BaseMessageChunk] = None @@ -979,7 +1165,14 @@ def test_code_execution() -> None: assert isinstance(full, AIMessageChunk) assert isinstance(full.content, list) block_types = {block["type"] for block in full.content} # type: ignore[index] - assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + if output_version == "v0": + assert block_types == {"text", "server_tool_use", "code_execution_tool_result"} + else: + assert block_types == { + "text", + "code_interpreter_call", + "code_interpreter_result", + } # Test we can pass back in next_message = { @@ -991,8 +1184,10 @@ def test_code_execution() -> None: ) +@pytest.mark.default_cassette("test_remote_mcp.yaml.gz") @pytest.mark.vcr -def test_remote_mcp() -> None: +@pytest.mark.parametrize("output_version", ["v0", "v1"]) +def test_remote_mcp(output_version: Literal["v0", "v1"]) -> None: mcp_servers = [ { "type": "url", @@ -1008,6 +1203,7 @@ def test_remote_mcp() -> None: betas=["mcp-client-2025-04-04"], mcp_servers=mcp_servers, max_tokens=10_000, # type: ignore[call-arg] + output_version=output_version, ) input_message = { @@ -1025,7 +1221,10 @@ def test_remote_mcp() -> None: response = llm.invoke([input_message]) assert all(isinstance(block, dict) for block in response.content) block_types = {block["type"] for block in response.content} # type: ignore[index] - assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + if output_version == "v0": + assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + else: + assert block_types == {"text", "non_standard"} # Test streaming full: Optional[BaseMessageChunk] = None @@ -1036,7 +1235,10 @@ def test_remote_mcp() -> None: assert isinstance(full.content, list) assert all(isinstance(block, dict) for block in full.content) block_types = {block["type"] for block in full.content} # type: ignore[index] - assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + if output_version == "v0": + assert block_types == {"text", "mcp_tool_use", "mcp_tool_result"} + else: + assert block_types == {"text", "non_standard"} # Test we can pass back in next_message = { @@ -1069,8 +1271,7 @@ def test_files_api_image(block_format: str) -> None: # standard block format block = { "type": "image", - "source_type": "id", - "id": image_file_id, + "file_id": image_file_id, } input_message = { "role": "user", @@ -1097,8 +1298,7 @@ def test_files_api_pdf(block_format: str) -> None: # standard block format block = { "type": "file", - "source_type": "id", - "id": pdf_file_id, + "file_id": pdf_file_id, } input_message = { "role": "user", @@ -1163,6 +1363,11 @@ def test_search_result_tool_message() -> None: assert isinstance(result.content, list) assert any("citations" in block for block in result.content) + assert ( + _convert_from_v1_to_anthropic(result.content_blocks, [], "anthropic") + == result.content + ) + def test_search_result_top_level() -> None: llm = ChatAnthropic( @@ -1209,6 +1414,11 @@ def test_search_result_top_level() -> None: assert isinstance(result.content, list) assert any("citations" in block for block in result.content) + assert ( + _convert_from_v1_to_anthropic(result.content_blocks, [], "anthropic") + == result.content + ) + def test_async_shared_client() -> None: llm = ChatAnthropic(model="claude-3-5-haiku-latest") # type: ignore[call-arg] diff --git a/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr b/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr index b831aef469b..5c9164caae3 100644 --- a/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr +++ b/libs/partners/anthropic/tests/unit_tests/__snapshots__/test_standard.ambr @@ -20,6 +20,7 @@ 'max_retries': 2, 'max_tokens': 100, 'model': 'claude-3-haiku-20240307', + 'output_version': 'v0', 'stop_sequences': list([ ]), 'stream_usage': True, diff --git a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py index 382d2f774c5..3cf2b0e44ee 100644 --- a/libs/partners/anthropic/tests/unit_tests/test_chat_models.py +++ b/libs/partners/anthropic/tests/unit_tests/test_chat_models.py @@ -211,6 +211,7 @@ def test__format_output() -> None: "total_tokens": 3, "input_token_details": {}, }, + response_metadata={"model_provider": "anthropic"}, ) llm = ChatAnthropic(model="test", anthropic_api_key="test") # type: ignore[call-arg, call-arg] actual = llm._format_output(anthropic_msg) @@ -241,6 +242,7 @@ def test__format_output_cached() -> None: "total_tokens": 10, "input_token_details": {"cache_creation": 3, "cache_read": 4}, }, + response_metadata={"model_provider": "anthropic"}, ) llm = ChatAnthropic(model="test", anthropic_api_key="test") # type: ignore[call-arg, call-arg] @@ -849,7 +851,7 @@ def test__format_messages_with_cache_control() -> None: assert expected_system == actual_system assert expected_messages == actual_messages - # Test standard multi-modal format + # Test standard multi-modal format (v0) messages = [ HumanMessage( [ @@ -891,6 +893,183 @@ def test__format_messages_with_cache_control() -> None: ] assert actual_messages == expected_messages + # Test standard multi-modal format (v1) + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "file", + "mime_type": "application/pdf", + "base64": "", + "extras": {"cache_control": {"type": "ephemeral"}}, + }, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "", + }, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + ] + assert actual_messages == expected_messages + + # Test standard multi-modal format (v1, unpacked extras) + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "file", + "mime_type": "application/pdf", + "base64": "", + "cache_control": {"type": "ephemeral"}, + }, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "", + }, + "cache_control": {"type": "ephemeral"}, + }, + ], + }, + ] + assert actual_messages == expected_messages + + # Also test file inputs + ## Images + for block in [ + # v1 + { + "type": "image", + "file_id": "abc123", + }, + # v0 + { + "type": "image", + "source_type": "id", + "id": "abc123", + }, + ]: + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this image:", + }, + block, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this image:", + }, + { + "type": "image", + "source": { + "type": "file", + "file_id": "abc123", + }, + }, + ], + }, + ] + assert actual_messages == expected_messages + + ## Documents + for block in [ + # v1 + { + "type": "file", + "file_id": "abc123", + }, + # v0 + { + "type": "file", + "source_type": "id", + "id": "abc123", + }, + ]: + messages = [ + HumanMessage( + [ + { + "type": "text", + "text": "Summarize this document:", + }, + block, + ], + ), + ] + actual_system, actual_messages = _format_messages(messages) + assert actual_system is None + expected_messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document:", + }, + { + "type": "document", + "source": { + "type": "file", + "file_id": "abc123", + }, + }, + ], + }, + ] + assert actual_messages == expected_messages + def test__format_messages_with_citations() -> None: input_messages = [