github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-04 18:53:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(core): standard content blocks (#32085)

Browse Source
This commit is contained in:
Mason Daugherty 2025-07-22 09:17:55 -04:00 committed by GitHub
parent 3c19cafab0
commit b24f90dabe
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 2169 additions and 1822 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
libs/core/langchain_core/messages/__init__.py
View File

@ -33,15 +33,25 @@ if TYPE_CHECKING:
)
from langchain_core.messages.chat import ChatMessage, ChatMessageChunk
from langchain_core.messages.content_blocks import (
Base64ContentBlock,
Annotation,
AudioContentBlock,
Citation,
CodeInterpreterCall,
CodeInterpreterOutput,
CodeInterpreterResult,
ContentBlock,
DocumentCitation,
DataContentBlock,
FileContentBlock,
ImageContentBlock,
NonStandardAnnotation,
NonStandardContentBlock,
PlainTextContentBlock,
ReasoningContentBlock,
SearchCall,
SearchResult,
TextContentBlock,
ToolCallContentBlock,
UrlCitation,
VideoContentBlock,
convert_to_openai_data_block,
convert_to_openai_image_block,
is_data_content_block,
@ -74,24 +84,34 @@ if TYPE_CHECKING:
__all__ = (
"AIMessage",
"AIMessageChunk",
"Annotation",
"AnyMessage",
"Base64ContentBlock",
"AudioContentBlock",
"BaseMessage",
"BaseMessageChunk",
"ChatMessage",
"ChatMessageChunk",
"Citation",
"CodeInterpreterCall",
"CodeInterpreterOutput",
"CodeInterpreterResult",
"ContentBlock",
"DocumentCitation",
"DataContentBlock",
"FileContentBlock",
"FunctionMessage",
"FunctionMessageChunk",
"HumanMessage",
"HumanMessageChunk",
"ImageContentBlock",
"InvalidToolCall",
"MessageLikeRepresentation",
"NonStandardAnnotation",
"NonStandardContentBlock",
"PlainTextContentBlock",
"ReasoningContentBlock",
"RemoveMessage",
"SearchCall",
"SearchResult",
"SystemMessage",
"SystemMessageChunk",
"TextContentBlock",
@ -100,7 +120,7 @@ __all__ = (
"ToolCallContentBlock",
"ToolMessage",
"ToolMessageChunk",
"UrlCitation",
"VideoContentBlock",
"_message_from_dict",
"convert_to_messages",
"convert_to_openai_data_block",
@ -121,26 +141,36 @@ __all__ = (
_dynamic_imports = {
"AIMessage": "ai",
"AIMessageChunk": "ai",
"Base64ContentBlock": "content_blocks",
"Annotation": "content_blocks",
"AudioContentBlock": "content_blocks",
"BaseMessage": "base",
"BaseMessageChunk": "base",
"merge_content": "base",
"message_to_dict": "base",
"messages_to_dict": "base",
"Citation": "content_blocks",
"ContentBlock": "content_blocks",
"ChatMessage": "chat",
"ChatMessageChunk": "chat",
"DocumentCitation": "content_blocks",
"CodeInterpreterCall": "content_blocks",
"CodeInterpreterOutput": "content_blocks",
"CodeInterpreterResult": "content_blocks",
"DataContentBlock": "content_blocks",
"FileContentBlock": "content_blocks",
"FunctionMessage": "function",
"FunctionMessageChunk": "function",
"HumanMessage": "human",
"HumanMessageChunk": "human",
"NonStandardAnnotation": "content_blocks",
"NonStandardContentBlock": "content_blocks",
"PlainTextContentBlock": "content_blocks",
"ReasoningContentBlock": "content_blocks",
"RemoveMessage": "modifier",
"SystemMessage": "system",
"SystemMessageChunk": "system",
"SearchCall": "content_blocks",
"SearchResult": "content_blocks",
"ImageContentBlock": "content_blocks",
"InvalidToolCall": "tool",
"TextContentBlock": "content_blocks",
"ToolCall": "tool",
@ -148,7 +178,7 @@ _dynamic_imports = {
"ToolCallContentBlock": "content_blocks",
"ToolMessage": "tool",
"ToolMessageChunk": "tool",
"UrlCitation": "content_blocks",
"VideoContentBlock": "content_blocks",
"AnyMessage": "utils",
"MessageLikeRepresentation": "utils",
"_message_from_dict": "utils",

55
libs/core/langchain_core/messages/ai.py
View File

@ -8,7 +8,6 @@ from typing import Any, Literal, Optional, Union, cast
from pydantic import model_validator
from typing_extensions import NotRequired, Self, TypedDict, override
from langchain_core.messages import content_blocks as types
from langchain_core.messages.base import (
BaseMessage,
BaseMessageChunk,
@ -197,60 +196,6 @@ class AIMessage(BaseMessage):
"invalid_tool_calls": self.invalid_tool_calls,
}
@property
def beta_content(self) -> list[types.ContentBlock]:
"""Return the content as a list of standard ContentBlocks.
To use this property, the corresponding chat model must support
``output_version="v1"`` or higher:
.. code-block:: python
from langchain.chat_models import init_chat_model
llm = init_chat_model("...", output_version="v1")
otherwise, does best-effort parsing to standard types.
"""
blocks: list[types.ContentBlock] = []
content = (
[self.content]
if isinstance(self.content, str) and self.content
else self.content
)
for item in content:
if isinstance(item, str):
blocks.append({"type": "text", "text": item})
elif isinstance(item, dict):
item_type = item.get("type")
if item_type not in types.KNOWN_BLOCK_TYPES:
msg = (
f"Non-standard content block type '{item_type}'. Ensure "
"the model supports `output_version='v1'` or higher and "
"that this attribute is set on initialization."
)
raise ValueError(msg)
blocks.append(cast("types.ContentBlock", item))
else:
pass
# Add from tool_calls if missing from content
content_tool_call_ids = {
block.get("id")
for block in self.content
if isinstance(block, dict) and block.get("type") == "tool_call"
}
for tool_call in self.tool_calls:
if (id_ := tool_call.get("id")) and id_ not in content_tool_call_ids:
tool_call_block: types.ToolCallContentBlock = {
"type": "tool_call",
"id": id_,
}
blocks.append(tool_call_block)
return blocks
# TODO: remove this logic if possible, reducing breaking nature of changes
@model_validator(mode="before")
@classmethod

635
libs/core/langchain_core/messages/content_blocks.py
View File

@ -1,4 +1,81 @@
"""Types for content blocks."""
"""Standard, multimodal content blocks for Large Language Model I/O.
.. warning::
This module is under active development. The API is unstable and subject to
change in future releases.
This module provides a standardized data structure for representing inputs to and
outputs from Large Language Models. The core abstraction is the **Content Block**, a
``TypedDict`` that can represent a piece of text, an image, a tool call, or other
structured data.
Data **not yet mapped** to a standard block may be represented using the
``NonStandardContentBlock``, which allows for provider-specific data to be included
without losing the benefits of type checking and validation.
Furthermore, provider-specific fields *within* a standard block will be allowed as extra
keys on the TypedDict per `PEP 728 <https://peps.python.org/pep-0728/>`__. This allows
for flexibility in the data structure while maintaining a consistent interface.
**Example using ``extra_items=Any``:**
.. code-block:: python
from langchain_core.messages.content_blocks import TextContentBlock
from typing import Any
my_block: TextContentBlock = {
"type": "text",
"text": "Hello, world!",
"extra_field": "This is allowed",
"another_field": 42, # Any type is allowed
}
# A type checker that supports PEP 728 would validate the object above.
# Accessing the provider-specific key is possible, and its type is 'Any'.
block_extra_field = my_block["extra_field"]
.. warning::
Type checkers such as MyPy do not yet support `PEP 728 <https://peps.python.org/pep-0728/>`__,
so you may see type errors when using provider-specific fields. These are safe to
ignore, as the fields are still validated at runtime.
**Rationale**
Different LLM providers use distinct and incompatible API schemas. This module
introduces a unified, provider-agnostic format to standardize these interactions. A
message to or from a model is simply a `list` of `ContentBlock` objects, allowing for
the natural interleaving of text, images, and other content in a single, ordered
sequence.
An adapter for a specific provider is responsible for translating this standard list of
blocks into the format required by its API.
**Key Block Types**
The module defines several types of content blocks, including:
- **``TextContentBlock``**: Standard text.
- **``ImageContentBlock``**, **``AudioContentBlock``**, **``VideoContentBlock``**: For
multimodal data.
- **``ToolCallContentBlock``**, **``ToolOutputContentBlock``**: For function calling.
- **``ReasoningContentBlock``**: To capture a model's thought process.
- **``Citation``**: For annotations that link generated text to a source document.
**Example Usage**
.. code-block:: python
from langchain_core.messages.content_blocks import TextContentBlock, ImageContentBlock
multimodal_message: AIMessage = [
TextContentBlock(type="text", text="What is shown in this image?"),
ImageContentBlock(
type="image",
url="https://www.langchain.com/images/brand/langchain_logo_text_w_white.png",
mime_type="image/png",
),
]
""" # noqa: E501
import warnings
from typing import Any, Literal, Union
@ -6,45 +83,60 @@ from typing import Any, Literal, Union
from pydantic import TypeAdapter, ValidationError
from typing_extensions import NotRequired, TypedDict, get_args, get_origin
# --- Text and annotations ---
# Text and annotations
class UrlCitation(TypedDict):
"""Citation from a URL."""
type: Literal["url_citation"]
class Citation(TypedDict):
"""Annotation for citing data from a document.
url: str
"""Source URL."""
.. note::
``start/end`` indices refer to the **response text**,
not the source text. This means that the indices are relative to the model's
response, not the original document (as specified in the ``url``).
"""
type: Literal["citation"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
url: NotRequired[str]
"""URL of the document source."""
# For future consideration, if needed:
# provenance: NotRequired[str]
# """Provenance of the document, e.g., "Wikipedia", "arXiv", etc.
# Included for future compatibility; not currently implemented.
# """
title: NotRequired[str]
"""Source title."""
"""Source document title.
cited_text: NotRequired[str]
"""Text from the source that is being cited."""
For example, the page title for a web page or the title of a paper.
"""
start_index: NotRequired[int]
"""Start index of the response text for which the annotation applies."""
"""Start index of the **response text** (``TextContentBlock.text``) for which the
annotation applies."""
end_index: NotRequired[int]
"""End index of the response text for which the annotation applies."""
class DocumentCitation(TypedDict):
"""Annotation for data from a document."""
type: Literal["document_citation"]
title: NotRequired[str]
"""Source title."""
"""End index of the **response text** (``TextContentBlock.text``) for which the
annotation applies."""
cited_text: NotRequired[str]
"""Text from the source that is being cited."""
"""Excerpt of source text being cited."""
start_index: NotRequired[int]
"""Start index of the response text for which the annotation applies."""
end_index: NotRequired[int]
"""End index of the response text for which the annotation applies."""
# NOTE: not including spans for the raw document text (such as `text_start_index`
# and `text_end_index`) as this is not currently supported by any provider. The
# thinking is that the `cited_text` should be sufficient for most use cases, and it
# is difficult to reliably extract spans from the raw document text across file
# formats or encoding schemes.
class NonStandardAnnotation(TypedDict):
@ -52,107 +144,423 @@ class NonStandardAnnotation(TypedDict):
type: Literal["non_standard_annotation"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
value: dict[str, Any]
"""Provider-specific annotation data."""
Annotation = Union[Citation, NonStandardAnnotation]
class TextContentBlock(TypedDict):
"""Content block for text output."""
"""Content block for text output.
This typically represents the main text content of a message, such as the response
from a language model or the text of a user message.
"""
type: Literal["text"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
text: str
"""Block text."""
annotations: NotRequired[
list[Union[UrlCitation, DocumentCitation, NonStandardAnnotation]]
]
annotations: NotRequired[list[Annotation]]
"""Citations and other annotations."""
# Tool calls
# --- Tool calls ---
class ToolCallContentBlock(TypedDict):
"""Content block for tool calls.
These are references to a :class:`~langchain_core.messages.tool.ToolCall` in the
message's ``tool_calls`` attribute.
"""
"""Content block for tool calls."""
type: Literal["tool_call"]
"""Type of the content block."""
id: str
"""Tool call ID."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
name: str
"""The name of the tool to be called."""
args: dict[str, Any]
"""The arguments for the tool, as a dictionary."""
call_id: str
"""The unique ID for this tool call."""
# Reasoning
# --- Provider tool calls (built-in tools) ---
# Note: These are not standard tool calls, but rather provider-specific built-in tools.
# Web search
class SearchCall(TypedDict):
"""Content block for a built-in web search tool call."""
type: Literal["search_call"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
query: NotRequired[str]
"""The search query used in the web search tool call."""
class SearchResult(TypedDict):
"""Content block for the result of a built-in search tool call."""
type: Literal["search_result"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
urls: NotRequired[list[str]]
"""List of URLs returned by the web search tool call."""
# Code interpreter
# Call
class CodeInterpreterCall(TypedDict):
"""Content block for a built-in code interpreter tool call."""
type: Literal["code_interpreter_call"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
language: NotRequired[str]
"""The programming language used in the code interpreter tool call."""
code: NotRequired[str]
"""The code to be executed by the code interpreter."""
# Result block is CodeInterpreterResult
class CodeInterpreterOutput(TypedDict):
"""Content block for the output of a singular code interpreter tool call.
Full output of a code interpreter tool call is represented by
``CodeInterpreterResult`` which is a list of these blocks.
"""
type: Literal["code_interpreter_output"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
return_code: NotRequired[int]
"""Return code of the executed code.
Example: 0 for success, non-zero for failure.
"""
stderr: NotRequired[str]
"""Standard error output of the executed code."""
stdout: NotRequired[str]
"""Standard output of the executed code."""
file_ids: NotRequired[list[str]]
"""List of file IDs generated by the code interpreter."""
class CodeInterpreterResult(TypedDict):
"""Content block for the result of a code interpreter tool call."""
type: Literal["code_interpreter_result"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
output: list[CodeInterpreterOutput]
"""List of outputs from the code interpreter tool call."""
# --- Reasoning ---
class ReasoningContentBlock(TypedDict):
"""Content block for reasoning output."""
type: Literal["reasoning"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
reasoning: NotRequired[str]
"""Reasoning text."""
"""Reasoning text.
Either the thought summary or the raw reasoning text itself. This is often parsed
from ``<think>`` tags in the model's response.
"""
thought_signature: NotRequired[str]
"""Opaque state handle representation of the model's internal thought process.
Maintains the context of the model's thinking across multiple interactions
(e.g. multi-turn conversations) since many APIs are stateless.
Not to be used to verify authenticity or integrity of the response (`'signature'`).
Examples:
- https://ai.google.dev/gemini-api/docs/thinking#signatures
"""
signature: NotRequired[str]
"""Signature of the reasoning content block used to verify **authenticity**.
Prevents from modifying or fabricating the model's reasoning process.
Examples:
- https://docs.anthropic.com/en/docs/build-with-claude/context-windows#the-context-window-with-extended-thinking-and-tool-use
"""
# Multi-modal
class BaseDataContentBlock(TypedDict):
"""Base class for data content blocks."""
# --- Multi-modal ---
# Note: `title` and `context` are fields that could be used to provide additional
# information about the file, such as a description or summary of its content.
# E.g. with Claude, you can provide a context for a file which is passed to the model.
class ImageContentBlock(TypedDict):
"""Content block for image data."""
type: Literal["image"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
file_id: NotRequired[str]
"""ID of the image file, e.g., from a file storage system."""
mime_type: NotRequired[str]
"""MIME type of the content block (if needed)."""
"""MIME type of the image. Required for base64.
`Examples from IANA <https://www.iana.org/assignments/media-types/media-types.xhtml#image>`__
"""
class URLContentBlock(BaseDataContentBlock):
"""Content block for data from a URL."""
url: NotRequired[str]
"""URL of the image."""
type: Literal["image", "audio", "file"]
"""Type of the content block."""
source_type: Literal["url"]
"""Source type (url)."""
url: str
"""URL for data."""
class Base64ContentBlock(BaseDataContentBlock):
"""Content block for inline data from a base64 string."""
type: Literal["image", "audio", "file"]
"""Type of the content block."""
source_type: Literal["base64"]
"""Source type (base64)."""
data: str
base64: NotRequired[str]
"""Data as a base64 string."""
# title: NotRequired[str]
# """Title of the image."""
class PlainTextContentBlock(BaseDataContentBlock):
"""Content block for plain text data (e.g., from a document)."""
# context: NotRequired[str]
# """Context for the image, e.g., a description or summary of the image's content.""" # noqa: E501
class VideoContentBlock(TypedDict):
"""Content block for video data."""
type: Literal["video"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
file_id: NotRequired[str]
"""ID of the video file, e.g., from a file storage system."""
mime_type: NotRequired[str]
"""MIME type of the video. Required for base64.
`Examples from IANA <https://www.iana.org/assignments/media-types/media-types.xhtml#video>`__
"""
url: NotRequired[str]
"""URL of the video."""
base64: NotRequired[str]
"""Data as a base64 string."""
# title: NotRequired[str]
# """Title of the video."""
# context: NotRequired[str]
# """Context for the video, e.g., a description or summary of the video's content.""" # noqa: E501
class AudioContentBlock(TypedDict):
"""Content block for audio data."""
type: Literal["audio"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
file_id: NotRequired[str]
"""ID of the audio file, e.g., from a file storage system."""
mime_type: NotRequired[str]
"""MIME type of the audio. Required for base64.
`Examples from IANA <https://www.iana.org/assignments/media-types/media-types.xhtml#audio>`__
"""
url: NotRequired[str]
"""URL of the audio."""
base64: NotRequired[str]
"""Data as a base64 string."""
# title: NotRequired[str]
# """Title of the audio."""
# context: NotRequired[str]
# """Context for the audio, e.g., a description or summary of the audio's content.""" # noqa: E501
class PlainTextContentBlock(TypedDict):
"""Content block for plaintext data (e.g., from a document).
.. note::
Title and context are optional fields that may be passed to the model. See
Anthropic `example <https://docs.anthropic.com/en/docs/build-with-claude/citations#citable-vs-non-citable-content>`__.
"""
type: Literal["text-plain"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
file_id: NotRequired[str]
"""ID of the plaintext file, e.g., from a file storage system."""
mime_type: Literal["text/plain"]
"""MIME type of the file. Required for base64."""
url: NotRequired[str]
"""URL of the plaintext."""
base64: NotRequired[str]
"""Data as a base64 string."""
text: NotRequired[str]
"""Plaintext content. This is optional if the data is provided as base64."""
title: NotRequired[str]
"""Title of the text data, e.g., the title of a document."""
context: NotRequired[str]
"""Context for the text, e.g., a description or summary of the text's content."""
class FileContentBlock(TypedDict):
"""Content block for file data.
This block is intended for files that are not images, audio, or plaintext. For
example, it can be used for PDFs, Word documents, etc.
If the file is an image, audio, or plaintext, you should use the corresponding
content block type (e.g., ``ImageContentBlock``, ``AudioContentBlock``,
``PlainTextContentBlock``).
"""
type: Literal["file"]
"""Type of the content block."""
source_type: Literal["text"]
"""Source type (text)."""
text: str
"""Text data."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
file_id: NotRequired[str]
"""ID of the file, e.g., from a file storage system."""
mime_type: NotRequired[str]
"""MIME type of the file. Required for base64.
`Examples from IANA <https://www.iana.org/assignments/media-types/media-types.xhtml>`__
"""
url: NotRequired[str]
"""URL of the file."""
base64: NotRequired[str]
"""Data as a base64 string."""
# title: NotRequired[str]
# """Title of the file, e.g., the name of a document or file."""
# context: NotRequired[str]
# """Context for the file, e.g., a description or summary of the file's content."""
class IDContentBlock(BaseDataContentBlock):
"""Content block for data specified by an identifier."""
type: Literal["image", "audio", "file"]
"""Type of the content block."""
source_type: Literal["id"]
"""Source type (id)."""
id: str
"""Identifier for data source."""
DataContentBlock = Union[
URLContentBlock,
Base64ContentBlock,
PlainTextContentBlock,
IDContentBlock,
]
_DataContentBlockAdapter: TypeAdapter[DataContentBlock] = TypeAdapter(DataContentBlock)
# Future modalities to consider:
# - 3D models
# - Tabular data
# Non-standard
@ -160,20 +568,52 @@ class NonStandardContentBlock(TypedDict):
"""Content block provider-specific data.
This block contains data for which there is not yet a standard type.
The purpose of this block should be to simply hold a provider-specific payload.
If a provider's non-standard output includes reasoning and tool calls, it should be
the adapter's job to parse that payload and emit the corresponding standard
ReasoningContentBlock and ToolCallContentBlocks.
"""
type: Literal["non_standard"]
"""Type of the content block."""
id: NotRequired[str]
"""Content block identifier. Either:
- Generated by the provider (e.g., OpenAI's file ID)
- Generated by LangChain upon creation (as ``UUID4``)
"""
value: dict[str, Any]
"""Provider-specific data."""
# --- Aliases ---
DataContentBlock = Union[
ImageContentBlock,
VideoContentBlock,
AudioContentBlock,
PlainTextContentBlock,
FileContentBlock,
]
ToolContentBlock = Union[
ToolCallContentBlock,
CodeInterpreterCall,
CodeInterpreterOutput,
CodeInterpreterResult,
SearchCall,
SearchResult,
]
ContentBlock = Union[
TextContentBlock,
ToolCallContentBlock,
ReasoningContentBlock,
DataContentBlock,
NonStandardContentBlock,
DataContentBlock,
ToolContentBlock,
]
@ -190,29 +630,32 @@ def _extract_typedict_type_values(union_type: Any) -> set[str]:
return result
# {"text", "tool_call", "reasoning", "non_standard", "image", "audio", "file"}
KNOWN_BLOCK_TYPES = _extract_typedict_type_values(ContentBlock)
KNOWN_BLOCK_TYPES = {
bt for bt in get_args(ContentBlock) for bt in get_args(bt.__annotations__["type"])
}
# Adapter for DataContentBlock
_DataAdapter: TypeAdapter[DataContentBlock] = TypeAdapter(DataContentBlock)
def is_data_content_block(
content_block: dict,
) -> bool:
def is_data_content_block(block: dict) -> bool:
"""Check if the content block is a standard data content block.
Args:
content_block: The content block to check.
block: The content block to check.
Returns:
True if the content block is a data content block, False otherwise.
"""
try:
_ = _DataContentBlockAdapter.validate_python(content_block)
_DataAdapter.validate_python(block)
except ValidationError:
return False
else:
return True
# TODO: don't use `source_type` anymore
def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict:
"""Convert image content block to format expected by OpenAI Chat Completions API."""
if content_block["source_type"] == "url":

71
libs/core/tests/unit_tests/messages/test_ai.py
View File

@ -1,8 +1,5 @@
from typing import Union, cast
from langchain_core.load import dumpd, load
from langchain_core.messages import AIMessage, AIMessageChunk
from langchain_core.messages import content_blocks as types
from langchain_core.messages.ai import (
InputTokenDetails,
OutputTokenDetails,
@ -199,71 +196,3 @@ def test_add_ai_message_chunks_usage() -> None:
output_token_details=OutputTokenDetails(audio=1, reasoning=2),
),
)
class ReasoningContentBlockWithID(types.ReasoningContentBlock):
id: str
def test_beta_content() -> None:
# Simple case
message = AIMessage("Hello")
assert len(message.beta_content) == 1
assert message.beta_content[0]["type"] == "text"
for block in message.beta_content:
if block["type"] == "text":
text_block: types.TextContentBlock = block
assert text_block == {"type": "text", "text": "Hello"}
# With tool calls
message = AIMessage(
"",
tool_calls=[
{"type": "tool_call", "name": "foo", "args": {"a": "b"}, "id": "abc_123"}
],
)
assert len(message.beta_content) == 1
assert message.beta_content[0]["type"] == "tool_call"
for block in message.beta_content:
if block["type"] == "tool_call":
tool_call_block: types.ToolCallContentBlock = block
assert tool_call_block == {"type": "tool_call", "id": "abc_123"}
# With standard blocks
reasoning_with_id: ReasoningContentBlockWithID = {
"type": "reasoning",
"reasoning": "foo",
"id": "rs_abc123",
}
standard_content: list[types.ContentBlock] = [
{"type": "reasoning", "reasoning": "foo"},
reasoning_with_id,
{"type": "text", "text": "bar"},
{
"type": "text",
"text": "baz",
"annotations": [{"type": "url_citation", "url": "http://example.com"}],
},
{
"type": "image",
"source_type": "url",
"url": "http://example.com/image.png",
},
{
"type": "non_standard",
"value": {"custom_key": "custom_value", "another_key": 123},
},
{
"type": "tool_call",
"id": "abc_123",
},
]
message = AIMessage(
cast("list[Union[str, dict]]", standard_content),
tool_calls=[
{"type": "tool_call", "name": "foo", "args": {"a": "b"}, "id": "abc_123"},
{"type": "tool_call", "name": "bar", "args": {"c": "d"}, "id": "abc_234"},
],
)
missing_tool_call = {"type": "tool_call", "id": "abc_234"}
assert message.beta_content == [*standard_content, missing_tool_call]

3182
libs/core/uv.lock
View File

File diff suppressed because it is too large Load Diff