github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2026-06-09 18:50:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

core, openai: support standard multi-modal blocks in convert_to_openai_messages (#30968)

Browse Source
This commit is contained in:
ccurme
2025-04-23 11:20:44 -04:00
committed by GitHub
parent e4877e5ef1
commit 4bc70766b5
6 changed files with 104 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
libs/partners/openai/langchain_openai/chat_models/base.py
View File

@@ -61,7 +61,7 @@ from langchain_core.messages import (
ToolCall,
ToolMessage,
ToolMessageChunk,
convert_to_openai_image_block,
convert_to_openai_data_block,
is_data_content_block,
)
from langchain_core.messages.ai import (
@@ -186,45 +186,6 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:
return ChatMessage(content=_dict.get("content", ""), role=role, id=id_) # type: ignore[arg-type]
def _format_data_content_block(block: dict) -> dict:
"""Format standard data content block to format expected by OpenAI."""
if block["type"] == "image":
formatted_block = convert_to_openai_image_block(block)
elif block["type"] == "file":
if block["source_type"] == "base64":
file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}
if filename := block.get("filename"):
file["filename"] = filename
elif (metadata := block.get("metadata")) and ("filename" in metadata):
file["filename"] = metadata["filename"]
else:
warnings.warn(
"OpenAI may require a filename for file inputs. Specify a filename "
"in the content block: {'type': 'file', 'source_type': 'base64', "
"'mime_type': 'application/pdf', 'data': '...', "
"'filename': 'my-pdf'}"
)
formatted_block = {"type": "file", "file": file}
elif block["source_type"] == "id":
formatted_block = {"type": "file", "file": {"file_id": block["id"]}}
else:
raise ValueError("source_type base64 or id is required for file blocks.")
elif block["type"] == "audio":
if block["source_type"] == "base64":
format = block["mime_type"].split("/")[-1]
formatted_block = {
"type": "input_audio",
"input_audio": {"data": block["data"], "format": format},
}
else:
raise ValueError("source_type base64 is required for audio blocks.")
else:
raise ValueError(f"Block of type {block['type']} is not supported.")
return formatted_block
def _format_message_content(content: Any) -> Any:
"""Format message content."""
if content and isinstance(content, list):
@@ -238,7 +199,7 @@ def _format_message_content(content: Any) -> Any:
):
continue
elif isinstance(block, dict) and is_data_content_block(block):
formatted_content.append(_format_data_content_block(block))
formatted_content.append(convert_to_openai_data_block(block))
# Anthropic image blocks
elif (
isinstance(block, dict)