From 4c23ceb9efed6de8b384e32fcbc8afe51f3bb382 Mon Sep 17 00:00:00 2001
From: Chester Curme <chester.curme@gmail.com>
Date: Tue, 8 Apr 2025 18:54:36 -0400
Subject: [PATCH] update openai

---
 .../langchain_openai/chat_models/base.py      | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py
index 8eb36f2bbd8..c4ed87118c2 100644
--- a/libs/partners/openai/langchain_openai/chat_models/base.py
+++ b/libs/partners/openai/langchain_openai/chat_models/base.py
@@ -68,6 +68,7 @@ from langchain_core.messages import (
     ToolCall,
     ToolMessage,
     ToolMessageChunk,
+    is_data_content_block,
 )
 from langchain_core.messages.ai import (
     InputTokenDetails,
@@ -191,6 +192,45 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:
         return ChatMessage(content=_dict.get("content", ""), role=role, id=id_)  # type: ignore[arg-type]
 
 
+def _format_data_content_block(block: dict) -> dict:
+    """Format standard data content block to format expected by OpenAI."""
+    if block["type"] == "image":
+        if block["source_type"] == "url":
+            formatted_block = {
+                "type": "image_url",
+                "image_url": {"url": block["source"]},
+            }
+        elif block["source_type"] == "base64":
+            formatted_block = {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:{block['mime_type']};base64,{block['source']}"
+                },
+            }
+        else:
+            raise ValueError(
+                "OpenAI only supports 'url' and 'base64' source_type for image "
+                "content blocks."
+            )
+
+    elif block["type"] == "file":
+        if block["source_type"] == "base64":
+            file = {"file_data": f"data:{block['mime_type']};base64,{block['source']}"}
+            if metadata := block.get("metadata"):
+                file = {**file, **metadata}
+            # Hack to support cross-compatibility with providers that do not require
+            # filename (OpenAI requires one).
+            if "filename" not in file:
+                file["filename"] = ""
+            formatted_block = {"type": "file", "file": file}
+        elif block["source_type"] == "id":
+            formatted_block = {"type": "file", "file": {"file_id": block["source"]}}
+    else:
+        raise ValueError(f"Block of type {block['type']} is not supported.")
+
+    return formatted_block
+
+
 def _format_message_content(content: Any) -> Any:
     """Format message content."""
     if content and isinstance(content, list):
@@ -203,6 +243,8 @@ def _format_message_content(content: Any) -> Any:
                 and block["type"] in ("tool_use", "thinking")
             ):
                 continue
+            elif is_data_content_block(block):
+                formatted_content.append(_format_data_content_block(block))
             # Anthropic image blocks
             elif (
                 isinstance(block, dict)