core: support both old and new data content blocks

libs/core/langchain_core/messages/content_blocks.py

@@ -661,25 +661,28 @@ def is_data_content_block(block: dict) -> bool:
     )
 
 
-# TODO: don't use `source_type` anymore
 def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict:
     """Convert image content block to format expected by OpenAI Chat Completions API."""
-    if content_block["source_type"] == "url":
+    if "url" in content_block:
         return {
             "type": "image_url",
             "image_url": {
                 "url": content_block["url"],
             },
         }
-    if content_block["source_type"] == "base64":
+    if "base64" in content_block or content_block.get("source_type") == "base64":
         if "mime_type" not in content_block:
             error_message = "mime_type key is required for base64 data."
             raise ValueError(error_message)
         mime_type = content_block["mime_type"]
+        if "data" in content_block:  # Backwards compatibility
+            base64_data = content_block["data"]
+        else:
+            base64_data = content_block["base64"]
         return {
             "type": "image_url",
             "image_url": {
-                "url": f"data:{mime_type};base64,{content_block['data']}",
+                "url": f"data:{mime_type};base64,{base64_data}",
             },
         }
     error_message = "Unsupported source type. Only 'url' and 'base64' are supported."
@@ -692,8 +695,9 @@ def convert_to_openai_data_block(block: dict) -> dict:
         formatted_block = convert_to_openai_image_block(block)
 
     elif block["type"] == "file":
-        if block["source_type"] == "base64":
+        if "base64" in block or block.get("source_type") == "base64":
-            file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}
+            base64_data = block["data"] if "source_type" in block else block["base64"]
+            file = {"file_data": f"data:{block['mime_type']};base64,{base64_data}"}
             if filename := block.get("filename"):
                 file["filename"] = filename
             elif (metadata := block.get("metadata")) and ("filename" in metadata):
@@ -701,27 +705,28 @@ def convert_to_openai_data_block(block: dict) -> dict:
             else:
                 warnings.warn(
                     "OpenAI may require a filename for file inputs. Specify a filename "
-                    "in the content block: {'type': 'file', 'source_type': 'base64', "
+                    "in the content block: {'type': 'file', 'mime_type': "
-                    "'mime_type': 'application/pdf', 'data': '...', "
+                    "'application/pdf', 'base64': '...', 'filename': 'my-pdf'}",
-                    "'filename': 'my-pdf'}",
                     stacklevel=1,
                 )
             formatted_block = {"type": "file", "file": file}
-        elif block["source_type"] == "id":
+        elif "file_id" in block or block.get("source_type") == "id":
-            formatted_block = {"type": "file", "file": {"file_id": block["id"]}}
+            file_id = block["id"] if "source_type" in block else block["file_id"]
+            formatted_block = {"type": "file", "file": {"file_id": file_id}}
         else:
-            error_msg = "source_type base64 or id is required for file blocks."
+            error_msg = "Keys base64 or file_id required for file blocks."
             raise ValueError(error_msg)
 
     elif block["type"] == "audio":
-        if block["source_type"] == "base64":
+        if "base64" in block or block.get("source_type") == "base64":
+            base64_data = block["data"] if "source_type" in block else block["base64"]
             audio_format = block["mime_type"].split("/")[-1]
             formatted_block = {
                 "type": "input_audio",
-                "input_audio": {"data": block["data"], "format": audio_format},
+                "input_audio": {"data": base64_data, "format": audio_format},
             }
         else:
-            error_msg = "source_type base64 is required for audio blocks."
+            error_msg = "Key base64 is required for audio blocks."
             raise ValueError(error_msg)
     else:
         error_msg = f"Block of type {block['type']} is not supported."

libs/core/tests/unit_tests/messages/test_utils.py

@@ -1194,15 +1194,30 @@ def test_convert_to_openai_messages_multimodal() -> None:
                 {"type": "text", "text": "Text message"},
                 {
                     "type": "image",
-                    "source_type": "url",
                     "url": "https://example.com/test.png",
                 },
+                {
+                    "type": "image",
+                    "source_type": "url",  # backward compatibility
+                    "url": "https://example.com/test.png",
+                },
+                {
+                    "type": "image",
+                    "base64": "<base64 string>",
+                    "mime_type": "image/png",
+                },
                 {
                     "type": "image",
                     "source_type": "base64",
                     "data": "<base64 string>",
                     "mime_type": "image/png",
                 },
+                {
+                    "type": "file",
+                    "base64": "<base64 string>",
+                    "mime_type": "application/pdf",
+                    "filename": "test.pdf",
+                },
                 {
                     "type": "file",
                     "source_type": "base64",
@@ -1217,11 +1232,20 @@ def test_convert_to_openai_messages_multimodal() -> None:
                         "file_data": "data:application/pdf;base64,<base64 string>",
                     },
                 },
+                {
+                    "type": "file",
+                    "file_id": "file-abc123",
+                },
                 {
                     "type": "file",
                     "source_type": "id",
                     "id": "file-abc123",
                 },
+                {
+                    "type": "audio",
+                    "base64": "<base64 string>",
+                    "mime_type": "audio/wav",
+                },
                 {
                     "type": "audio",
                     "source_type": "base64",
@@ -1241,7 +1265,7 @@ def test_convert_to_openai_messages_multimodal() -> None:
     result = convert_to_openai_messages(messages, text_format="block")
     assert len(result) == 1
     message = result[0]
-    assert len(message["content"]) == 8
+    assert len(message["content"]) == 13
 
     # Test adding filename
     messages = [
@@ -1249,8 +1273,7 @@ def test_convert_to_openai_messages_multimodal() -> None:
             content=[
                 {
                     "type": "file",
-                    "source_type": "base64",
+                    "base64": "<base64 string>",
-                    "data": "<base64 string>",
                     "mime_type": "application/pdf",
                 },
             ]

libs/core/tests/unit_tests/test_messages.py

@@ -1165,34 +1165,49 @@ def test_is_data_content_block() -> None:
 
 
 def test_convert_to_openai_image_block() -> None:
-    input_block = {
+    for input_block in [
-        "type": "image",
+        {
-        "source_type": "url",
+            "type": "image",
-        "url": "https://...",
+            "url": "https://...",
-        "cache_control": {"type": "ephemeral"},
+            "cache_control": {"type": "ephemeral"},
-    }
-    expected = {
-        "type": "image_url",
-        "image_url": {"url": "https://..."},
-    }
-    result = convert_to_openai_image_block(input_block)
-    assert result == expected
-
-    input_block = {
-        "type": "image",
-        "source_type": "base64",
-        "data": "<base64 data>",
-        "mime_type": "image/jpeg",
-        "cache_control": {"type": "ephemeral"},
-    }
-    expected = {
-        "type": "image_url",
-        "image_url": {
-            "url": "data:image/jpeg;base64,<base64 data>",
         },
-    }
+        {
-    result = convert_to_openai_image_block(input_block)
+            "type": "image",
-    assert result == expected
+            "source_type": "url",
+            "url": "https://...",
+            "cache_control": {"type": "ephemeral"},
+        },
+    ]:
+        expected = {
+            "type": "image_url",
+            "image_url": {"url": "https://..."},
+        }
+        result = convert_to_openai_image_block(input_block)
+        assert result == expected
+
+    for input_block in [
+        {
+            "type": "image",
+            "base64": "<base64 data>",
+            "mime_type": "image/jpeg",
+            "cache_control": {"type": "ephemeral"},
+        },
+        {
+            "type": "image",
+            "source_type": "base64",
+            "data": "<base64 data>",
+            "mime_type": "image/jpeg",
+            "cache_control": {"type": "ephemeral"},
+        },
+    ]:
+        expected = {
+            "type": "image_url",
+            "image_url": {
+                "url": "data:image/jpeg;base64,<base64 data>",
+            },
+        }
+        result = convert_to_openai_image_block(input_block)
+        assert result == expected
 
 
 def test_known_block_types() -> None: