diff --git a/libs/core/langchain_core/messages/content_blocks.py b/libs/core/langchain_core/messages/content_blocks.py index 5c2e881cf2b..d5696251ed3 100644 --- a/libs/core/langchain_core/messages/content_blocks.py +++ b/libs/core/langchain_core/messages/content_blocks.py @@ -661,25 +661,28 @@ def is_data_content_block(block: dict) -> bool: ) -# TODO: don't use `source_type` anymore def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict: """Convert image content block to format expected by OpenAI Chat Completions API.""" - if content_block["source_type"] == "url": + if "url" in content_block: return { "type": "image_url", "image_url": { "url": content_block["url"], }, } - if content_block["source_type"] == "base64": + if "base64" in content_block or content_block.get("source_type") == "base64": if "mime_type" not in content_block: error_message = "mime_type key is required for base64 data." raise ValueError(error_message) mime_type = content_block["mime_type"] + if "data" in content_block: # Backwards compatibility + base64_data = content_block["data"] + else: + base64_data = content_block["base64"] return { "type": "image_url", "image_url": { - "url": f"data:{mime_type};base64,{content_block['data']}", + "url": f"data:{mime_type};base64,{base64_data}", }, } error_message = "Unsupported source type. Only 'url' and 'base64' are supported." @@ -692,8 +695,9 @@ def convert_to_openai_data_block(block: dict) -> dict: formatted_block = convert_to_openai_image_block(block) elif block["type"] == "file": - if block["source_type"] == "base64": - file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"} + if "base64" in block or block.get("source_type") == "base64": + base64_data = block["data"] if "source_type" in block else block["base64"] + file = {"file_data": f"data:{block['mime_type']};base64,{base64_data}"} if filename := block.get("filename"): file["filename"] = filename elif (metadata := block.get("metadata")) and ("filename" in metadata): @@ -701,27 +705,28 @@ def convert_to_openai_data_block(block: dict) -> dict: else: warnings.warn( "OpenAI may require a filename for file inputs. Specify a filename " - "in the content block: {'type': 'file', 'source_type': 'base64', " - "'mime_type': 'application/pdf', 'data': '...', " - "'filename': 'my-pdf'}", + "in the content block: {'type': 'file', 'mime_type': " + "'application/pdf', 'base64': '...', 'filename': 'my-pdf'}", stacklevel=1, ) formatted_block = {"type": "file", "file": file} - elif block["source_type"] == "id": - formatted_block = {"type": "file", "file": {"file_id": block["id"]}} + elif "file_id" in block or block.get("source_type") == "id": + file_id = block["id"] if "source_type" in block else block["file_id"] + formatted_block = {"type": "file", "file": {"file_id": file_id}} else: - error_msg = "source_type base64 or id is required for file blocks." + error_msg = "Keys base64 or file_id required for file blocks." raise ValueError(error_msg) elif block["type"] == "audio": - if block["source_type"] == "base64": + if "base64" in block or block.get("source_type") == "base64": + base64_data = block["data"] if "source_type" in block else block["base64"] audio_format = block["mime_type"].split("/")[-1] formatted_block = { "type": "input_audio", - "input_audio": {"data": block["data"], "format": audio_format}, + "input_audio": {"data": base64_data, "format": audio_format}, } else: - error_msg = "source_type base64 is required for audio blocks." + error_msg = "Key base64 is required for audio blocks." raise ValueError(error_msg) else: error_msg = f"Block of type {block['type']} is not supported." diff --git a/libs/core/tests/unit_tests/messages/test_utils.py b/libs/core/tests/unit_tests/messages/test_utils.py index 0898341dda8..19ff670b39e 100644 --- a/libs/core/tests/unit_tests/messages/test_utils.py +++ b/libs/core/tests/unit_tests/messages/test_utils.py @@ -1194,15 +1194,30 @@ def test_convert_to_openai_messages_multimodal() -> None: {"type": "text", "text": "Text message"}, { "type": "image", - "source_type": "url", "url": "https://example.com/test.png", }, + { + "type": "image", + "source_type": "url", # backward compatibility + "url": "https://example.com/test.png", + }, + { + "type": "image", + "base64": "", + "mime_type": "image/png", + }, { "type": "image", "source_type": "base64", "data": "", "mime_type": "image/png", }, + { + "type": "file", + "base64": "", + "mime_type": "application/pdf", + "filename": "test.pdf", + }, { "type": "file", "source_type": "base64", @@ -1217,11 +1232,20 @@ def test_convert_to_openai_messages_multimodal() -> None: "file_data": "data:application/pdf;base64,", }, }, + { + "type": "file", + "file_id": "file-abc123", + }, { "type": "file", "source_type": "id", "id": "file-abc123", }, + { + "type": "audio", + "base64": "", + "mime_type": "audio/wav", + }, { "type": "audio", "source_type": "base64", @@ -1241,7 +1265,7 @@ def test_convert_to_openai_messages_multimodal() -> None: result = convert_to_openai_messages(messages, text_format="block") assert len(result) == 1 message = result[0] - assert len(message["content"]) == 8 + assert len(message["content"]) == 13 # Test adding filename messages = [ @@ -1249,8 +1273,7 @@ def test_convert_to_openai_messages_multimodal() -> None: content=[ { "type": "file", - "source_type": "base64", - "data": "", + "base64": "", "mime_type": "application/pdf", }, ] diff --git a/libs/core/tests/unit_tests/test_messages.py b/libs/core/tests/unit_tests/test_messages.py index 5a9b73e7b83..85f5e585a5d 100644 --- a/libs/core/tests/unit_tests/test_messages.py +++ b/libs/core/tests/unit_tests/test_messages.py @@ -1165,34 +1165,49 @@ def test_is_data_content_block() -> None: def test_convert_to_openai_image_block() -> None: - input_block = { - "type": "image", - "source_type": "url", - "url": "https://...", - "cache_control": {"type": "ephemeral"}, - } - expected = { - "type": "image_url", - "image_url": {"url": "https://..."}, - } - result = convert_to_openai_image_block(input_block) - assert result == expected - - input_block = { - "type": "image", - "source_type": "base64", - "data": "", - "mime_type": "image/jpeg", - "cache_control": {"type": "ephemeral"}, - } - expected = { - "type": "image_url", - "image_url": { - "url": "data:image/jpeg;base64,", + for input_block in [ + { + "type": "image", + "url": "https://...", + "cache_control": {"type": "ephemeral"}, }, - } - result = convert_to_openai_image_block(input_block) - assert result == expected + { + "type": "image", + "source_type": "url", + "url": "https://...", + "cache_control": {"type": "ephemeral"}, + }, + ]: + expected = { + "type": "image_url", + "image_url": {"url": "https://..."}, + } + result = convert_to_openai_image_block(input_block) + assert result == expected + + for input_block in [ + { + "type": "image", + "base64": "", + "mime_type": "image/jpeg", + "cache_control": {"type": "ephemeral"}, + }, + { + "type": "image", + "source_type": "base64", + "data": "", + "mime_type": "image/jpeg", + "cache_control": {"type": "ephemeral"}, + }, + ]: + expected = { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,", + }, + } + result = convert_to_openai_image_block(input_block) + assert result == expected def test_known_block_types() -> None: