mirror of
https://github.com/hwchase17/langchain.git
synced 2025-08-17 16:39:52 +00:00
core: support both old and new data content blocks
This commit is contained in:
Chester Curme
parent
e1f034c795
commit
6572656cd2
@ -661,25 +661,28 @@ def is_data_content_block(block: dict) -> bool:
|
||||
)
|
||||
|
||||
|
||||
# TODO: don't use `source_type` anymore
|
||||
def convert_to_openai_image_block(content_block: dict[str, Any]) -> dict:
|
||||
"""Convert image content block to format expected by OpenAI Chat Completions API."""
|
||||
if content_block["source_type"] == "url":
|
||||
if "url" in content_block:
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": content_block["url"],
|
||||
},
|
||||
}
|
||||
if content_block["source_type"] == "base64":
|
||||
if "base64" in content_block or content_block.get("source_type") == "base64":
|
||||
if "mime_type" not in content_block:
|
||||
error_message = "mime_type key is required for base64 data."
|
||||
raise ValueError(error_message)
|
||||
mime_type = content_block["mime_type"]
|
||||
if "data" in content_block: # Backwards compatibility
|
||||
base64_data = content_block["data"]
|
||||
else:
|
||||
base64_data = content_block["base64"]
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:{mime_type};base64,{content_block['data']}",
|
||||
"url": f"data:{mime_type};base64,{base64_data}",
|
||||
},
|
||||
}
|
||||
error_message = "Unsupported source type. Only 'url' and 'base64' are supported."
|
||||
@ -692,8 +695,9 @@ def convert_to_openai_data_block(block: dict) -> dict:
|
||||
formatted_block = convert_to_openai_image_block(block)
|
||||
|
||||
elif block["type"] == "file":
|
||||
if block["source_type"] == "base64":
|
||||
file = {"file_data": f"data:{block['mime_type']};base64,{block['data']}"}
|
||||
if "base64" in block or block.get("source_type") == "base64":
|
||||
base64_data = block["data"] if "source_type" in block else block["base64"]
|
||||
file = {"file_data": f"data:{block['mime_type']};base64,{base64_data}"}
|
||||
if filename := block.get("filename"):
|
||||
file["filename"] = filename
|
||||
elif (metadata := block.get("metadata")) and ("filename" in metadata):
|
||||
@ -701,27 +705,28 @@ def convert_to_openai_data_block(block: dict) -> dict:
|
||||
else:
|
||||
warnings.warn(
|
||||
"OpenAI may require a filename for file inputs. Specify a filename "
|
||||
"in the content block: {'type': 'file', 'source_type': 'base64', "
|
||||
"'mime_type': 'application/pdf', 'data': '...', "
|
||||
"'filename': 'my-pdf'}",
|
||||
"in the content block: {'type': 'file', 'mime_type': "
|
||||
"'application/pdf', 'base64': '...', 'filename': 'my-pdf'}",
|
||||
stacklevel=1,
|
||||
)
|
||||
formatted_block = {"type": "file", "file": file}
|
||||
elif block["source_type"] == "id":
|
||||
formatted_block = {"type": "file", "file": {"file_id": block["id"]}}
|
||||
elif "file_id" in block or block.get("source_type") == "id":
|
||||
file_id = block["id"] if "source_type" in block else block["file_id"]
|
||||
formatted_block = {"type": "file", "file": {"file_id": file_id}}
|
||||
else:
|
||||
error_msg = "source_type base64 or id is required for file blocks."
|
||||
error_msg = "Keys base64 or file_id required for file blocks."
|
||||
raise ValueError(error_msg)
|
||||
|
||||
elif block["type"] == "audio":
|
||||
if block["source_type"] == "base64":
|
||||
if "base64" in block or block.get("source_type") == "base64":
|
||||
base64_data = block["data"] if "source_type" in block else block["base64"]
|
||||
audio_format = block["mime_type"].split("/")[-1]
|
||||
formatted_block = {
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": block["data"], "format": audio_format},
|
||||
"input_audio": {"data": base64_data, "format": audio_format},
|
||||
}
|
||||
else:
|
||||
error_msg = "source_type base64 is required for audio blocks."
|
||||
error_msg = "Key base64 is required for audio blocks."
|
||||
raise ValueError(error_msg)
|
||||
else:
|
||||
error_msg = f"Block of type {block['type']} is not supported."
|
||||
|
||||
@ -1194,15 +1194,30 @@ def test_convert_to_openai_messages_multimodal() -> None:
|
||||
{"type": "text", "text": "Text message"},
|
||||
{
|
||||
"type": "image",
|
||||
"source_type": "url",
|
||||
"url": "https://example.com/test.png",
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"source_type": "url", # backward compatibility
|
||||
"url": "https://example.com/test.png",
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"base64": "<base64 string>",
|
||||
"mime_type": "image/png",
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"source_type": "base64",
|
||||
"data": "<base64 string>",
|
||||
"mime_type": "image/png",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"base64": "<base64 string>",
|
||||
"mime_type": "application/pdf",
|
||||
"filename": "test.pdf",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"source_type": "base64",
|
||||
@ -1217,11 +1232,20 @@ def test_convert_to_openai_messages_multimodal() -> None:
|
||||
"file_data": "data:application/pdf;base64,<base64 string>",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"file_id": "file-abc123",
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"source_type": "id",
|
||||
"id": "file-abc123",
|
||||
},
|
||||
{
|
||||
"type": "audio",
|
||||
"base64": "<base64 string>",
|
||||
"mime_type": "audio/wav",
|
||||
},
|
||||
{
|
||||
"type": "audio",
|
||||
"source_type": "base64",
|
||||
@ -1241,7 +1265,7 @@ def test_convert_to_openai_messages_multimodal() -> None:
|
||||
result = convert_to_openai_messages(messages, text_format="block")
|
||||
assert len(result) == 1
|
||||
message = result[0]
|
||||
assert len(message["content"]) == 8
|
||||
assert len(message["content"]) == 13
|
||||
|
||||
# Test adding filename
|
||||
messages = [
|
||||
@ -1249,8 +1273,7 @@ def test_convert_to_openai_messages_multimodal() -> None:
|
||||
content=[
|
||||
{
|
||||
"type": "file",
|
||||
"source_type": "base64",
|
||||
"data": "<base64 string>",
|
||||
"base64": "<base64 string>",
|
||||
"mime_type": "application/pdf",
|
||||
},
|
||||
]
|
||||
|
||||
@ -1165,12 +1165,19 @@ def test_is_data_content_block() -> None:
|
||||
|
||||
|
||||
def test_convert_to_openai_image_block() -> None:
|
||||
input_block = {
|
||||
for input_block in [
|
||||
{
|
||||
"type": "image",
|
||||
"url": "https://...",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"source_type": "url",
|
||||
"url": "https://...",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
},
|
||||
]:
|
||||
expected = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://..."},
|
||||
@ -1178,13 +1185,21 @@ def test_convert_to_openai_image_block() -> None:
|
||||
result = convert_to_openai_image_block(input_block)
|
||||
assert result == expected
|
||||
|
||||
input_block = {
|
||||
for input_block in [
|
||||
{
|
||||
"type": "image",
|
||||
"base64": "<base64 data>",
|
||||
"mime_type": "image/jpeg",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"source_type": "base64",
|
||||
"data": "<base64 data>",
|
||||
"mime_type": "image/jpeg",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
},
|
||||
]:
|
||||
expected = {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user