standard-tests, openai[patch]: add support standard audio inputs (#30904)

libs/partners/openai/langchain_openai/chat_models/base.py

@@ -208,6 +208,17 @@ def _format_data_content_block(block: dict) -> dict:
             formatted_block = {"type": "file", "file": file}
         elif block["source_type"] == "id":
             formatted_block = {"type": "file", "file": {"file_id": block["id"]}}
+        else:
+            raise ValueError("source_type base64 or id is required for file blocks.")
+    elif block["type"] == "audio":
+        if block["source_type"] == "base64":
+            format = block["mime_type"].split("/")[-1]
+            formatted_block = {
+                "type": "input_audio",
+                "input_audio": {"data": block["data"], "format": format},
+            }
+        else:
+            raise ValueError("source_type base64 is required for audio blocks.")
     else:
         raise ValueError(f"Block of type {block['type']} is not supported.")
 

libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from typing import Literal, cast
 
 import httpx
+import pytest
 from langchain_core.language_models import BaseChatModel
 from langchain_core.messages import AIMessage, HumanMessage
 from langchain_tests.integration_tests import ChatModelIntegrationTests
@@ -111,3 +112,30 @@ def _invoke(llm: ChatOpenAI, input_: str, stream: bool) -> AIMessage:
         return cast(AIMessage, full)
     else:
         return cast(AIMessage, llm.invoke(input_))
+
+
+@pytest.mark.skip()  # Test either finishes in 5 seconds or 5 minutes.
+def test_audio_model() -> None:
+    class AudioModelTests(ChatModelIntegrationTests):
+        @property
+        def chat_model_class(self) -> type[ChatOpenAI]:
+            return ChatOpenAI
+
+        @property
+        def chat_model_params(self) -> dict:
+            return {
+                "model": "gpt-4o-audio-preview",
+                "temperature": 0,
+                "model_kwargs": {
+                    "modalities": ["text", "audio"],
+                    "audio": {"voice": "alloy", "format": "wav"},
+                },
+            }
+
+        @property
+        def supports_audio_inputs(self) -> bool:
+            return True
+
+    test_instance = AudioModelTests()
+    model = test_instance.chat_model_class(**test_instance.chat_model_params)
+    AudioModelTests().test_audio_inputs(model)