openai: audio modality, remove sockets from unit tests (#27436)

libs/partners/openai/tests/integration_tests/chat_models/test_base.py

@@ -2,6 +2,7 @@
 
 import base64
 import json
+from pathlib import Path
 from typing import Any, AsyncIterator, List, Literal, Optional, cast
 
 import httpx
@@ -949,3 +950,71 @@ async def test_json_mode_async() -> None:
     assert isinstance(full, AIMessageChunk)
     assert isinstance(full.content, str)
     assert json.loads(full.content) == {"a": 1}
+
+
+def test_audio_output_modality() -> None:
+    llm = ChatOpenAI(
+        model="gpt-4o-audio-preview",
+        temperature=0,
+        model_kwargs={
+            "modalities": ["text", "audio"],
+            "audio": {"voice": "alloy", "format": "wav"},
+        },
+    )
+
+    history: List[BaseMessage] = [
+        HumanMessage("Make me a short audio clip of you yelling")
+    ]
+
+    output = llm.invoke(history)
+
+    assert isinstance(output, AIMessage)
+    assert "audio" in output.additional_kwargs
+
+    history.append(output)
+    history.append(HumanMessage("Make me a short audio clip of you whispering"))
+
+    output = llm.invoke(history)
+
+    assert isinstance(output, AIMessage)
+    assert "audio" in output.additional_kwargs
+
+
+def test_audio_input_modality() -> None:
+    llm = ChatOpenAI(
+        model="gpt-4o-audio-preview",
+        temperature=0,
+        model_kwargs={
+            "modalities": ["text", "audio"],
+            "audio": {"voice": "alloy", "format": "wav"},
+        },
+    )
+    filepath = Path(__file__).parent / "audio_input.wav"
+
+    audio_data = filepath.read_bytes()
+    b64_audio_data = base64.b64encode(audio_data).decode("utf-8")
+
+    history: list[BaseMessage] = [
+        HumanMessage(
+            [
+                {"type": "text", "text": "What is happening in this audio clip"},
+                {
+                    "type": "input_audio",
+                    "input_audio": {"data": b64_audio_data, "format": "wav"},
+                },
+            ]
+        )
+    ]
+
+    output = llm.invoke(history)
+
+    assert isinstance(output, AIMessage)
+    assert "audio" in output.additional_kwargs
+
+    history.append(output)
+    history.append(HumanMessage("Why?"))
+
+    output = llm.invoke(history)
+
+    assert isinstance(output, AIMessage)
+    assert "audio" in output.additional_kwargs

libs/partners/openai/tests/unit_tests/chat_models/test_base.py

@@ -162,7 +162,12 @@ def test__convert_dict_to_message_tool_call() -> None:
                 name="GenerateUsername",
                 args="oops",
                 id="call_wm0JY6CdwOMZ4eTxHWUThDNz",
-                error="Function GenerateUsername arguments:\n\noops\n\nare not valid JSON. Received JSONDecodeError Expecting value: line 1 column 1 (char 0)",  # noqa: E501
+                error=(
+                    "Function GenerateUsername arguments:\n\noops\n\nare not "
+                    "valid JSON. Received JSONDecodeError Expecting value: line 1 "
+                    "column 1 (char 0)\nFor troubleshooting, visit: https://python"
+                    ".langchain.com/docs/troubleshooting/errors/OUTPUT_PARSING_FAILURE"
+                ),
                 type="invalid_tool_call",
             )
         ],

libs/partners/openai/tests/unit_tests/llms/test_base.py

@@ -58,7 +58,7 @@ def mock_completion() -> dict:
     }
 
 
-@pytest.mark.parametrize("model", ["gpt-3.5-turbo-instruct", "text-davinci-003"])
+@pytest.mark.parametrize("model", ["gpt-3.5-turbo-instruct"])
 def test_get_token_ids(model: str) -> None:
     OpenAI(model=model).get_token_ids("foo")
     return

libs/partners/openai/tests/unit_tests/test_token_counts.py

@@ -16,6 +16,7 @@ _MODELS = models = ["ada", "babbage", "curie", "davinci"]
 _CHAT_MODELS = ["gpt-4", "gpt-4-32k", "gpt-3.5-turbo"]
 
 
+@pytest.mark.xfail(reason="Old models require different tiktoken cached file")
 @pytest.mark.parametrize("model", _MODELS)
 def test_openai_get_num_tokens(model: str) -> None:
     """Test get_tokens."""