diff --git a/libs/partners/openai/langchain_openai/chat_models/_compat.py b/libs/partners/openai/langchain_openai/chat_models/_compat.py new file mode 100644 index 00000000000..969fc192daa --- /dev/null +++ b/libs/partners/openai/langchain_openai/chat_models/_compat.py @@ -0,0 +1,232 @@ +""" +This module converts between AIMessage output formats for the Responses API. + +ChatOpenAI v0.3 stores reasoning and tool outputs in AIMessage.additional_kwargs: + +.. code-block:: python + + AIMessage( + content=[ + {"type": "text", "text": "Hello, world!", "annotations": [{"type": "foo"}]} + ], + additional_kwargs={ + "reasoning": { + "type": "reasoning", + "id": "rs_123", + "summary": [{"type": "summary_text", "text": "Reasoning summary"}], + }, + "tool_outputs": [ + {"type": "web_search_call", "id": "websearch_123", "status": "completed"} + ], + "refusal": "I cannot assist with that.", + }, + response_metadata={"id": "resp_123"}, + id="msg_123", + ) + +To retain information about response item sequencing (and to accommodate multiple +reasoning items), ChatOpenAI now stores these items in the content sequence: + +.. code-block:: python + + AIMessage( + content=[ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Reasoning summary"}], + "id": "rs_123", + }, + { + "type": "text", + "text": "Hello, world!", + "annotations": [{"type": "foo"}], + "id": "msg_123", + }, + {"type": "refusal", "refusal": "I cannot assist with that."}, + {"type": "web_search_call", "id": "websearch_123", "status": "completed"}, + ], + response_metadata={"id": "resp_123"}, + id="resp_123", + ) + +There are other, small improvements as well-- e.g., we store message IDs on text +content blocks, rather than on the AIMessage.id, which now stores the response ID. + +For backwards compatibility, this module provides functions to convert between the +old and new formats. The functions are used internally by ChatOpenAI. +""" # noqa: E501 + +import json +from typing import Union + +from langchain_core.messages import AIMessage + +_FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__" + + +def _convert_to_v03_ai_message( + message: AIMessage, has_reasoning: bool = False +) -> AIMessage: + """Mutate an AIMessage to the old-style v0.3 format.""" + if isinstance(message.content, list): + new_content: list[Union[dict, str]] = [] + for block in message.content: + if isinstance(block, dict): + if block.get("type") == "reasoning" or "summary" in block: + # Store a reasoning item in additional_kwargs (overwriting as in + # v0.3) + _ = block.pop("index", None) + if has_reasoning: + _ = block.pop("id", None) + _ = block.pop("type", None) + message.additional_kwargs["reasoning"] = block + elif block.get("type") in ( + "web_search_call", + "file_search_call", + "computer_call", + "code_interpreter_call", + "mcp_call", + "mcp_list_tools", + "mcp_approval_request", + "image_generation_call", + ): + # Store built-in tool calls in additional_kwargs + if "tool_outputs" not in message.additional_kwargs: + message.additional_kwargs["tool_outputs"] = [] + message.additional_kwargs["tool_outputs"].append(block) + elif block.get("type") == "function_call": + # Store function call item IDs in additional_kwargs, otherwise + # discard function call items. + if _FUNCTION_CALL_IDS_MAP_KEY not in message.additional_kwargs: + message.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {} + if (call_id := block.get("call_id")) and ( + function_call_id := block.get("id") + ): + message.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][ + call_id + ] = function_call_id + elif (block.get("type") == "refusal") and ( + refusal := block.get("refusal") + ): + # Store a refusal item in additional_kwargs (overwriting as in + # v0.3) + message.additional_kwargs["refusal"] = refusal + elif block.get("type") == "text": + # Store a message item ID on AIMessage.id + if "id" in block: + message.id = block["id"] + new_content.append({k: v for k, v in block.items() if k != "id"}) + elif ( + set(block.keys()) == {"id", "index"} + and isinstance(block["id"], str) + and block["id"].startswith("msg_") + ): + # Drop message IDs in streaming case + new_content.append({"index": block["index"]}) + else: + new_content.append(block) + else: + new_content.append(block) + message.content = new_content + else: + pass + + return message + + +def _convert_from_v03_ai_message(message: AIMessage) -> AIMessage: + """Convert an old-style v0.3 AIMessage into the new content-block format.""" + # Only update ChatOpenAI v0.3 AIMessages + if not ( + isinstance(message.content, list) + and all(isinstance(b, dict) for b in message.content) + ) or not any( + item in message.additional_kwargs + for item in ["reasoning", "tool_outputs", "refusal"] + ): + return message + + content_order = [ + "reasoning", + "code_interpreter_call", + "mcp_call", + "image_generation_call", + "text", + "refusal", + "function_call", + "computer_call", + "mcp_list_tools", + "mcp_approval_request", + # N. B. "web_search_call" and "file_search_call" were not passed back in + # in v0.3 + ] + + # Build a bucket for every known block type + buckets: dict[str, list] = {key: [] for key in content_order} + unknown_blocks = [] + + # Reasoning + if reasoning := message.additional_kwargs.get("reasoning"): + buckets["reasoning"].append(reasoning) + + # Refusal + if refusal := message.additional_kwargs.get("refusal"): + buckets["refusal"].append({"type": "refusal", "refusal": refusal}) + + # Text + for block in message.content: + if isinstance(block, dict) and block.get("type") == "text": + block_copy = block.copy() + if isinstance(message.id, str) and message.id.startswith("msg_"): + block_copy["id"] = message.id + buckets["text"].append(block_copy) + else: + unknown_blocks.append(block) + + # Function calls + function_call_ids = message.additional_kwargs.get(_FUNCTION_CALL_IDS_MAP_KEY) + for tool_call in message.tool_calls: + function_call = { + "type": "function_call", + "name": tool_call["name"], + "arguments": json.dumps(tool_call["args"]), + "call_id": tool_call["id"], + } + if function_call_ids is not None and ( + _id := function_call_ids.get(tool_call["id"]) + ): + function_call["id"] = _id + buckets["function_call"].append(function_call) + + # Tool outputs + tool_outputs = message.additional_kwargs.get("tool_outputs", []) + for block in tool_outputs: + if isinstance(block, dict) and (key := block.get("type")) and key in buckets: + buckets[key].append(block) + else: + unknown_blocks.append(block) + + # Re-assemble the content list in the canonical order + new_content = [] + for key in content_order: + new_content.extend(buckets[key]) + new_content.extend(unknown_blocks) + + new_additional_kwargs = dict(message.additional_kwargs) + new_additional_kwargs.pop("reasoning", None) + new_additional_kwargs.pop("refusal", None) + new_additional_kwargs.pop("tool_outputs", None) + + if "id" in message.response_metadata: + new_id = message.response_metadata["id"] + else: + new_id = message.id + + return message.model_copy( + update={ + "content": new_content, + "additional_kwargs": new_additional_kwargs, + "id": new_id, + }, + deep=False, + ) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 9d649a7b8e2..a17f3cc27aa 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -106,6 +106,10 @@ from langchain_openai.chat_models._client_utils import ( _get_default_async_httpx_client, _get_default_httpx_client, ) +from langchain_openai.chat_models._compat import ( + _convert_from_v03_ai_message, + _convert_to_v03_ai_message, +) if TYPE_CHECKING: from openai.types.responses import Response @@ -116,8 +120,6 @@ logger = logging.getLogger(__name__) # https://www.python-httpx.org/advanced/ssl/#configuring-client-instances global_ssl_context = ssl.create_default_context(cafile=certifi.where()) -_FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__" - WellKnownTools = ( "file_search", "web_search_preview", @@ -797,15 +799,27 @@ class BaseChatOpenAI(BaseChatModel): with context_manager as response: is_first_chunk = True + current_index = -1 + current_output_index = -1 + current_sub_index = -1 has_reasoning = False for chunk in response: metadata = headers if is_first_chunk else {} - if generation_chunk := _convert_responses_chunk_to_generation_chunk( + ( + current_index, + current_output_index, + current_sub_index, + generation_chunk, + ) = _convert_responses_chunk_to_generation_chunk( chunk, + current_index, + current_output_index, + current_sub_index, schema=original_schema_obj, metadata=metadata, has_reasoning=has_reasoning, - ): + ) + if generation_chunk: if run_manager: run_manager.on_llm_new_token( generation_chunk.text, chunk=generation_chunk @@ -839,15 +853,27 @@ class BaseChatOpenAI(BaseChatModel): async with context_manager as response: is_first_chunk = True + current_index = -1 + current_output_index = -1 + current_sub_index = -1 has_reasoning = False async for chunk in response: metadata = headers if is_first_chunk else {} - if generation_chunk := _convert_responses_chunk_to_generation_chunk( + ( + current_index, + current_output_index, + current_sub_index, + generation_chunk, + ) = _convert_responses_chunk_to_generation_chunk( chunk, + current_index, + current_output_index, + current_sub_index, schema=original_schema_obj, metadata=metadata, has_reasoning=has_reasoning, - ): + ) + if generation_chunk: if run_manager: await run_manager.on_llm_new_token( generation_chunk.text, chunk=generation_chunk @@ -3209,27 +3235,26 @@ def _make_computer_call_output_from_message(message: ToolMessage) -> dict: return computer_call_output -def _pop_summary_index_from_reasoning(reasoning: dict) -> dict: - """When streaming, langchain-core uses the ``index`` key to aggregate reasoning +def _pop_index_and_sub_index(block: dict) -> dict: + """When streaming, langchain-core uses the ``index`` key to aggregate text blocks. OpenAI API does not support this key, so we need to remove it. - - N.B. OpenAI also does not appear to support the ``summary_inex`` key when passed - back in. """ - new_reasoning = reasoning.copy() - if "summary" in reasoning and isinstance(reasoning["summary"], list): + new_block = {k: v for k, v in block.items() if k != "index"} + if "summary" in new_block and isinstance(new_block["summary"], list): new_summary = [] - for block in reasoning["summary"]: - new_block = {k: v for k, v in block.items() if k != "index"} - new_summary.append(new_block) - new_reasoning["summary"] = new_summary - return new_reasoning + for sub_block in new_block["summary"]: + new_sub_block = {k: v for k, v in sub_block.items() if k != "index"} + new_summary.append(new_sub_block) + new_block["summary"] = new_summary + return new_block def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list: """Construct the input for the OpenAI Responses API.""" input_ = [] for lc_msg in messages: + if isinstance(lc_msg, AIMessage): + lc_msg = _convert_from_v03_ai_message(lc_msg) msg = _convert_message_to_dict(lc_msg) # "name" parameter unsupported if "name" in msg: @@ -3251,97 +3276,85 @@ def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list: } input_.append(function_call_output) elif msg["role"] == "assistant": - # Reasoning items - reasoning_items = [] - if reasoning := lc_msg.additional_kwargs.get("reasoning"): - reasoning_items.append(_pop_summary_index_from_reasoning(reasoning)) - input_.extend(reasoning_items) - # Function calls - function_calls = [] - if tool_calls := msg.pop("tool_calls", None): - # TODO: should you be able to preserve the function call object id on - # the langchain tool calls themselves? - function_call_ids = lc_msg.additional_kwargs.get( - _FUNCTION_CALL_IDS_MAP_KEY - ) - for tool_call in tool_calls: - function_call = { - "type": "function_call", - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - "call_id": tool_call["id"], - } - if function_call_ids is not None and ( - _id := function_call_ids.get(tool_call["id"]) - ): - function_call["id"] = _id - function_calls.append(function_call) - # Built-in tool calls - computer_calls = [] - code_interpreter_calls = [] - mcp_calls = [] - image_generation_calls = [] - tool_outputs = lc_msg.additional_kwargs.get("tool_outputs", []) - for tool_output in tool_outputs: - if tool_output.get("type") == "computer_call": - computer_calls.append(tool_output) - elif tool_output.get("type") == "code_interpreter_call": - code_interpreter_calls.append(tool_output) - elif tool_output.get("type") == "mcp_call": - mcp_calls.append(tool_output) - elif tool_output.get("type") == "image_generation_call": - image_generation_calls.append(tool_output) - else: - pass - input_.extend(code_interpreter_calls) - input_.extend(mcp_calls) - - # A previous image generation call can be referenced by ID - - input_.extend( - [ - {"type": "image_generation_call", "id": image_generation_call["id"]} - for image_generation_call in image_generation_calls - ] - ) - - msg["content"] = msg.get("content") or [] - if lc_msg.additional_kwargs.get("refusal"): - if isinstance(msg["content"], str): - msg["content"] = [ - { - "type": "output_text", - "text": msg["content"], - "annotations": [], - } - ] - msg["content"] = msg["content"] + [ - {"type": "refusal", "refusal": lc_msg.additional_kwargs["refusal"]} - ] - if isinstance(msg["content"], list): - new_blocks = [] + if isinstance(msg.get("content"), list): for block in msg["content"]: - # chat api: {"type": "text", "text": "..."} - # responses api: {"type": "output_text", "text": "...", "annotations": [...]} # noqa: E501 - if block["type"] == "text": - new_blocks.append( - { - "type": "output_text", - "text": block["text"], - "annotations": block.get("annotations") or [], - } - ) - elif block["type"] in ("output_text", "refusal"): - new_blocks.append(block) - else: - pass - msg["content"] = new_blocks - if msg["content"]: - if lc_msg.id and lc_msg.id.startswith("msg_"): - msg["id"] = lc_msg.id - input_.append(msg) - input_.extend(function_calls) - input_.extend(computer_calls) + if isinstance(block, dict) and (block_type := block.get("type")): + # Aggregate content blocks for a single message + if block_type in ("text", "output_text", "refusal"): + msg_id = block.get("id") + if block_type in ("text", "output_text"): + new_block = { + "type": "output_text", + "text": block["text"], + "annotations": block.get("annotations") or [], + } + elif block_type == "refusal": + new_block = { + "type": "refusal", + "refusal": block["refusal"], + } + for item in input_: + if (item_id := item.get("id")) and item_id == msg_id: + # If existing block with this ID, append to it + if "content" not in item: + item["content"] = [] + item["content"].append(new_block) + break + else: + # If no block with this ID, create a new one + input_.append( + { + "type": "message", + "content": [new_block], + "role": "assistant", + "id": msg_id, + } + ) + elif block_type in ( + "reasoning", + "web_search_call", + "file_search_call", + "function_call", + "computer_call", + "code_interpreter_call", + "mcp_call", + "mcp_list_tools", + "mcp_approval_request", + ): + input_.append(_pop_index_and_sub_index(block)) + elif block_type == "image_generation_call": + # A previous image generation call can be referenced by ID + input_.append( + {"type": "image_generation_call", "id": block["id"]} + ) + else: + pass + elif isinstance(msg.get("content"), str): + input_.append( + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": msg["content"]}], + } + ) + + # Add function calls from tool calls if not already present + if tool_calls := msg.pop("tool_calls", None): + content_call_ids = { + block["call_id"] + for block in input_ + if block.get("type") == "function_call" and "call_id" in block + } + for tool_call in tool_calls: + if tool_call["id"] not in content_call_ids: + function_call = { + "type": "function_call", + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + "call_id": tool_call["id"], + } + input_.append(function_call) + elif msg["role"] in ("user", "system", "developer"): if isinstance(msg["content"], list): new_blocks = [] @@ -3396,6 +3409,8 @@ def _construct_lc_result_from_responses_api( if k in ( "created_at", + # backwards compatibility: keep response ID in response_metadata as well as + # top-level-id "id", "incomplete_details", "metadata", @@ -3419,7 +3434,6 @@ def _construct_lc_result_from_responses_api( tool_calls = [] invalid_tool_calls = [] additional_kwargs: dict = {} - msg_id = None for output in response.output: if output.type == "message": for content in output.content: @@ -3431,14 +3445,17 @@ def _construct_lc_result_from_responses_api( annotation.model_dump() for annotation in content.annotations ], + "id": output.id, } content_blocks.append(block) if hasattr(content, "parsed"): additional_kwargs["parsed"] = content.parsed if content.type == "refusal": - additional_kwargs["refusal"] = content.refusal - msg_id = output.id + content_blocks.append( + {"type": "refusal", "refusal": content.refusal, "id": output.id} + ) elif output.type == "function_call": + content_blocks.append(output.model_dump(exclude_none=True, mode="json")) try: args = json.loads(output.arguments, strict=False) error = None @@ -3462,19 +3479,19 @@ def _construct_lc_result_from_responses_api( "error": error, } invalid_tool_calls.append(tool_call) - if _FUNCTION_CALL_IDS_MAP_KEY not in additional_kwargs: - additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {} - additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][output.call_id] = output.id - elif output.type == "reasoning": - additional_kwargs["reasoning"] = output.model_dump( - exclude_none=True, mode="json" - ) - else: - tool_output = output.model_dump(exclude_none=True, mode="json") - if "tool_outputs" in additional_kwargs: - additional_kwargs["tool_outputs"].append(tool_output) - else: - additional_kwargs["tool_outputs"] = [tool_output] + elif output.type in ( + "reasoning", + "web_search_call", + "file_search_call", + "computer_call", + "code_interpreter_call", + "mcp_call", + "mcp_list_tools", + "mcp_approval_request", + "image_generation_call", + ): + content_blocks.append(output.model_dump(exclude_none=True, mode="json")) + # Workaround for parsing structured output in the streaming case. # from openai import OpenAI # from pydantic import BaseModel @@ -3510,22 +3527,70 @@ def _construct_lc_result_from_responses_api( pass message = AIMessage( content=content_blocks, - id=msg_id, + id=response.id, usage_metadata=usage_metadata, response_metadata=response_metadata, additional_kwargs=additional_kwargs, tool_calls=tool_calls, invalid_tool_calls=invalid_tool_calls, ) + message = _convert_to_v03_ai_message(message) return ChatResult(generations=[ChatGeneration(message=message)]) def _convert_responses_chunk_to_generation_chunk( chunk: Any, + current_index: int, # index in content + current_output_index: int, # index in Response output + current_sub_index: int, # index of content block in output item schema: Optional[type[_BM]] = None, metadata: Optional[dict] = None, has_reasoning: bool = False, -) -> Optional[ChatGenerationChunk]: +) -> tuple[int, int, int, Optional[ChatGenerationChunk]]: + def _advance(output_idx: int, sub_idx: Optional[int] = None) -> None: + """Advance indexes tracked during streaming. + + Example: we stream a response item of the form: + + .. code-block:: python + + { + "type": "message", # output_index 0 + "role": "assistant", + "id": "msg_123", + "content": [ + {"type": "output_text", "text": "foo"}, # sub_index 0 + {"type": "output_text", "text": "bar"}, # sub_index 1 + ], + } + + This is a single item with a shared ``output_index`` and two sub-indexes, one + for each content block. + + This will be processed into an AIMessage with two text blocks: + + .. code-block:: python + + AIMessage( + [ + {"type": "text", "text": "foo", "id": "msg_123"}, # index 0 + {"type": "text", "text": "bar", "id": "msg_123"}, # index 1 + ] + ) + + This function just identifies updates in output or sub-indexes and increments + the current index accordingly. + """ + nonlocal current_index, current_output_index, current_sub_index + if sub_idx is None: + if current_output_index != output_idx: + current_index += 1 + else: + if (current_output_index != output_idx) or (current_sub_index != sub_idx): + current_index += 1 + current_sub_index = sub_idx + current_output_index = output_idx + content = [] tool_call_chunks: list = [] additional_kwargs: dict = {} @@ -3536,16 +3601,18 @@ def _convert_responses_chunk_to_generation_chunk( usage_metadata = None id = None if chunk.type == "response.output_text.delta": - content.append( - {"type": "text", "text": chunk.delta, "index": chunk.content_index} - ) + _advance(chunk.output_index, chunk.content_index) + content.append({"type": "text", "text": chunk.delta, "index": current_index}) elif chunk.type == "response.output_text.annotation.added": + _advance(chunk.output_index, chunk.content_index) if isinstance(chunk.annotation, dict): # Appears to be a breaking change in openai==1.82.0 annotation = chunk.annotation else: annotation = chunk.annotation.model_dump(exclude_none=True, mode="json") - content.append({"annotations": [annotation], "index": chunk.content_index}) + content.append({"annotations": [annotation], "index": current_index}) + elif chunk.type == "response.output_text.done": + content.append({"id": chunk.item_id, "index": current_index}) elif chunk.type == "response.created": response_metadata["id"] = chunk.response.id elif chunk.type == "response.completed": @@ -3569,18 +3636,26 @@ def _convert_responses_chunk_to_generation_chunk( chunk.type == "response.output_item.added" and chunk.item.type == "function_call" ): + _advance(chunk.output_index) tool_call_chunks.append( { "type": "tool_call_chunk", "name": chunk.item.name, "args": chunk.item.arguments, "id": chunk.item.call_id, - "index": chunk.output_index, + "index": current_index, + } + ) + content.append( + { + "type": "function_call", + "name": chunk.item.name, + "arguments": chunk.item.arguments, + "call_id": chunk.item.call_id, + "id": chunk.item.id, + "index": current_index, } ) - additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = { - chunk.item.call_id: chunk.item.id - } elif chunk.type == "response.output_item.done" and chunk.item.type in ( "web_search_call", "file_search_call", @@ -3591,55 +3666,70 @@ def _convert_responses_chunk_to_generation_chunk( "mcp_approval_request", "image_generation_call", ): - additional_kwargs["tool_outputs"] = [ - chunk.item.model_dump(exclude_none=True, mode="json") - ] + _advance(chunk.output_index) + tool_output = chunk.item.model_dump(exclude_none=True, mode="json") + tool_output["index"] = current_index + content.append(tool_output) elif chunk.type == "response.function_call_arguments.delta": + _advance(chunk.output_index) tool_call_chunks.append( - { - "type": "tool_call_chunk", - "args": chunk.delta, - "index": chunk.output_index, - } + {"type": "tool_call_chunk", "args": chunk.delta, "index": current_index} + ) + content.append( + {"type": "function_call", "arguments": chunk.delta, "index": current_index} ) elif chunk.type == "response.refusal.done": - additional_kwargs["refusal"] = chunk.refusal + content.append({"type": "refusal", "refusal": chunk.refusal}) elif chunk.type == "response.output_item.added" and chunk.item.type == "reasoning": - if not has_reasoning: - # Hack until breaking release: store first reasoning item ID. - additional_kwargs["reasoning"] = chunk.item.model_dump( - exclude_none=True, mode="json" - ) + _advance(chunk.output_index) + reasoning = chunk.item.model_dump(exclude_none=True, mode="json") + reasoning["index"] = current_index + content.append(reasoning) elif chunk.type == "response.reasoning_summary_part.added": - additional_kwargs["reasoning"] = { - # langchain-core uses the `index` key to aggregate text blocks. - "summary": [ - {"index": chunk.summary_index, "type": "summary_text", "text": ""} - ] - } + _advance(chunk.output_index) + content.append( + { + # langchain-core uses the `index` key to aggregate text blocks. + "summary": [ + {"index": chunk.summary_index, "type": "summary_text", "text": ""} + ], + "index": current_index, + } + ) elif chunk.type == "response.image_generation_call.partial_image": # Partial images are not supported yet. pass elif chunk.type == "response.reasoning_summary_text.delta": - additional_kwargs["reasoning"] = { - "summary": [ - { - "index": chunk.summary_index, - "type": "summary_text", - "text": chunk.delta, - } - ] - } - else: - return None - - return ChatGenerationChunk( - message=AIMessageChunk( - content=content, # type: ignore[arg-type] - tool_call_chunks=tool_call_chunks, - usage_metadata=usage_metadata, - response_metadata=response_metadata, - additional_kwargs=additional_kwargs, - id=id, + _advance(chunk.output_index) + content.append( + { + "summary": [ + { + "index": chunk.summary_index, + "type": "summary_text", + "text": chunk.delta, + } + ], + "index": current_index, + } ) + else: + return current_index, current_output_index, current_sub_index, None + + message = AIMessageChunk( + content=content, # type: ignore[arg-type] + tool_call_chunks=tool_call_chunks, + usage_metadata=usage_metadata, + response_metadata=response_metadata, + additional_kwargs=additional_kwargs, + id=id, + ) + message = cast( + AIMessageChunk, _convert_to_v03_ai_message(message, has_reasoning=has_reasoning) + ) + return ( + current_index, + current_output_index, + current_sub_index, + ChatGenerationChunk(message=message), ) diff --git a/libs/partners/openai/pyproject.toml b/libs/partners/openai/pyproject.toml index 242bd726da0..3ba59e81588 100644 --- a/libs/partners/openai/pyproject.toml +++ b/libs/partners/openai/pyproject.toml @@ -7,8 +7,8 @@ authors = [] license = { text = "MIT" } requires-python = ">=3.9" dependencies = [ - "langchain-core<1.0.0,>=0.3.65", - "openai<2.0.0,>=1.68.2", + "langchain-core<1.0.0,>=0.3.64", + "openai<2.0.0,>=1.86.0", "tiktoken<1,>=0.7", ] name = "langchain-openai" diff --git a/libs/partners/openai/tests/cassettes/test_code_interpreter.yaml.gz b/libs/partners/openai/tests/cassettes/test_code_interpreter.yaml.gz index f7b822e4638..e6db4e8cd3f 100644 Binary files a/libs/partners/openai/tests/cassettes/test_code_interpreter.yaml.gz and b/libs/partners/openai/tests/cassettes/test_code_interpreter.yaml.gz differ diff --git a/libs/partners/openai/tests/cassettes/test_image_generation_multi_turn.yaml.gz b/libs/partners/openai/tests/cassettes/test_image_generation_multi_turn.yaml.gz index cbcebfb82ac..61ae4c52862 100644 Binary files a/libs/partners/openai/tests/cassettes/test_image_generation_multi_turn.yaml.gz and b/libs/partners/openai/tests/cassettes/test_image_generation_multi_turn.yaml.gz differ diff --git a/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz b/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz new file mode 100644 index 00000000000..d63dc1f1668 Binary files /dev/null and b/libs/partners/openai/tests/cassettes/test_web_search.yaml.gz differ diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py index 8f6d3843561..f4976d43476 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py @@ -56,7 +56,7 @@ def _check_response(response: Optional[BaseMessage]) -> None: assert tool_output["type"] -@pytest.mark.flaky(retries=3, delay=1) +@pytest.mark.vcr def test_web_search() -> None: llm = ChatOpenAI(model=MODEL_NAME) first_response = llm.invoke( @@ -442,6 +442,7 @@ def test_mcp_builtin() -> None: ), } response = llm_with_tools.invoke([input_message]) + assert all(isinstance(block, dict) for block in response.content) approval_message = HumanMessage( [ @@ -457,10 +458,53 @@ def test_mcp_builtin() -> None: _ = llm_with_tools.invoke( [approval_message], previous_response_id=response.response_metadata["id"] ) - # Zero-data retention (e.g., as below) requires change in output format. - # _ = llm_with_tools.invoke( - # [input_message, response, approval_message] - # ) + + +@pytest.mark.skip +def test_mcp_builtin_zdr() -> None: + llm = ChatOpenAI( + model="o4-mini", + use_responses_api=True, + model_kwargs={"store": False, "include": ["reasoning.encrypted_content"]}, + ) + + llm_with_tools = llm.bind_tools( + [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "require_approval": {"always": {"tool_names": ["read_wiki_structure"]}}, + } + ] + ) + input_message = { + "role": "user", + "content": ( + "What transport protocols does the 2025-03-26 version of the MCP spec " + "support?" + ), + } + full: Optional[BaseMessageChunk] = None + for chunk in llm_with_tools.stream([input_message]): + assert isinstance(chunk, AIMessageChunk) + full = chunk if full is None else full + chunk + + assert isinstance(full, AIMessageChunk) + assert all(isinstance(block, dict) for block in full.content) + + approval_message = HumanMessage( + [ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": block["id"], # type: ignore[index] + } + for block in full.content + if block["type"] == "mcp_approval_request" # type: ignore[index] + ] + ) + _ = llm_with_tools.invoke([input_message, full, approval_message]) @pytest.mark.vcr() @@ -494,6 +538,7 @@ def test_image_generation_streaming() -> None: expected_keys = { "id", + "index", "background", "output_format", "quality", diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py index 49f9ed2b541..aa7bead4a9b 100644 --- a/libs/partners/openai/tests/unit_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/unit_tests/chat_models/test_base.py @@ -23,7 +23,7 @@ from langchain_core.outputs import ChatGeneration, ChatResult from langchain_core.runnables import RunnableLambda from langchain_core.tracers.base import BaseTracer from langchain_core.tracers.schemas import Run -from openai.types.responses import ResponseOutputMessage +from openai.types.responses import ResponseOutputMessage, ResponseReasoningItem from openai.types.responses.response import IncompleteDetails, Response, ResponseUsage from openai.types.responses.response_error import ResponseError from openai.types.responses.response_file_search_tool_call import ( @@ -36,6 +36,7 @@ from openai.types.responses.response_function_web_search import ( ) from openai.types.responses.response_output_refusal import ResponseOutputRefusal from openai.types.responses.response_output_text import ResponseOutputText +from openai.types.responses.response_reasoning_item import Summary from openai.types.responses.response_usage import ( InputTokensDetails, OutputTokensDetails, @@ -44,8 +45,12 @@ from pydantic import BaseModel, Field from typing_extensions import TypedDict from langchain_openai import ChatOpenAI -from langchain_openai.chat_models.base import ( +from langchain_openai.chat_models._compat import ( _FUNCTION_CALL_IDS_MAP_KEY, + _convert_from_v03_ai_message, + _convert_to_v03_ai_message, +) +from langchain_openai.chat_models.base import ( _construct_lc_result_from_responses_api, _construct_responses_api_input, _convert_dict_to_message, @@ -1209,8 +1214,62 @@ def test__construct_lc_result_from_responses_api_multiple_text_blocks() -> None: result = _construct_lc_result_from_responses_api(response) assert len(result.generations[0].message.content) == 2 - assert result.generations[0].message.content[0]["text"] == "First part" # type: ignore - assert result.generations[0].message.content[1]["text"] == "Second part" # type: ignore + assert result.generations[0].message.content == [ + {"type": "text", "text": "First part", "annotations": []}, + {"type": "text", "text": "Second part", "annotations": []}, + ] + + +def test__construct_lc_result_from_responses_api_multiple_messages() -> None: + """Test a response with multiple text blocks.""" + response = Response( + id="resp_123", + created_at=1234567890, + model="gpt-4o", + object="response", + parallel_tool_calls=True, + tools=[], + tool_choice="auto", + output=[ + ResponseOutputMessage( + type="message", + id="msg_123", + content=[ + ResponseOutputText(type="output_text", text="foo", annotations=[]) + ], + role="assistant", + status="completed", + ), + ResponseReasoningItem( + type="reasoning", + id="rs_123", + summary=[Summary(type="summary_text", text="reasoning foo")], + ), + ResponseOutputMessage( + type="message", + id="msg_234", + content=[ + ResponseOutputText(type="output_text", text="bar", annotations=[]) + ], + role="assistant", + status="completed", + ), + ], + ) + + result = _construct_lc_result_from_responses_api(response) + + assert result.generations[0].message.content == [ + {"type": "text", "text": "foo", "annotations": []}, + {"type": "text", "text": "bar", "annotations": []}, + ] + assert result.generations[0].message.additional_kwargs == { + "reasoning": { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "reasoning foo"}], + "id": "rs_123", + } + } def test__construct_lc_result_from_responses_api_refusal_response() -> None: @@ -1240,10 +1299,8 @@ def test__construct_lc_result_from_responses_api_refusal_response() -> None: result = _construct_lc_result_from_responses_api(response) - assert result.generations[0].message.content == [] - assert ( - result.generations[0].message.additional_kwargs["refusal"] - == "I cannot assist with that request." + assert result.generations[0].message.additional_kwargs["refusal"] == ( + "I cannot assist with that request." ) @@ -1620,6 +1677,40 @@ def test__construct_responses_api_input_human_message_with_text_blocks_conversio assert result[0]["content"][0]["text"] == "What's in this image?" +def test__construct_responses_api_input_multiple_message_components() -> None: + """Test that human messages with text blocks are properly converted.""" + messages: list = [ + AIMessage( + content=[ + {"type": "text", "text": "foo", "id": "msg_123"}, + {"type": "text", "text": "bar", "id": "msg_123"}, + {"type": "refusal", "refusal": "I refuse.", "id": "msg_123"}, + {"type": "text", "text": "baz", "id": "msg_234"}, + ] + ) + ] + result = _construct_responses_api_input(messages) + + assert result == [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "foo", "annotations": []}, + {"type": "output_text", "text": "bar", "annotations": []}, + {"type": "refusal", "refusal": "I refuse."}, + ], + "id": "msg_123", + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "baz", "annotations": []}], + "id": "msg_234", + }, + ] + + def test__construct_responses_api_input_human_message_with_image_url_conversion() -> ( None ): @@ -1666,13 +1757,17 @@ def test__construct_responses_api_input_ai_message_with_tool_calls() -> None: } ] - # Create a mapping from tool call IDs to function call IDs - function_call_ids = {"call_123": "func_456"} - ai_message = AIMessage( - content="", + content=[ + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "San Francisco"}', + "call_id": "call_123", + "id": "fc_456", + } + ], tool_calls=tool_calls, - additional_kwargs={_FUNCTION_CALL_IDS_MAP_KEY: function_call_ids}, ) result = _construct_responses_api_input([ai_message]) @@ -1682,7 +1777,19 @@ def test__construct_responses_api_input_ai_message_with_tool_calls() -> None: assert result[0]["name"] == "get_weather" assert result[0]["arguments"] == '{"location": "San Francisco"}' assert result[0]["call_id"] == "call_123" - assert result[0]["id"] == "func_456" + assert result[0]["id"] == "fc_456" + + # Message with only tool calls attribute provided + ai_message = AIMessage(content="", tool_calls=tool_calls) + + result = _construct_responses_api_input([ai_message]) + + assert len(result) == 1 + assert result[0]["type"] == "function_call" + assert result[0]["name"] == "get_weather" + assert result[0]["arguments"] == '{"location": "San Francisco"}' + assert result[0]["call_id"] == "call_123" + assert "id" not in result[0] def test__construct_responses_api_input_ai_message_with_tool_calls_and_content() -> ( @@ -1698,29 +1805,59 @@ def test__construct_responses_api_input_ai_message_with_tool_calls_and_content() } ] - # Create a mapping from tool call IDs to function call IDs - function_call_ids = {"call_123": "func_456"} - + # Content blocks ai_message = AIMessage( - content="I'll check the weather for you.", + content=[ + {"type": "text", "text": "I'll check the weather for you."}, + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "San Francisco"}', + "call_id": "call_123", + "id": "fc_456", + }, + ], tool_calls=tool_calls, - additional_kwargs={_FUNCTION_CALL_IDS_MAP_KEY: function_call_ids}, ) result = _construct_responses_api_input([ai_message]) assert len(result) == 2 - # Check content assert result[0]["role"] == "assistant" - assert result[0]["content"] == "I'll check the weather for you." + assert result[0]["content"] == [ + { + "type": "output_text", + "text": "I'll check the weather for you.", + "annotations": [], + } + ] - # Check function call assert result[1]["type"] == "function_call" assert result[1]["name"] == "get_weather" assert result[1]["arguments"] == '{"location": "San Francisco"}' assert result[1]["call_id"] == "call_123" - assert result[1]["id"] == "func_456" + assert result[1]["id"] == "fc_456" + + # String content + ai_message = AIMessage( + content="I'll check the weather for you.", tool_calls=tool_calls + ) + + result = _construct_responses_api_input([ai_message]) + + assert len(result) == 2 + + assert result[0]["role"] == "assistant" + assert result[0]["content"] == [ + {"type": "output_text", "text": "I'll check the weather for you."} + ] + + assert result[1]["type"] == "function_call" + assert result[1]["name"] == "get_weather" + assert result[1]["arguments"] == '{"location": "San Francisco"}' + assert result[1]["call_id"] == "call_123" + assert "id" not in result[1] def test__construct_responses_api_input_tool_message_conversion() -> None: @@ -1761,7 +1898,6 @@ def test__construct_responses_api_input_multiple_message_types() -> None: "args": {"location": "San Francisco"}, } ], - additional_kwargs={_FUNCTION_CALL_IDS_MAP_KEY: {"call_123": "func_456"}}, ), ToolMessage( content='{"temperature": 72, "conditions": "sunny"}', @@ -1777,7 +1913,7 @@ def test__construct_responses_api_input_multiple_message_types() -> None: ] ), ] - messages_copy = [m.copy(deep=True) for m in messages] + messages_copy = [m.model_copy(deep=True) for m in messages] result = _construct_responses_api_input(messages) @@ -1805,7 +1941,6 @@ def test__construct_responses_api_input_multiple_message_types() -> None: assert result[4]["name"] == "get_weather" assert result[4]["arguments"] == '{"location": "San Francisco"}' assert result[4]["call_id"] == "call_123" - assert result[4]["id"] == "func_456" # Check function call output assert result[5]["type"] == "function_call_output" @@ -1813,7 +1948,12 @@ def test__construct_responses_api_input_multiple_message_types() -> None: assert result[5]["call_id"] == "call_123" assert result[6]["role"] == "assistant" - assert result[6]["content"] == "The weather in San Francisco is 72°F and sunny." + assert result[6]["content"] == [ + { + "type": "output_text", + "text": "The weather in San Francisco is 72°F and sunny.", + } + ] assert result[7]["role"] == "assistant" assert result[7]["content"] == [ @@ -1925,3 +2065,64 @@ def test_mcp_tracing() -> None: # Test headers are correctly propagated to request payload = llm_with_tools._get_request_payload([input_message], tools=tools) # type: ignore[attr-defined] assert payload["tools"][0]["headers"]["Authorization"] == "Bearer PLACEHOLDER" + + +def test_compat() -> None: + # Check compatibility with v0.3 message format + message_v03 = AIMessage( + content=[ + {"type": "text", "text": "Hello, world!", "annotations": [{"type": "foo"}]} + ], + additional_kwargs={ + "reasoning": { + "type": "reasoning", + "id": "rs_123", + "summary": [{"type": "summary_text", "text": "Reasoning summary"}], + }, + "tool_outputs": [ + { + "type": "web_search_call", + "id": "websearch_123", + "status": "completed", + } + ], + "refusal": "I cannot assist with that.", + }, + response_metadata={"id": "resp_123"}, + id="msg_123", + ) + + message = _convert_from_v03_ai_message(message_v03) + expected = AIMessage( + content=[ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Reasoning summary"}], + "id": "rs_123", + }, + { + "type": "text", + "text": "Hello, world!", + "annotations": [{"type": "foo"}], + "id": "msg_123", + }, + {"type": "refusal", "refusal": "I cannot assist with that."}, + {"type": "web_search_call", "id": "websearch_123", "status": "completed"}, + ], + response_metadata={"id": "resp_123"}, + id="resp_123", + ) + assert message == expected + + ## Check no mutation + assert message != message_v03 + assert len(message_v03.content) == 1 + assert all( + item in message_v03.additional_kwargs + for item in ["reasoning", "tool_outputs", "refusal"] + ) + + # Convert back + message_v03_output = _convert_to_v03_ai_message(message) + assert message_v03_output == message_v03 + assert message_v03_output is not message_v03 diff --git a/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py b/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py new file mode 100644 index 00000000000..6e91ff9a3fd --- /dev/null +++ b/libs/partners/openai/tests/unit_tests/chat_models/test_responses_stream.py @@ -0,0 +1,646 @@ +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest +from langchain_core.messages import AIMessageChunk, BaseMessageChunk +from openai.types.responses import ( + ResponseCompletedEvent, + ResponseContentPartAddedEvent, + ResponseContentPartDoneEvent, + ResponseCreatedEvent, + ResponseInProgressEvent, + ResponseOutputItemAddedEvent, + ResponseOutputItemDoneEvent, + ResponseOutputMessage, + ResponseReasoningItem, + ResponseReasoningSummaryPartAddedEvent, + ResponseReasoningSummaryPartDoneEvent, + ResponseReasoningSummaryTextDeltaEvent, + ResponseReasoningSummaryTextDoneEvent, + ResponseTextConfig, + ResponseTextDeltaEvent, + ResponseTextDoneEvent, +) +from openai.types.responses.response import Response, ResponseUsage +from openai.types.responses.response_output_text import ResponseOutputText +from openai.types.responses.response_reasoning_item import Summary +from openai.types.responses.response_reasoning_summary_part_added_event import ( + Part as PartAdded, +) +from openai.types.responses.response_reasoning_summary_part_done_event import ( + Part as PartDone, +) +from openai.types.responses.response_usage import ( + InputTokensDetails, + OutputTokensDetails, +) +from openai.types.shared.reasoning import Reasoning +from openai.types.shared.response_format_text import ResponseFormatText + +from langchain_openai import ChatOpenAI +from tests.unit_tests.chat_models.test_base import MockSyncContextManager + +responses_stream = [ + ResponseCreatedEvent( + response=Response( + id="resp_123", + created_at=1749734255.0, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model="o4-mini-2025-04-16", + object="response", + output=[], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + background=False, + max_output_tokens=None, + previous_response_id=None, + reasoning=Reasoning( + effort="medium", generate_summary=None, summary="detailed" + ), + service_tier="auto", + status="in_progress", + text=ResponseTextConfig(format=ResponseFormatText(type="text")), + truncation="disabled", + usage=None, + user=None, + ), + sequence_number=0, + type="response.created", + ), + ResponseInProgressEvent( + response=Response( + id="resp_123", + created_at=1749734255.0, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model="o4-mini-2025-04-16", + object="response", + output=[], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + background=False, + max_output_tokens=None, + previous_response_id=None, + reasoning=Reasoning( + effort="medium", generate_summary=None, summary="detailed" + ), + service_tier="auto", + status="in_progress", + text=ResponseTextConfig(format=ResponseFormatText(type="text")), + truncation="disabled", + usage=None, + user=None, + ), + sequence_number=1, + type="response.in_progress", + ), + ResponseOutputItemAddedEvent( + item=ResponseReasoningItem( + id="rs_123", + summary=[], + type="reasoning", + encrypted_content=None, + status=None, + ), + output_index=0, + sequence_number=2, + type="response.output_item.added", + ), + ResponseReasoningSummaryPartAddedEvent( + item_id="rs_123", + output_index=0, + part=PartAdded(text="", type="summary_text"), + sequence_number=3, + summary_index=0, + type="response.reasoning_summary_part.added", + ), + ResponseReasoningSummaryTextDeltaEvent( + delta="reasoning block", + item_id="rs_123", + output_index=0, + sequence_number=4, + summary_index=0, + type="response.reasoning_summary_text.delta", + ), + ResponseReasoningSummaryTextDeltaEvent( + delta=" one", + item_id="rs_123", + output_index=0, + sequence_number=5, + summary_index=0, + type="response.reasoning_summary_text.delta", + ), + ResponseReasoningSummaryTextDoneEvent( + item_id="rs_123", + output_index=0, + sequence_number=6, + summary_index=0, + text="reasoning block one", + type="response.reasoning_summary_text.done", + ), + ResponseReasoningSummaryPartDoneEvent( + item_id="rs_123", + output_index=0, + part=PartDone(text="reasoning block one", type="summary_text"), + sequence_number=7, + summary_index=0, + type="response.reasoning_summary_part.done", + ), + ResponseReasoningSummaryPartAddedEvent( + item_id="rs_123", + output_index=0, + part=PartAdded(text="", type="summary_text"), + sequence_number=8, + summary_index=1, + type="response.reasoning_summary_part.added", + ), + ResponseReasoningSummaryTextDeltaEvent( + delta="another reasoning", + item_id="rs_123", + output_index=0, + sequence_number=9, + summary_index=1, + type="response.reasoning_summary_text.delta", + ), + ResponseReasoningSummaryTextDeltaEvent( + delta=" block", + item_id="rs_123", + output_index=0, + sequence_number=10, + summary_index=1, + type="response.reasoning_summary_text.delta", + ), + ResponseReasoningSummaryTextDoneEvent( + item_id="rs_123", + output_index=0, + sequence_number=11, + summary_index=1, + text="another reasoning block", + type="response.reasoning_summary_text.done", + ), + ResponseReasoningSummaryPartDoneEvent( + item_id="rs_123", + output_index=0, + part=PartDone(text="another reasoning block", type="summary_text"), + sequence_number=12, + summary_index=1, + type="response.reasoning_summary_part.done", + ), + ResponseOutputItemDoneEvent( + item=ResponseReasoningItem( + id="rs_123", + summary=[ + Summary(text="reasoning block one", type="summary_text"), + Summary(text="another reasoning block", type="summary_text"), + ], + type="reasoning", + encrypted_content=None, + status=None, + ), + output_index=0, + sequence_number=13, + type="response.output_item.done", + ), + ResponseOutputItemAddedEvent( + item=ResponseOutputMessage( + id="msg_123", + content=[], + role="assistant", + status="in_progress", + type="message", + ), + output_index=1, + sequence_number=14, + type="response.output_item.added", + ), + ResponseContentPartAddedEvent( + content_index=0, + item_id="msg_123", + output_index=1, + part=ResponseOutputText(annotations=[], text="", type="output_text"), + sequence_number=15, + type="response.content_part.added", + ), + ResponseTextDeltaEvent( + content_index=0, + delta="text block", + item_id="msg_123", + output_index=1, + sequence_number=16, + type="response.output_text.delta", + ), + ResponseTextDeltaEvent( + content_index=0, + delta=" one", + item_id="msg_123", + output_index=1, + sequence_number=17, + type="response.output_text.delta", + ), + ResponseTextDoneEvent( + content_index=0, + item_id="msg_123", + output_index=1, + sequence_number=18, + text="text block one", + type="response.output_text.done", + ), + ResponseContentPartDoneEvent( + content_index=0, + item_id="msg_123", + output_index=1, + part=ResponseOutputText( + annotations=[], text="text block one", type="output_text" + ), + sequence_number=19, + type="response.content_part.done", + ), + ResponseContentPartAddedEvent( + content_index=1, + item_id="msg_123", + output_index=1, + part=ResponseOutputText(annotations=[], text="", type="output_text"), + sequence_number=20, + type="response.content_part.added", + ), + ResponseTextDeltaEvent( + content_index=1, + delta="another text", + item_id="msg_123", + output_index=1, + sequence_number=21, + type="response.output_text.delta", + ), + ResponseTextDeltaEvent( + content_index=1, + delta=" block", + item_id="msg_123", + output_index=1, + sequence_number=22, + type="response.output_text.delta", + ), + ResponseTextDoneEvent( + content_index=1, + item_id="msg_123", + output_index=1, + sequence_number=23, + text="another text block", + type="response.output_text.done", + ), + ResponseContentPartDoneEvent( + content_index=1, + item_id="msg_123", + output_index=1, + part=ResponseOutputText( + annotations=[], text="another text block", type="output_text" + ), + sequence_number=24, + type="response.content_part.done", + ), + ResponseOutputItemDoneEvent( + item=ResponseOutputMessage( + id="msg_123", + content=[ + ResponseOutputText( + annotations=[], text="text block one", type="output_text" + ), + ResponseOutputText( + annotations=[], text="another text block", type="output_text" + ), + ], + role="assistant", + status="completed", + type="message", + ), + output_index=1, + sequence_number=25, + type="response.output_item.done", + ), + ResponseOutputItemAddedEvent( + item=ResponseReasoningItem( + id="rs_234", + summary=[], + type="reasoning", + encrypted_content=None, + status=None, + ), + output_index=2, + sequence_number=26, + type="response.output_item.added", + ), + ResponseReasoningSummaryPartAddedEvent( + item_id="rs_234", + output_index=2, + part=PartAdded(text="", type="summary_text"), + sequence_number=27, + summary_index=0, + type="response.reasoning_summary_part.added", + ), + ResponseReasoningSummaryTextDeltaEvent( + delta="more reasoning", + item_id="rs_234", + output_index=2, + sequence_number=28, + summary_index=0, + type="response.reasoning_summary_text.delta", + ), + ResponseReasoningSummaryTextDoneEvent( + item_id="rs_234", + output_index=2, + sequence_number=29, + summary_index=0, + text="more reasoning", + type="response.reasoning_summary_text.done", + ), + ResponseReasoningSummaryPartDoneEvent( + item_id="rs_234", + output_index=2, + part=PartDone(text="more reasoning", type="summary_text"), + sequence_number=30, + summary_index=0, + type="response.reasoning_summary_part.done", + ), + ResponseReasoningSummaryPartAddedEvent( + item_id="rs_234", + output_index=2, + part=PartAdded(text="", type="summary_text"), + sequence_number=31, + summary_index=1, + type="response.reasoning_summary_part.added", + ), + ResponseReasoningSummaryTextDeltaEvent( + delta="still more reasoning", + item_id="rs_234", + output_index=2, + sequence_number=32, + summary_index=1, + type="response.reasoning_summary_text.delta", + ), + ResponseReasoningSummaryTextDoneEvent( + item_id="rs_234", + output_index=2, + sequence_number=33, + summary_index=1, + text="still more reasoning", + type="response.reasoning_summary_text.done", + ), + ResponseReasoningSummaryPartDoneEvent( + item_id="rs_234", + output_index=2, + part=PartDone(text="still more reasoning", type="summary_text"), + sequence_number=34, + summary_index=1, + type="response.reasoning_summary_part.done", + ), + ResponseOutputItemDoneEvent( + item=ResponseReasoningItem( + id="rs_234", + summary=[ + Summary(text="more reasoning", type="summary_text"), + Summary(text="still more reasoning", type="summary_text"), + ], + type="reasoning", + encrypted_content=None, + status=None, + ), + output_index=2, + sequence_number=35, + type="response.output_item.done", + ), + ResponseOutputItemAddedEvent( + item=ResponseOutputMessage( + id="msg_234", + content=[], + role="assistant", + status="in_progress", + type="message", + ), + output_index=3, + sequence_number=36, + type="response.output_item.added", + ), + ResponseContentPartAddedEvent( + content_index=0, + item_id="msg_234", + output_index=3, + part=ResponseOutputText(annotations=[], text="", type="output_text"), + sequence_number=37, + type="response.content_part.added", + ), + ResponseTextDeltaEvent( + content_index=0, + delta="more", + item_id="msg_234", + output_index=3, + sequence_number=38, + type="response.output_text.delta", + ), + ResponseTextDoneEvent( + content_index=0, + item_id="msg_234", + output_index=3, + sequence_number=39, + text="more", + type="response.output_text.done", + ), + ResponseContentPartDoneEvent( + content_index=0, + item_id="msg_234", + output_index=3, + part=ResponseOutputText(annotations=[], text="more", type="output_text"), + sequence_number=40, + type="response.content_part.done", + ), + ResponseContentPartAddedEvent( + content_index=1, + item_id="msg_234", + output_index=3, + part=ResponseOutputText(annotations=[], text="", type="output_text"), + sequence_number=41, + type="response.content_part.added", + ), + ResponseTextDeltaEvent( + content_index=1, + delta="text", + item_id="msg_234", + output_index=3, + sequence_number=42, + type="response.output_text.delta", + ), + ResponseTextDoneEvent( + content_index=1, + item_id="msg_234", + output_index=3, + sequence_number=43, + text="text", + type="response.output_text.done", + ), + ResponseContentPartDoneEvent( + content_index=1, + item_id="msg_234", + output_index=3, + part=ResponseOutputText(annotations=[], text="text", type="output_text"), + sequence_number=44, + type="response.content_part.done", + ), + ResponseOutputItemDoneEvent( + item=ResponseOutputMessage( + id="msg_234", + content=[ + ResponseOutputText(annotations=[], text="more", type="output_text"), + ResponseOutputText(annotations=[], text="text", type="output_text"), + ], + role="assistant", + status="completed", + type="message", + ), + output_index=3, + sequence_number=45, + type="response.output_item.done", + ), + ResponseCompletedEvent( + response=Response( + id="resp_123", + created_at=1749734255.0, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model="o4-mini-2025-04-16", + object="response", + output=[ + ResponseReasoningItem( + id="rs_123", + summary=[ + Summary(text="reasoning block one", type="summary_text"), + Summary(text="another reasoning block", type="summary_text"), + ], + type="reasoning", + encrypted_content=None, + status=None, + ), + ResponseOutputMessage( + id="msg_123", + content=[ + ResponseOutputText( + annotations=[], text="text block one", type="output_text" + ), + ResponseOutputText( + annotations=[], + text="another text block", + type="output_text", + ), + ], + role="assistant", + status="completed", + type="message", + ), + ResponseReasoningItem( + id="rs_234", + summary=[ + Summary(text="more reasoning", type="summary_text"), + Summary(text="still more reasoning", type="summary_text"), + ], + type="reasoning", + encrypted_content=None, + status=None, + ), + ResponseOutputMessage( + id="msg_234", + content=[ + ResponseOutputText( + annotations=[], text="more", type="output_text" + ), + ResponseOutputText( + annotations=[], text="text", type="output_text" + ), + ], + role="assistant", + status="completed", + type="message", + ), + ], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + background=False, + max_output_tokens=None, + previous_response_id=None, + reasoning=Reasoning( + effort="medium", generate_summary=None, summary="detailed" + ), + service_tier="default", + status="completed", + text=ResponseTextConfig(format=ResponseFormatText(type="text")), + truncation="disabled", + usage=ResponseUsage( + input_tokens=13, + input_tokens_details=InputTokensDetails(cached_tokens=0), + output_tokens=71, + output_tokens_details=OutputTokensDetails(reasoning_tokens=64), + total_tokens=84, + ), + user=None, + ), + sequence_number=46, + type="response.completed", + ), +] + + +@pytest.mark.xfail(reason="Will be fixed with output format flags.") +def test_responses_stream() -> None: + llm = ChatOpenAI(model="o4-mini", use_responses_api=True) + mock_client = MagicMock() + + def mock_create(*args: Any, **kwargs: Any) -> MockSyncContextManager: + return MockSyncContextManager(responses_stream) + + mock_client.responses.create = mock_create + + full: Optional[BaseMessageChunk] = None + with patch.object(llm, "root_client", mock_client): + for chunk in llm.stream("test"): + assert isinstance(chunk, AIMessageChunk) + full = chunk if full is None else full + chunk + assert isinstance(full, AIMessageChunk) + + expected_content = [ + { + "id": "rs_123", + "summary": [ + {"index": 0, "type": "summary_text", "text": "reasoning block one"}, + {"index": 1, "type": "summary_text", "text": "another reasoning block"}, + ], + "type": "reasoning", + "index": 0, + }, + {"type": "text", "text": "text block one", "index": 1, "id": "msg_123"}, + {"type": "text", "text": "another text block", "index": 2, "id": "msg_123"}, + { + "id": "rs_234", + "summary": [ + {"index": 0, "type": "summary_text", "text": "more reasoning"}, + {"index": 1, "type": "summary_text", "text": "still more reasoning"}, + ], + "type": "reasoning", + "index": 3, + }, + {"type": "text", "text": "more", "index": 4, "id": "msg_234"}, + {"type": "text", "text": "text", "index": 5, "id": "msg_234"}, + ] + assert full.content == expected_content + assert full.additional_kwargs == {} diff --git a/libs/partners/openai/uv.lock b/libs/partners/openai/uv.lock index adb9c07bc70..a78b818aab5 100644 --- a/libs/partners/openai/uv.lock +++ b/libs/partners/openai/uv.lock @@ -587,7 +587,7 @@ typing = [ [package.metadata] requires-dist = [ { name = "langchain-core", editable = "../../core" }, - { name = "openai", specifier = ">=1.68.2,<2.0.0" }, + { name = "openai", specifier = ">=1.86.0,<2.0.0" }, { name = "tiktoken", specifier = ">=0.7,<1" }, ]