From c99773b652eef6fbc6aaa59c9668d55f8eace81d Mon Sep 17 00:00:00 2001 From: Sydney Runkle <54324534+sydney-runkle@users.noreply.github.com> Date: Thu, 9 Oct 2025 17:51:28 -0400 Subject: [PATCH] feat(langchain_v1): refactoring HITL API (#33397) Easiest to review side by side (not inline) * Adding `dict` type requests + responses so that we can ship config w/ interrupts. Also more extensible. * Keeping things generic in terms of `interrupt_on` rather than `tool_config` * Renaming allowed decisions -- approve, edit, reject * Draws differentiation between actions (requested + performed by the agent), in this case tool calls, though we generalize beyond that and decisions - human feedback for said actions New request structure ```py class Action(TypedDict): """Represents an action with a name and arguments.""" name: str """The type or name of action being requested (e.g., "add_numbers").""" arguments: dict[str, Any] """Key-value pairs of arguments needed for the action (e.g., {"a": 1, "b": 2}).""" DecisionType = Literal["approve", "edit", "reject"] class ReviewConfig(TypedDict): """Policy for reviewing a HITL request.""" action_name: str """Name of the action associated with this review configuration.""" allowed_decisions: list[DecisionType] """The decisions that are allowed for this request.""" description: NotRequired[str] """The description of the action to be reviewed.""" arguments_schema: NotRequired[dict[str, Any]] """JSON schema for the arguments associated with the action, if edits are allowed.""" class HITLRequest(TypedDict): """Request for human feedback on a sequence of actions requested by a model.""" action_requests: list[Action] """A list of agent actions for human review.""" review_configs: list[ReviewConfig] """Review configuration for all possible actions.""" ``` New response structure ```py class ApproveDecision(TypedDict): """Response when a human approves the action.""" type: Literal["approve"] """The type of response when a human approves the action.""" class EditDecision(TypedDict): """Response when a human edits the action.""" type: Literal["edit"] """The type of response when a human edits the action.""" edited_action: Action """Edited action for the agent to perform. Ex: for a tool call, a human reviewer can edit the tool name and args. """ class RejectDecision(TypedDict): """Response when a human rejects the action.""" type: Literal["reject"] """The type of response when a human rejects the action.""" message: NotRequired[str] """The message sent to the model explaining why the action was rejected.""" Decision = ApproveDecision | EditDecision | RejectDecision class HITLResponse(TypedDict): """Response payload for a HITLRequest.""" decisions: list[Decision] """The decisions made by the human.""" ``` User facing API: NEW ```py HumanInTheLoopMiddleware(interrupt_on={ 'send_email': True, # can also use a callable for description that takes tool call, state, and runtime 'execute_sql': { 'allowed_decisions': ['approve', 'edit', 'reject'], 'description': 'please review sensitive tool execution'}, } }) Command(resume={"decisions": [{"type": "approve"}, {"type": "reject": "message": "db down"}]}) ``` OLD ```py HumanInTheLoopMiddleware(interrupt_on={ 'send_email': True, 'execute_sql': { 'allow_accept': True, 'allow_edit': True, 'allow_respond': True, description='please review sensitive tool execution' }, }) Command(resume=[{"type": "approve"}, {"type": "reject": "message": "db down"}]) ``` --- .../langchain/agents/middleware/__init__.py | 6 +- .../agents/middleware/human_in_the_loop.py | 345 ++++++++++-------- .../agents/test_middleware_agent.py | 269 +++++++------- 3 files changed, 337 insertions(+), 283 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/middleware/__init__.py b/libs/langchain_v1/langchain/agents/middleware/__init__.py index d7066c67920..27caab235f3 100644 --- a/libs/langchain_v1/langchain/agents/middleware/__init__.py +++ b/libs/langchain_v1/langchain/agents/middleware/__init__.py @@ -4,7 +4,10 @@ from .context_editing import ( ClearToolUsesEdit, ContextEditingMiddleware, ) -from .human_in_the_loop import HumanInTheLoopMiddleware +from .human_in_the_loop import ( + HumanInTheLoopMiddleware, + InterruptOnConfig, +) from .model_call_limit import ModelCallLimitMiddleware from .model_fallback import ModelFallbackMiddleware from .pii import PIIDetectionError, PIIMiddleware @@ -34,6 +37,7 @@ __all__ = [ "ClearToolUsesEdit", "ContextEditingMiddleware", "HumanInTheLoopMiddleware", + "InterruptOnConfig", "LLMToolSelectorMiddleware", "ModelCallLimitMiddleware", "ModelFallbackMiddleware", diff --git a/libs/langchain_v1/langchain/agents/middleware/human_in_the_loop.py b/libs/langchain_v1/langchain/agents/middleware/human_in_the_loop.py index 071a8d9cf4a..5ee6b87d8b5 100644 --- a/libs/langchain_v1/langchain/agents/middleware/human_in_the_loop.py +++ b/libs/langchain_v1/langchain/agents/middleware/human_in_the_loop.py @@ -10,89 +10,93 @@ from typing_extensions import NotRequired, TypedDict from langchain.agents.middleware.types import AgentMiddleware, AgentState -class HumanInTheLoopConfig(TypedDict): - """Configuration that defines what actions are allowed for a human interrupt. +class Action(TypedDict): + """Represents an action with a name and arguments.""" - This controls the available interaction options when the graph is paused for human input. - """ - - allow_accept: NotRequired[bool] - """Whether the human can approve the current action without changes.""" - allow_edit: NotRequired[bool] - """Whether the human can approve the current action with edited content.""" - allow_respond: NotRequired[bool] - """Whether the human can reject the current action with feedback.""" - - -class ActionRequest(TypedDict): - """Represents a request with a name and arguments.""" - - action: str + name: str """The type or name of action being requested (e.g., "add_numbers").""" - args: dict + + arguments: dict[str, Any] """Key-value pairs of arguments needed for the action (e.g., {"a": 1, "b": 2}).""" -class HumanInTheLoopRequest(TypedDict): - """Represents an interrupt triggered by the graph that requires human intervention. +class ActionRequest(TypedDict): + """Represents an action request with a name, arguments, and description.""" - Example: - ```python - # Extract a tool call from the state and create an interrupt request - request = HumanInterrupt( - action_request=ActionRequest( - action="run_command", # The action being requested - args={"command": "ls", "args": ["-l"]}, # Arguments for the action - ), - config=HumanInTheLoopConfig( - allow_accept=True, # Allow approval - allow_respond=True, # Allow rejection with feedback - allow_edit=False, # Don't allow approval with edits - ), - description="Please review the command before execution", - ) - # Send the interrupt request and get the response - response = interrupt([request])[0] - ``` - """ + name: str + """The name of the action being requested.""" - action_request: ActionRequest - """The specific action being requested from the human.""" - config: HumanInTheLoopConfig - """Configuration defining what response types are allowed.""" - description: str | None - """Optional detailed description of what input is needed.""" + arguments: dict[str, Any] + """Key-value pairs of arguments needed for the action (e.g., {"a": 1, "b": 2}).""" + + description: NotRequired[str] + """The description of the action to be reviewed.""" -class AcceptPayload(TypedDict): +DecisionType = Literal["approve", "edit", "reject"] + + +class ReviewConfig(TypedDict): + """Policy for reviewing a HITL request.""" + + action_name: str + """Name of the action associated with this review configuration.""" + + allowed_decisions: list[DecisionType] + """The decisions that are allowed for this request.""" + + arguments_schema: NotRequired[dict[str, Any]] + """JSON schema for the arguments associated with the action, if edits are allowed.""" + + +class HITLRequest(TypedDict): + """Request for human feedback on a sequence of actions requested by a model.""" + + action_requests: list[ActionRequest] + """A list of agent actions for human review.""" + + review_configs: list[ReviewConfig] + """Review configuration for all possible actions.""" + + +class ApproveDecision(TypedDict): """Response when a human approves the action.""" - type: Literal["accept"] + type: Literal["approve"] """The type of response when a human approves the action.""" -class ResponsePayload(TypedDict): - """Response when a human rejects the action.""" - - type: Literal["response"] - """The type of response when a human rejects the action.""" - - args: NotRequired[str] - """The message to be sent to the model explaining why the action was rejected.""" - - -class EditPayload(TypedDict): +class EditDecision(TypedDict): """Response when a human edits the action.""" type: Literal["edit"] """The type of response when a human edits the action.""" - args: ActionRequest - """The action request with the edited content.""" + edited_action: Action + """Edited action for the agent to perform. + + Ex: for a tool call, a human reviewer can edit the tool name and args. + """ -HumanInTheLoopResponse = AcceptPayload | ResponsePayload | EditPayload -"""Aggregated response type for all possible human in the loop responses.""" +class RejectDecision(TypedDict): + """Response when a human rejects the action.""" + + type: Literal["reject"] + """The type of response when a human rejects the action.""" + + message: NotRequired[str] + """The message sent to the model explaining why the action was rejected.""" + + +Decision = ApproveDecision | EditDecision | RejectDecision + + +class HITLResponse(TypedDict): + """Response payload for a HITLRequest.""" + + decisions: list[Decision] + """The decisions made by the human.""" class _DescriptionFactory(Protocol): @@ -103,15 +107,15 @@ class _DescriptionFactory(Protocol): ... -class ToolConfig(TypedDict): - """Configuration for a tool requiring human in the loop.""" +class InterruptOnConfig(TypedDict): + """Configuration for an action requiring human in the loop. + + This is the configuration format used in the `HumanInTheLoopMiddleware.__init__` method. + """ + + allowed_decisions: list[DecisionType] + """The decisions that are allowed for this action.""" - allow_accept: NotRequired[bool] - """Whether the human can approve the current action without changes.""" - allow_edit: NotRequired[bool] - """Whether the human can approve the current action with edited content.""" - allow_respond: NotRequired[bool] - """Whether the human can reject the current action with feedback.""" description: NotRequired[str | _DescriptionFactory] """The description attached to the request for human input. @@ -124,7 +128,7 @@ class ToolConfig(TypedDict): ```python # Static string description config = ToolConfig( - allow_accept=True, + allowed_decisions=["approve", "reject"], description="Please review this tool execution" ) @@ -146,6 +150,8 @@ class ToolConfig(TypedDict): ) ``` """ + arguments_schema: NotRequired[dict[str, Any]] + """JSON schema for the arguments associated with the action, if edits are allowed.""" class HumanInTheLoopMiddleware(AgentMiddleware): @@ -153,7 +159,7 @@ class HumanInTheLoopMiddleware(AgentMiddleware): def __init__( self, - interrupt_on: dict[str, bool | ToolConfig], + interrupt_on: dict[str, bool | InterruptOnConfig], *, description_prefix: str = "Tool execution requires approval", ) -> None: @@ -163,32 +169,106 @@ class HumanInTheLoopMiddleware(AgentMiddleware): interrupt_on: Mapping of tool name to allowed actions. If a tool doesn't have an entry, it's auto-approved by default. - * `True` indicates all actions are allowed: accept, edit, and respond. + * `True` indicates all decisions are allowed: approve, edit, and reject. * `False` indicates that the tool is auto-approved. - * `ToolConfig` indicates the specific actions allowed for this tool. - The ToolConfig can include a `description` field (str or callable) for + * `InterruptOnConfig` indicates the specific decisions allowed for this tool. + The InterruptOnConfig can include a `description` field (str or callable) for custom formatting of the interrupt description. description_prefix: The prefix to use when constructing action requests. This is used to provide context about the tool call and the action being requested. - Not used if a tool has a `description` in its ToolConfig. + Not used if a tool has a `description` in its InterruptOnConfig. """ super().__init__() - resolved_tool_configs: dict[str, ToolConfig] = {} + resolved_configs: dict[str, InterruptOnConfig] = {} for tool_name, tool_config in interrupt_on.items(): if isinstance(tool_config, bool): if tool_config is True: - resolved_tool_configs[tool_name] = ToolConfig( - allow_accept=True, - allow_edit=True, - allow_respond=True, + resolved_configs[tool_name] = InterruptOnConfig( + allowed_decisions=["approve", "edit", "reject"] ) - elif any( - tool_config.get(x, False) for x in ["allow_accept", "allow_edit", "allow_respond"] - ): - resolved_tool_configs[tool_name] = tool_config - self.interrupt_on = resolved_tool_configs + elif tool_config.get("allowed_decisions"): + resolved_configs[tool_name] = tool_config + self.interrupt_on = resolved_configs self.description_prefix = description_prefix + def _create_action_and_config( + self, + tool_call: ToolCall, + config: InterruptOnConfig, + state: AgentState, + runtime: Runtime, + ) -> tuple[ActionRequest, ReviewConfig]: + """Create an ActionRequest and ReviewConfig for a tool call.""" + tool_name = tool_call["name"] + tool_args = tool_call["args"] + + # Generate description using the description field (str or callable) + description_value = config.get("description") + if callable(description_value): + description = description_value(tool_call, state, runtime) + elif description_value is not None: + description = description_value + else: + description = f"{self.description_prefix}\n\nTool: {tool_name}\nArgs: {tool_args}" + + # Create ActionRequest with description + action_request = ActionRequest( + name=tool_name, + arguments=tool_args, + description=description, + ) + + # Create ReviewConfig + # eventually can get tool information and populate arguments_schema from there + review_config = ReviewConfig( + action_name=tool_name, + allowed_decisions=config["allowed_decisions"], + ) + + return action_request, review_config + + def _process_decision( + self, + decision: Decision, + tool_call: ToolCall, + config: InterruptOnConfig, + ) -> tuple[ToolCall | None, ToolMessage | None]: + """Process a single decision and return the revised tool call and optional tool message.""" + allowed_decisions = config["allowed_decisions"] + + if decision["type"] == "approve" and "approve" in allowed_decisions: + return tool_call, None + if decision["type"] == "edit" and "edit" in allowed_decisions: + edited_action = decision["edited_action"] + return ( + ToolCall( + type="tool_call", + name=edited_action["name"], + args=edited_action["arguments"], + id=tool_call["id"], + ), + None, + ) + if decision["type"] == "reject" and "reject" in allowed_decisions: + # Create a tool message with the human's text response + content = decision.get("message") or ( + f"User rejected the tool call for `{tool_call['name']}` with id {tool_call['id']}" + ) + tool_message = ToolMessage( + content=content, + name=tool_call["name"], + tool_call_id=tool_call["id"], + status="error", + ) + return tool_call, tool_message + msg = ( + f"Unexpected human decision: {decision}. " + f"Decision type '{decision.get('type')}' " + f"is not allowed for tool '{tool_call['name']}'. " + f"Expected one of {allowed_decisions} based on the tool's configuration." + ) + raise ValueError(msg) + def after_model(self, state: AgentState, runtime: Runtime) -> dict[str, Any] | None: """Trigger interrupt flows for relevant tool calls after an AIMessage.""" messages = state["messages"] @@ -216,87 +296,50 @@ class HumanInTheLoopMiddleware(AgentMiddleware): revised_tool_calls: list[ToolCall] = auto_approved_tool_calls.copy() artificial_tool_messages: list[ToolMessage] = [] - # Create interrupt requests for all tools that need approval - interrupt_requests: list[HumanInTheLoopRequest] = [] + # Create action requests and review configs for all tools that need approval + action_requests: list[ActionRequest] = [] + review_configs: list[ReviewConfig] = [] + for tool_call in interrupt_tool_calls: - tool_name = tool_call["name"] - tool_args = tool_call["args"] - config = self.interrupt_on[tool_name] + config = self.interrupt_on[tool_call["name"]] - # Generate description using the description field (str or callable) - description_value = config.get("description") - if callable(description_value): - description = description_value(tool_call, state, runtime) - elif description_value is not None: - description = description_value - else: - description = f"{self.description_prefix}\n\nTool: {tool_name}\nArgs: {tool_args}" + # Create ActionRequest and ReviewConfig using helper method + action_request, review_config = self._create_action_and_config( + tool_call, config, state, runtime + ) + action_requests.append(action_request) + review_configs.append(review_config) - request: HumanInTheLoopRequest = { - "action_request": ActionRequest( - action=tool_name, - args=tool_args, - ), - "config": config, - "description": description, - } - interrupt_requests.append(request) + # Create single HITLRequest with all actions and configs + hitl_request = HITLRequest( + action_requests=action_requests, + review_configs=review_configs, + ) - responses: list[HumanInTheLoopResponse] = interrupt(interrupt_requests) + # Send interrupt and get response + hitl_response: HITLResponse = interrupt(hitl_request) + decisions = hitl_response["decisions"] - # Validate that the number of responses matches the number of interrupt tool calls - if (responses_len := len(responses)) != ( + # Validate that the number of decisions matches the number of interrupt tool calls + if (decisions_len := len(decisions)) != ( interrupt_tool_calls_len := len(interrupt_tool_calls) ): msg = ( - f"Number of human responses ({responses_len}) does not match " + f"Number of human decisions ({decisions_len}) does not match " f"number of hanging tool calls ({interrupt_tool_calls_len})." ) raise ValueError(msg) - for i, response in enumerate(responses): + # Process each decision using helper method + for i, decision in enumerate(decisions): tool_call = interrupt_tool_calls[i] config = self.interrupt_on[tool_call["name"]] - if response["type"] == "accept" and config.get("allow_accept"): - revised_tool_calls.append(tool_call) - elif response["type"] == "edit" and config.get("allow_edit"): - edited_action = response["args"] - revised_tool_calls.append( - ToolCall( - type="tool_call", - name=edited_action["action"], - args=edited_action["args"], - id=tool_call["id"], - ) - ) - elif response["type"] == "response" and config.get("allow_respond"): - # Create a tool message with the human's text response - content = response.get("args") or ( - f"User rejected the tool call for `{tool_call['name']}` " - f"with id {tool_call['id']}" - ) - tool_message = ToolMessage( - content=content, - name=tool_call["name"], - tool_call_id=tool_call["id"], - status="error", - ) - revised_tool_calls.append(tool_call) + revised_tool_call, tool_message = self._process_decision(decision, tool_call, config) + if revised_tool_call: + revised_tool_calls.append(revised_tool_call) + if tool_message: artificial_tool_messages.append(tool_message) - else: - allowed_actions = [ - action - for action in ["accept", "edit", "response"] - if config.get(f"allow_{'respond' if action == 'response' else action}") - ] - msg = ( - f"Unexpected human response: {response}. " - f"Response action '{response.get('type')}' " - f"is not allowed for tool '{tool_call['name']}'. " - f"Expected one of {allowed_actions} based on the tool's configuration." - ) - raise ValueError(msg) # Update the AI message to only include approved tool calls last_ai_msg.tool_calls = revised_tool_calls diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py index df696090b1d..0aeb70828f9 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py @@ -29,7 +29,7 @@ from syrupy.assertion import SnapshotAssertion from typing_extensions import Annotated from langchain.agents.middleware.human_in_the_loop import ( - ActionRequest, + Action, HumanInTheLoopMiddleware, ) from langchain.agents.middleware.planning import ( @@ -463,14 +463,12 @@ def test_human_in_the_loop_middleware_initialization() -> None: """Test HumanInTheLoopMiddleware initialization.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True} - }, + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}, description_prefix="Custom prefix", ) assert middleware.interrupt_on == { - "test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True} + "test_tool": {"allowed_decisions": ["approve", "edit", "reject"]} } assert middleware.description_prefix == "Custom prefix" @@ -479,9 +477,7 @@ def test_human_in_the_loop_middleware_no_interrupts_needed() -> None: """Test HumanInTheLoopMiddleware when no interrupts are needed.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) # Test with no messages @@ -508,9 +504,7 @@ def test_human_in_the_loop_middleware_single_tool_accept() -> None: """Test HumanInTheLoopMiddleware with single tool accept response.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) ai_message = AIMessage( @@ -520,7 +514,7 @@ def test_human_in_the_loop_middleware_single_tool_accept() -> None: state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_accept(requests): - return [{"type": "accept", "args": None}] + return {"decisions": [{"type": "approve"}]} with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_accept): result = middleware.after_model(state, None) @@ -543,9 +537,7 @@ def test_human_in_the_loop_middleware_single_tool_accept() -> None: def test_human_in_the_loop_middleware_single_tool_edit() -> None: """Test HumanInTheLoopMiddleware with single tool edit response.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) ai_message = AIMessage( @@ -555,15 +547,17 @@ def test_human_in_the_loop_middleware_single_tool_edit() -> None: state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_edit(requests): - return [ - { - "type": "edit", - "args": ActionRequest( - action="test_tool", - args={"input": "edited"}, - ), - } - ] + return { + "decisions": [ + { + "type": "edit", + "edited_action": Action( + name="test_tool", + arguments={"input": "edited"}, + ), + } + ] + } with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_edit): result = middleware.after_model(state, None) @@ -578,9 +572,7 @@ def test_human_in_the_loop_middleware_single_tool_response() -> None: """Test HumanInTheLoopMiddleware with single tool response with custom message.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) ai_message = AIMessage( @@ -590,7 +582,7 @@ def test_human_in_the_loop_middleware_single_tool_response() -> None: state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_response(requests): - return [{"type": "response", "args": "Custom response message"}] + return {"decisions": [{"type": "reject", "message": "Custom response message"}]} with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_response @@ -611,8 +603,8 @@ def test_human_in_the_loop_middleware_multiple_tools_mixed_responses() -> None: middleware = HumanInTheLoopMiddleware( interrupt_on={ - "get_forecast": {"allow_accept": True, "allow_edit": True, "allow_respond": True}, - "get_temperature": {"allow_accept": True, "allow_edit": True, "allow_respond": True}, + "get_forecast": {"allowed_decisions": ["approve", "edit", "reject"]}, + "get_temperature": {"allowed_decisions": ["approve", "edit", "reject"]}, } ) @@ -626,10 +618,12 @@ def test_human_in_the_loop_middleware_multiple_tools_mixed_responses() -> None: state = {"messages": [HumanMessage(content="What's the weather?"), ai_message]} def mock_mixed_responses(requests): - return [ - {"type": "accept", "args": None}, - {"type": "response", "args": "User rejected this tool call"}, - ] + return { + "decisions": [ + {"type": "approve"}, + {"type": "reject", "message": "User rejected this tool call"}, + ] + } with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_mixed_responses @@ -659,8 +653,8 @@ def test_human_in_the_loop_middleware_multiple_tools_edit_responses() -> None: middleware = HumanInTheLoopMiddleware( interrupt_on={ - "get_forecast": {"allow_accept": True, "allow_edit": True, "allow_respond": True}, - "get_temperature": {"allow_accept": True, "allow_edit": True, "allow_respond": True}, + "get_forecast": {"allowed_decisions": ["approve", "edit", "reject"]}, + "get_temperature": {"allowed_decisions": ["approve", "edit", "reject"]}, } ) @@ -674,22 +668,24 @@ def test_human_in_the_loop_middleware_multiple_tools_edit_responses() -> None: state = {"messages": [HumanMessage(content="What's the weather?"), ai_message]} def mock_edit_responses(requests): - return [ - { - "type": "edit", - "args": ActionRequest( - action="get_forecast", - args={"location": "New York"}, - ), - }, - { - "type": "edit", - "args": ActionRequest( - action="get_temperature", - args={"location": "New York"}, - ), - }, - ] + return { + "decisions": [ + { + "type": "edit", + "edited_action": Action( + name="get_forecast", + arguments={"location": "New York"}, + ), + }, + { + "type": "edit", + "edited_action": Action( + name="get_temperature", + arguments={"location": "New York"}, + ), + }, + ] + } with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_edit_responses @@ -710,9 +706,7 @@ def test_human_in_the_loop_middleware_edit_with_modified_args() -> None: """Test HumanInTheLoopMiddleware with edit action that includes modified args.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) ai_message = AIMessage( @@ -722,15 +716,17 @@ def test_human_in_the_loop_middleware_edit_with_modified_args() -> None: state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_edit_with_args(requests): - return [ - { - "type": "edit", - "args": ActionRequest( - action="test_tool", - args={"input": "modified"}, - ), - } - ] + return { + "decisions": [ + { + "type": "edit", + "edited_action": Action( + name="test_tool", + arguments={"input": "modified"}, + ), + } + ] + } with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", @@ -750,9 +746,7 @@ def test_human_in_the_loop_middleware_edit_with_modified_args() -> None: def test_human_in_the_loop_middleware_unknown_response_type() -> None: """Test HumanInTheLoopMiddleware with unknown response type.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) ai_message = AIMessage( @@ -762,12 +756,12 @@ def test_human_in_the_loop_middleware_unknown_response_type() -> None: state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_unknown(requests): - return [{"type": "unknown", "args": None}] + return {"decisions": [{"type": "unknown"}]} with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_unknown): with pytest.raises( ValueError, - match=r"Unexpected human response: {'type': 'unknown', 'args': None}. Response action 'unknown' is not allowed for tool 'test_tool'. Expected one of \['accept', 'edit', 'response'\] based on the tool's configuration.", + match=r"Unexpected human decision: {'type': 'unknown'}. Decision type 'unknown' is not allowed for tool 'test_tool'. Expected one of \['approve', 'edit', 'reject'\] based on the tool's configuration.", ): middleware.after_model(state, None) @@ -777,9 +771,7 @@ def test_human_in_the_loop_middleware_disallowed_action() -> None: # edit is not allowed by tool config middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_respond": True, "allow_edit": False, "allow_accept": True} - } + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "reject"]}} ) ai_message = AIMessage( @@ -789,15 +781,17 @@ def test_human_in_the_loop_middleware_disallowed_action() -> None: state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_disallowed_action(requests): - return [ - { - "type": "edit", - "args": ActionRequest( - action="test_tool", - args={"input": "modified"}, - ), - } - ] + return { + "decisions": [ + { + "type": "edit", + "edited_action": Action( + name="test_tool", + arguments={"input": "modified"}, + ), + } + ] + } with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", @@ -805,7 +799,7 @@ def test_human_in_the_loop_middleware_disallowed_action() -> None: ): with pytest.raises( ValueError, - match=r"Unexpected human response: {'type': 'edit', 'args': {'action': 'test_tool', 'args': {'input': 'modified'}}}. Response action 'edit' is not allowed for tool 'test_tool'. Expected one of \['accept', 'response'\] based on the tool's configuration.", + match=r"Unexpected human decision: {'type': 'edit', 'edited_action': {'name': 'test_tool', 'arguments': {'input': 'modified'}}}. Decision type 'edit' is not allowed for tool 'test_tool'. Expected one of \['approve', 'reject'\] based on the tool's configuration.", ): middleware.after_model(state, None) @@ -814,9 +808,7 @@ def test_human_in_the_loop_middleware_mixed_auto_approved_and_interrupt() -> Non """Test HumanInTheLoopMiddleware with mix of auto-approved and interrupt tools.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "interrupt_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True} - } + interrupt_on={"interrupt_tool": {"allowed_decisions": ["approve", "edit", "reject"]}} ) ai_message = AIMessage( @@ -829,7 +821,7 @@ def test_human_in_the_loop_middleware_mixed_auto_approved_and_interrupt() -> Non state = {"messages": [HumanMessage(content="Hello"), ai_message]} def mock_accept(requests): - return [{"type": "accept", "args": None}] + return {"decisions": [{"type": "approve"}]} with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_accept): result = middleware.after_model(state, None) @@ -848,9 +840,7 @@ def test_human_in_the_loop_middleware_interrupt_request_structure() -> None: """Test that interrupt requests are structured correctly.""" middleware = HumanInTheLoopMiddleware( - interrupt_on={ - "test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True} - }, + interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}, description_prefix="Custom prefix", ) @@ -860,31 +850,34 @@ def test_human_in_the_loop_middleware_interrupt_request_structure() -> None: ) state = {"messages": [HumanMessage(content="Hello"), ai_message]} - captured_requests = [] + captured_request = None - def mock_capture_requests(requests): - captured_requests.extend(requests) - return [{"type": "accept", "args": None}] + def mock_capture_requests(request): + nonlocal captured_request + captured_request = request + return {"decisions": [{"type": "approve"}]} with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_capture_requests ): middleware.after_model(state, None) - assert len(captured_requests) == 1 - request = captured_requests[0] + assert captured_request is not None + assert "action_requests" in captured_request + assert "review_configs" in captured_request - assert "action_request" in request - assert "config" in request - assert "description" in request + assert len(captured_request["action_requests"]) == 1 + action_request = captured_request["action_requests"][0] + assert action_request["name"] == "test_tool" + assert action_request["arguments"] == {"input": "test", "location": "SF"} + assert "Custom prefix" in action_request["description"] + assert "Tool: test_tool" in action_request["description"] + assert "Args: {'input': 'test', 'location': 'SF'}" in action_request["description"] - assert request["action_request"]["action"] == "test_tool" - assert request["action_request"]["args"] == {"input": "test", "location": "SF"} - expected_config = {"allow_accept": True, "allow_edit": True, "allow_respond": True} - assert request["config"] == expected_config - assert "Custom prefix" in request["description"] - assert "Tool: test_tool" in request["description"] - assert "Args: {'input': 'test', 'location': 'SF'}" in request["description"] + assert len(captured_request["review_configs"]) == 1 + review_config = captured_request["review_configs"][0] + assert review_config["action_name"] == "test_tool" + assert review_config["allowed_decisions"] == ["approve", "edit", "reject"] def test_human_in_the_loop_middleware_boolean_configs() -> None: @@ -900,7 +893,7 @@ def test_human_in_the_loop_middleware_boolean_configs() -> None: # Test accept with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", - return_value=[{"type": "accept", "args": None}], + return_value={"decisions": [{"type": "approve"}]}, ): result = middleware.after_model(state, None) assert result is not None @@ -911,15 +904,17 @@ def test_human_in_the_loop_middleware_boolean_configs() -> None: # Test edit with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", - return_value=[ - { - "type": "edit", - "args": ActionRequest( - action="test_tool", - args={"input": "edited"}, - ), - } - ], + return_value={ + "decisions": [ + { + "type": "edit", + "edited_action": Action( + name="test_tool", + arguments={"input": "edited"}, + ), + } + ] + }, ): result = middleware.after_model(state, None) assert result is not None @@ -947,25 +942,27 @@ def test_human_in_the_loop_middleware_sequence_mismatch() -> None: # Test with too few responses with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", - return_value=[], # No responses for 1 tool call + return_value={"decisions": []}, # No responses for 1 tool call ): with pytest.raises( ValueError, - match=r"Number of human responses \(0\) does not match number of hanging tool calls \(1\)\.", + match=r"Number of human decisions \(0\) does not match number of hanging tool calls \(1\)\.", ): middleware.after_model(state, None) # Test with too many responses with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", - return_value=[ - {"type": "accept", "args": None}, - {"type": "accept", "args": None}, - ], # 2 responses for 1 tool call + return_value={ + "decisions": [ + {"type": "approve"}, + {"type": "approve"}, + ] + }, # 2 responses for 1 tool call ): with pytest.raises( ValueError, - match=r"Number of human responses \(2\) does not match number of hanging tool calls \(1\)\.", + match=r"Number of human decisions \(2\) does not match number of hanging tool calls \(1\)\.", ): middleware.after_model(state, None) @@ -979,8 +976,14 @@ def test_human_in_the_loop_middleware_description_as_callable() -> None: middleware = HumanInTheLoopMiddleware( interrupt_on={ - "tool_with_callable": {"allow_accept": True, "description": custom_description}, - "tool_with_string": {"allow_accept": True, "description": "Static description"}, + "tool_with_callable": { + "allowed_decisions": ["approve"], + "description": custom_description, + }, + "tool_with_string": { + "allowed_decisions": ["approve"], + "description": "Static description", + }, } ) @@ -993,26 +996,30 @@ def test_human_in_the_loop_middleware_description_as_callable() -> None: ) state = {"messages": [HumanMessage(content="Hello"), ai_message]} - captured_requests = [] + captured_request = None - def mock_capture_requests(requests): - captured_requests.extend(requests) - return [{"type": "accept"}, {"type": "accept"}] + def mock_capture_requests(request): + nonlocal captured_request + captured_request = request + return {"decisions": [{"type": "approve"}, {"type": "approve"}]} with patch( "langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_capture_requests ): middleware.after_model(state, None) - assert len(captured_requests) == 2 + assert captured_request is not None + assert "action_requests" in captured_request + assert len(captured_request["action_requests"]) == 2 # Check callable description assert ( - captured_requests[0]["description"] == "Custom: tool_with_callable with args {'x': 1}" + captured_request["action_requests"][0]["description"] + == "Custom: tool_with_callable with args {'x': 1}" ) # Check string description - assert captured_requests[1]["description"] == "Static description" + assert captured_request["action_requests"][1]["description"] == "Static description" # Tests for AnthropicPromptCachingMiddleware