github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2026-07-13 20:47:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(langchain_v1): refactoring HITL API (#33397)

Browse Source 
Easiest to review side by side (not inline)

* Adding `dict` type requests + responses so that we can ship config w/
interrupts. Also more extensible.
* Keeping things generic in terms of `interrupt_on` rather than
`tool_config`
* Renaming allowed decisions -- approve, edit, reject
* Draws differentiation between actions (requested + performed by the
agent), in this case tool calls, though we generalize beyond that and
decisions - human feedback for said actions

New request structure

```py
class Action(TypedDict):
    """Represents an action with a name and arguments."""

    name: str
    """The type or name of action being requested (e.g., "add_numbers")."""

    arguments: dict[str, Any]
    """Key-value pairs of arguments needed for the action (e.g., {"a": 1, "b": 2})."""


DecisionType = Literal["approve", "edit", "reject"]


class ReviewConfig(TypedDict):
    """Policy for reviewing a HITL request."""

    action_name: str
    """Name of the action associated with this review configuration."""

    allowed_decisions: list[DecisionType]
    """The decisions that are allowed for this request."""

    description: NotRequired[str]
    """The description of the action to be reviewed."""

    arguments_schema: NotRequired[dict[str, Any]]
    """JSON schema for the arguments associated with the action, if edits are allowed."""

class HITLRequest(TypedDict):
    """Request for human feedback on a sequence of actions requested by a model."""

    action_requests: list[Action]
    """A list of agent actions for human review."""

    review_configs: list[ReviewConfig]
    """Review configuration for all possible actions."""
```

New response structure

```py
class ApproveDecision(TypedDict):
    """Response when a human approves the action."""

    type: Literal["approve"]
    """The type of response when a human approves the action."""


class EditDecision(TypedDict):
    """Response when a human edits the action."""

    type: Literal["edit"]
    """The type of response when a human edits the action."""

    edited_action: Action
    """Edited action for the agent to perform.

    Ex: for a tool call, a human reviewer can edit the tool name and args.
    """


class RejectDecision(TypedDict):
    """Response when a human rejects the action."""

    type: Literal["reject"]
    """The type of response when a human rejects the action."""

    message: NotRequired[str]
    """The message sent to the model explaining why the action was rejected."""


Decision = ApproveDecision | EditDecision | RejectDecision


class HITLResponse(TypedDict):
    """Response payload for a HITLRequest."""

    decisions: list[Decision]
    """The decisions made by the human."""
```

User facing API:

NEW

```py
HumanInTheLoopMiddleware(interrupt_on={
    'send_email': True,
    # can also use a callable for description that takes tool call, state, and runtime
    'execute_sql': {
        'allowed_decisions': ['approve', 'edit', 'reject'], 
        'description': 'please review sensitive tool execution'},
    }
})

Command(resume={"decisions": [{"type": "approve"}, {"type": "reject": "message": "db down"}]})
```

OLD

```py
HumanInTheLoopMiddleware(interrupt_on={
    'send_email': True,
    'execute_sql': {
        'allow_accept': True, 
        'allow_edit': True, 
        'allow_respond': True, 
        description='please review sensitive tool execution'
    },
})

Command(resume=[{"type": "approve"}, {"type": "reject": "message": "db down"}])
```
This commit is contained in:
Sydney Runkle
2025-10-09 17:51:28 -04:00
committed by GitHub
parent 5f9e3e33cd
commit c99773b652
3 changed files with 337 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
libs/langchain_v1/langchain/agents/middleware/__init__.py
View File

@@ -4,7 +4,10 @@ from .context_editing import (
ClearToolUsesEdit,
ContextEditingMiddleware,
)
from .human_in_the_loop import HumanInTheLoopMiddleware
from .human_in_the_loop import (
HumanInTheLoopMiddleware,
InterruptOnConfig,
)
from .model_call_limit import ModelCallLimitMiddleware
from .model_fallback import ModelFallbackMiddleware
from .pii import PIIDetectionError, PIIMiddleware
@@ -34,6 +37,7 @@ __all__ = [
"ClearToolUsesEdit",
"ContextEditingMiddleware",
"HumanInTheLoopMiddleware",
"InterruptOnConfig",
"LLMToolSelectorMiddleware",
"ModelCallLimitMiddleware",
"ModelFallbackMiddleware",

345
libs/langchain_v1/langchain/agents/middleware/human_in_the_loop.py
View File

@@ -10,89 +10,93 @@ from typing_extensions import NotRequired, TypedDict
from langchain.agents.middleware.types import AgentMiddleware, AgentState
class HumanInTheLoopConfig(TypedDict):
"""Configuration that defines what actions are allowed for a human interrupt.
class Action(TypedDict):
"""Represents an action with a name and arguments."""
This controls the available interaction options when the graph is paused for human input.
"""
allow_accept: NotRequired[bool]
"""Whether the human can approve the current action without changes."""
allow_edit: NotRequired[bool]
"""Whether the human can approve the current action with edited content."""
allow_respond: NotRequired[bool]
"""Whether the human can reject the current action with feedback."""
class ActionRequest(TypedDict):
"""Represents a request with a name and arguments."""
action: str
name: str
"""The type or name of action being requested (e.g., "add_numbers")."""
args: dict
arguments: dict[str, Any]
"""Key-value pairs of arguments needed for the action (e.g., {"a": 1, "b": 2})."""
class HumanInTheLoopRequest(TypedDict):
"""Represents an interrupt triggered by the graph that requires human intervention.
class ActionRequest(TypedDict):
"""Represents an action request with a name, arguments, and description."""
Example:
```python
# Extract a tool call from the state and create an interrupt request
request = HumanInterrupt(
action_request=ActionRequest(
action="run_command", # The action being requested
args={"command": "ls", "args": ["-l"]}, # Arguments for the action
),
config=HumanInTheLoopConfig(
allow_accept=True, # Allow approval
allow_respond=True, # Allow rejection with feedback
allow_edit=False, # Don't allow approval with edits
),
description="Please review the command before execution",
)
# Send the interrupt request and get the response
response = interrupt([request])[0]
```
"""
name: str
"""The name of the action being requested."""
action_request: ActionRequest
"""The specific action being requested from the human."""
config: HumanInTheLoopConfig
"""Configuration defining what response types are allowed."""
description: str | None
"""Optional detailed description of what input is needed."""
arguments: dict[str, Any]
"""Key-value pairs of arguments needed for the action (e.g., {"a": 1, "b": 2})."""
description: NotRequired[str]
"""The description of the action to be reviewed."""
class AcceptPayload(TypedDict):
DecisionType = Literal["approve", "edit", "reject"]
class ReviewConfig(TypedDict):
"""Policy for reviewing a HITL request."""
action_name: str
"""Name of the action associated with this review configuration."""
allowed_decisions: list[DecisionType]
"""The decisions that are allowed for this request."""
arguments_schema: NotRequired[dict[str, Any]]
"""JSON schema for the arguments associated with the action, if edits are allowed."""
class HITLRequest(TypedDict):
"""Request for human feedback on a sequence of actions requested by a model."""
action_requests: list[ActionRequest]
"""A list of agent actions for human review."""
review_configs: list[ReviewConfig]
"""Review configuration for all possible actions."""
class ApproveDecision(TypedDict):
"""Response when a human approves the action."""
type: Literal["accept"]
type: Literal["approve"]
"""The type of response when a human approves the action."""
class ResponsePayload(TypedDict):
"""Response when a human rejects the action."""
type: Literal["response"]
"""The type of response when a human rejects the action."""
args: NotRequired[str]
"""The message to be sent to the model explaining why the action was rejected."""
class EditPayload(TypedDict):
class EditDecision(TypedDict):
"""Response when a human edits the action."""
type: Literal["edit"]
"""The type of response when a human edits the action."""
args: ActionRequest
"""The action request with the edited content."""
edited_action: Action
"""Edited action for the agent to perform.
Ex: for a tool call, a human reviewer can edit the tool name and args.
"""
HumanInTheLoopResponse = AcceptPayload | ResponsePayload | EditPayload
"""Aggregated response type for all possible human in the loop responses."""
class RejectDecision(TypedDict):
"""Response when a human rejects the action."""
type: Literal["reject"]
"""The type of response when a human rejects the action."""
message: NotRequired[str]
"""The message sent to the model explaining why the action was rejected."""
Decision = ApproveDecision | EditDecision | RejectDecision
class HITLResponse(TypedDict):
"""Response payload for a HITLRequest."""
decisions: list[Decision]
"""The decisions made by the human."""
class _DescriptionFactory(Protocol):
@@ -103,15 +107,15 @@ class _DescriptionFactory(Protocol):
...
class ToolConfig(TypedDict):
"""Configuration for a tool requiring human in the loop."""
class InterruptOnConfig(TypedDict):
"""Configuration for an action requiring human in the loop.
This is the configuration format used in the `HumanInTheLoopMiddleware.__init__` method.
"""
allowed_decisions: list[DecisionType]
"""The decisions that are allowed for this action."""
allow_accept: NotRequired[bool]
"""Whether the human can approve the current action without changes."""
allow_edit: NotRequired[bool]
"""Whether the human can approve the current action with edited content."""
allow_respond: NotRequired[bool]
"""Whether the human can reject the current action with feedback."""
description: NotRequired[str | _DescriptionFactory]
"""The description attached to the request for human input.
@@ -124,7 +128,7 @@ class ToolConfig(TypedDict):
```python
# Static string description
config = ToolConfig(
allow_accept=True,
allowed_decisions=["approve", "reject"],
description="Please review this tool execution"
)
@@ -146,6 +150,8 @@ class ToolConfig(TypedDict):
)
```
"""
arguments_schema: NotRequired[dict[str, Any]]
"""JSON schema for the arguments associated with the action, if edits are allowed."""
class HumanInTheLoopMiddleware(AgentMiddleware):
@@ -153,7 +159,7 @@ class HumanInTheLoopMiddleware(AgentMiddleware):
def __init__(
self,
interrupt_on: dict[str, bool | ToolConfig],
interrupt_on: dict[str, bool | InterruptOnConfig],
*,
description_prefix: str = "Tool execution requires approval",
) -> None:
@@ -163,32 +169,106 @@ class HumanInTheLoopMiddleware(AgentMiddleware):
interrupt_on: Mapping of tool name to allowed actions.
If a tool doesn't have an entry, it's auto-approved by default.
* `True` indicates all actions are allowed: accept, edit, and respond.
* `True` indicates all decisions are allowed: approve, edit, and reject.
* `False` indicates that the tool is auto-approved.
* `ToolConfig` indicates the specific actions allowed for this tool.
The ToolConfig can include a `description` field (str or callable) for
* `InterruptOnConfig` indicates the specific decisions allowed for this tool.
The InterruptOnConfig can include a `description` field (str or callable) for
custom formatting of the interrupt description.
description_prefix: The prefix to use when constructing action requests.
This is used to provide context about the tool call and the action being requested.
Not used if a tool has a `description` in its ToolConfig.
Not used if a tool has a `description` in its InterruptOnConfig.
"""
super().__init__()
resolved_tool_configs: dict[str, ToolConfig] = {}
resolved_configs: dict[str, InterruptOnConfig] = {}
for tool_name, tool_config in interrupt_on.items():
if isinstance(tool_config, bool):
if tool_config is True:
resolved_tool_configs[tool_name] = ToolConfig(
allow_accept=True,
allow_edit=True,
allow_respond=True,
resolved_configs[tool_name] = InterruptOnConfig(
allowed_decisions=["approve", "edit", "reject"]
)
elif any(
tool_config.get(x, False) for x in ["allow_accept", "allow_edit", "allow_respond"]
):
resolved_tool_configs[tool_name] = tool_config
self.interrupt_on = resolved_tool_configs
elif tool_config.get("allowed_decisions"):
resolved_configs[tool_name] = tool_config
self.interrupt_on = resolved_configs
self.description_prefix = description_prefix
def _create_action_and_config(
self,
tool_call: ToolCall,
config: InterruptOnConfig,
state: AgentState,
runtime: Runtime,
) -> tuple[ActionRequest, ReviewConfig]:
"""Create an ActionRequest and ReviewConfig for a tool call."""
tool_name = tool_call["name"]
tool_args = tool_call["args"]
# Generate description using the description field (str or callable)
description_value = config.get("description")
if callable(description_value):
description = description_value(tool_call, state, runtime)
elif description_value is not None:
description = description_value
else:
description = f"{self.description_prefix}\n\nTool: {tool_name}\nArgs: {tool_args}"
# Create ActionRequest with description
action_request = ActionRequest(
name=tool_name,
arguments=tool_args,
description=description,
)
# Create ReviewConfig
# eventually can get tool information and populate arguments_schema from there
review_config = ReviewConfig(
action_name=tool_name,
allowed_decisions=config["allowed_decisions"],
)
return action_request, review_config
def _process_decision(
self,
decision: Decision,
tool_call: ToolCall,
config: InterruptOnConfig,
) -> tuple[ToolCall | None, ToolMessage | None]:
"""Process a single decision and return the revised tool call and optional tool message."""
allowed_decisions = config["allowed_decisions"]
if decision["type"] == "approve" and "approve" in allowed_decisions:
return tool_call, None
if decision["type"] == "edit" and "edit" in allowed_decisions:
edited_action = decision["edited_action"]
return (
ToolCall(
type="tool_call",
name=edited_action["name"],
args=edited_action["arguments"],
id=tool_call["id"],
),
None,
)
if decision["type"] == "reject" and "reject" in allowed_decisions:
# Create a tool message with the human's text response
content = decision.get("message") or (
f"User rejected the tool call for `{tool_call['name']}` with id {tool_call['id']}"
)
tool_message = ToolMessage(
content=content,
name=tool_call["name"],
tool_call_id=tool_call["id"],
status="error",
)
return tool_call, tool_message
msg = (
f"Unexpected human decision: {decision}. "
f"Decision type '{decision.get('type')}' "
f"is not allowed for tool '{tool_call['name']}'. "
f"Expected one of {allowed_decisions} based on the tool's configuration."
)
raise ValueError(msg)
def after_model(self, state: AgentState, runtime: Runtime) -> dict[str, Any] | None:
"""Trigger interrupt flows for relevant tool calls after an AIMessage."""
messages = state["messages"]
@@ -216,87 +296,50 @@ class HumanInTheLoopMiddleware(AgentMiddleware):
revised_tool_calls: list[ToolCall] = auto_approved_tool_calls.copy()
artificial_tool_messages: list[ToolMessage] = []
# Create interrupt requests for all tools that need approval
interrupt_requests: list[HumanInTheLoopRequest] = []
# Create action requests and review configs for all tools that need approval
action_requests: list[ActionRequest] = []
review_configs: list[ReviewConfig] = []
for tool_call in interrupt_tool_calls:
tool_name = tool_call["name"]
tool_args = tool_call["args"]
config = self.interrupt_on[tool_name]
config = self.interrupt_on[tool_call["name"]]
# Generate description using the description field (str or callable)
description_value = config.get("description")
if callable(description_value):
description = description_value(tool_call, state, runtime)
elif description_value is not None:
description = description_value
else:
description = f"{self.description_prefix}\n\nTool: {tool_name}\nArgs: {tool_args}"
# Create ActionRequest and ReviewConfig using helper method
action_request, review_config = self._create_action_and_config(
tool_call, config, state, runtime
)
action_requests.append(action_request)
review_configs.append(review_config)
request: HumanInTheLoopRequest = {
"action_request": ActionRequest(
action=tool_name,
args=tool_args,
),
"config": config,
"description": description,
}
interrupt_requests.append(request)
# Create single HITLRequest with all actions and configs
hitl_request = HITLRequest(
action_requests=action_requests,
review_configs=review_configs,
)
responses: list[HumanInTheLoopResponse] = interrupt(interrupt_requests)
# Send interrupt and get response
hitl_response: HITLResponse = interrupt(hitl_request)
decisions = hitl_response["decisions"]
# Validate that the number of responses matches the number of interrupt tool calls
if (responses_len := len(responses)) != (
# Validate that the number of decisions matches the number of interrupt tool calls
if (decisions_len := len(decisions)) != (
interrupt_tool_calls_len := len(interrupt_tool_calls)
):
msg = (
f"Number of human responses ({responses_len}) does not match "
f"Number of human decisions ({decisions_len}) does not match "
f"number of hanging tool calls ({interrupt_tool_calls_len})."
)
raise ValueError(msg)
for i, response in enumerate(responses):
# Process each decision using helper method
for i, decision in enumerate(decisions):
tool_call = interrupt_tool_calls[i]
config = self.interrupt_on[tool_call["name"]]
if response["type"] == "accept" and config.get("allow_accept"):
revised_tool_calls.append(tool_call)
elif response["type"] == "edit" and config.get("allow_edit"):
edited_action = response["args"]
revised_tool_calls.append(
ToolCall(
type="tool_call",
name=edited_action["action"],
args=edited_action["args"],
id=tool_call["id"],
)
)
elif response["type"] == "response" and config.get("allow_respond"):
# Create a tool message with the human's text response
content = response.get("args") or (
f"User rejected the tool call for `{tool_call['name']}` "
f"with id {tool_call['id']}"
)
tool_message = ToolMessage(
content=content,
name=tool_call["name"],
tool_call_id=tool_call["id"],
status="error",
)
revised_tool_calls.append(tool_call)
revised_tool_call, tool_message = self._process_decision(decision, tool_call, config)
if revised_tool_call:
revised_tool_calls.append(revised_tool_call)
if tool_message:
artificial_tool_messages.append(tool_message)
else:
allowed_actions = [
action
for action in ["accept", "edit", "response"]
if config.get(f"allow_{'respond' if action == 'response' else action}")
]
msg = (
f"Unexpected human response: {response}. "
f"Response action '{response.get('type')}' "
f"is not allowed for tool '{tool_call['name']}'. "
f"Expected one of {allowed_actions} based on the tool's configuration."
)
raise ValueError(msg)
# Update the AI message to only include approved tool calls
last_ai_msg.tool_calls = revised_tool_calls

269
libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py
View File

@@ -29,7 +29,7 @@ from syrupy.assertion import SnapshotAssertion
from typing_extensions import Annotated
from langchain.agents.middleware.human_in_the_loop import (
ActionRequest,
Action,
HumanInTheLoopMiddleware,
)
from langchain.agents.middleware.planning import (
@@ -463,14 +463,12 @@ def test_human_in_the_loop_middleware_initialization() -> None:
"""Test HumanInTheLoopMiddleware initialization."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True}
},
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}},
description_prefix="Custom prefix",
)
assert middleware.interrupt_on == {
"test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True}
"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}
}
assert middleware.description_prefix == "Custom prefix"
@@ -479,9 +477,7 @@ def test_human_in_the_loop_middleware_no_interrupts_needed() -> None:
"""Test HumanInTheLoopMiddleware when no interrupts are needed."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
# Test with no messages
@@ -508,9 +504,7 @@ def test_human_in_the_loop_middleware_single_tool_accept() -> None:
"""Test HumanInTheLoopMiddleware with single tool accept response."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
ai_message = AIMessage(
@@ -520,7 +514,7 @@ def test_human_in_the_loop_middleware_single_tool_accept() -> None:
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_accept(requests):
return [{"type": "accept", "args": None}]
return {"decisions": [{"type": "approve"}]}
with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_accept):
result = middleware.after_model(state, None)
@@ -543,9 +537,7 @@ def test_human_in_the_loop_middleware_single_tool_accept() -> None:
def test_human_in_the_loop_middleware_single_tool_edit() -> None:
"""Test HumanInTheLoopMiddleware with single tool edit response."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
ai_message = AIMessage(
@@ -555,15 +547,17 @@ def test_human_in_the_loop_middleware_single_tool_edit() -> None:
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_edit(requests):
return [
{
"type": "edit",
"args": ActionRequest(
action="test_tool",
args={"input": "edited"},
),
}
]
return {
"decisions": [
{
"type": "edit",
"edited_action": Action(
name="test_tool",
arguments={"input": "edited"},
),
}
]
}
with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_edit):
result = middleware.after_model(state, None)
@@ -578,9 +572,7 @@ def test_human_in_the_loop_middleware_single_tool_response() -> None:
"""Test HumanInTheLoopMiddleware with single tool response with custom message."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
ai_message = AIMessage(
@@ -590,7 +582,7 @@ def test_human_in_the_loop_middleware_single_tool_response() -> None:
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_response(requests):
return [{"type": "response", "args": "Custom response message"}]
return {"decisions": [{"type": "reject", "message": "Custom response message"}]}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_response
@@ -611,8 +603,8 @@ def test_human_in_the_loop_middleware_multiple_tools_mixed_responses() -> None:
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"get_forecast": {"allow_accept": True, "allow_edit": True, "allow_respond": True},
"get_temperature": {"allow_accept": True, "allow_edit": True, "allow_respond": True},
"get_forecast": {"allowed_decisions": ["approve", "edit", "reject"]},
"get_temperature": {"allowed_decisions": ["approve", "edit", "reject"]},
}
)
@@ -626,10 +618,12 @@ def test_human_in_the_loop_middleware_multiple_tools_mixed_responses() -> None:
state = {"messages": [HumanMessage(content="What's the weather?"), ai_message]}
def mock_mixed_responses(requests):
return [
{"type": "accept", "args": None},
{"type": "response", "args": "User rejected this tool call"},
]
return {
"decisions": [
{"type": "approve"},
{"type": "reject", "message": "User rejected this tool call"},
]
}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_mixed_responses
@@ -659,8 +653,8 @@ def test_human_in_the_loop_middleware_multiple_tools_edit_responses() -> None:
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"get_forecast": {"allow_accept": True, "allow_edit": True, "allow_respond": True},
"get_temperature": {"allow_accept": True, "allow_edit": True, "allow_respond": True},
"get_forecast": {"allowed_decisions": ["approve", "edit", "reject"]},
"get_temperature": {"allowed_decisions": ["approve", "edit", "reject"]},
}
)
@@ -674,22 +668,24 @@ def test_human_in_the_loop_middleware_multiple_tools_edit_responses() -> None:
state = {"messages": [HumanMessage(content="What's the weather?"), ai_message]}
def mock_edit_responses(requests):
return [
{
"type": "edit",
"args": ActionRequest(
action="get_forecast",
args={"location": "New York"},
),
},
{
"type": "edit",
"args": ActionRequest(
action="get_temperature",
args={"location": "New York"},
),
},
]
return {
"decisions": [
{
"type": "edit",
"edited_action": Action(
name="get_forecast",
arguments={"location": "New York"},
),
},
{
"type": "edit",
"edited_action": Action(
name="get_temperature",
arguments={"location": "New York"},
),
},
]
}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_edit_responses
@@ -710,9 +706,7 @@ def test_human_in_the_loop_middleware_edit_with_modified_args() -> None:
"""Test HumanInTheLoopMiddleware with edit action that includes modified args."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
ai_message = AIMessage(
@@ -722,15 +716,17 @@ def test_human_in_the_loop_middleware_edit_with_modified_args() -> None:
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_edit_with_args(requests):
return [
{
"type": "edit",
"args": ActionRequest(
action="test_tool",
args={"input": "modified"},
),
}
]
return {
"decisions": [
{
"type": "edit",
"edited_action": Action(
name="test_tool",
arguments={"input": "modified"},
),
}
]
}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt",
@@ -750,9 +746,7 @@ def test_human_in_the_loop_middleware_edit_with_modified_args() -> None:
def test_human_in_the_loop_middleware_unknown_response_type() -> None:
"""Test HumanInTheLoopMiddleware with unknown response type."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
ai_message = AIMessage(
@@ -762,12 +756,12 @@ def test_human_in_the_loop_middleware_unknown_response_type() -> None:
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_unknown(requests):
return [{"type": "unknown", "args": None}]
return {"decisions": [{"type": "unknown"}]}
with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_unknown):
with pytest.raises(
ValueError,
match=r"Unexpected human response: {'type': 'unknown', 'args': None}. Response action 'unknown' is not allowed for tool 'test_tool'. Expected one of \['accept', 'edit', 'response'\] based on the tool's configuration.",
match=r"Unexpected human decision: {'type': 'unknown'}. Decision type 'unknown' is not allowed for tool 'test_tool'. Expected one of \['approve', 'edit', 'reject'\] based on the tool's configuration.",
):
middleware.after_model(state, None)
@@ -777,9 +771,7 @@ def test_human_in_the_loop_middleware_disallowed_action() -> None:
# edit is not allowed by tool config
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_respond": True, "allow_edit": False, "allow_accept": True}
}
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "reject"]}}
)
ai_message = AIMessage(
@@ -789,15 +781,17 @@ def test_human_in_the_loop_middleware_disallowed_action() -> None:
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_disallowed_action(requests):
return [
{
"type": "edit",
"args": ActionRequest(
action="test_tool",
args={"input": "modified"},
),
}
]
return {
"decisions": [
{
"type": "edit",
"edited_action": Action(
name="test_tool",
arguments={"input": "modified"},
),
}
]
}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt",
@@ -805,7 +799,7 @@ def test_human_in_the_loop_middleware_disallowed_action() -> None:
):
with pytest.raises(
ValueError,
match=r"Unexpected human response: {'type': 'edit', 'args': {'action': 'test_tool', 'args': {'input': 'modified'}}}. Response action 'edit' is not allowed for tool 'test_tool'. Expected one of \['accept', 'response'\] based on the tool's configuration.",
match=r"Unexpected human decision: {'type': 'edit', 'edited_action': {'name': 'test_tool', 'arguments': {'input': 'modified'}}}. Decision type 'edit' is not allowed for tool 'test_tool'. Expected one of \['approve', 'reject'\] based on the tool's configuration.",
):
middleware.after_model(state, None)
@@ -814,9 +808,7 @@ def test_human_in_the_loop_middleware_mixed_auto_approved_and_interrupt() -> Non
"""Test HumanInTheLoopMiddleware with mix of auto-approved and interrupt tools."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"interrupt_tool": {"allow_respond": True, "allow_edit": True, "allow_accept": True}
}
interrupt_on={"interrupt_tool": {"allowed_decisions": ["approve", "edit", "reject"]}}
)
ai_message = AIMessage(
@@ -829,7 +821,7 @@ def test_human_in_the_loop_middleware_mixed_auto_approved_and_interrupt() -> Non
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
def mock_accept(requests):
return [{"type": "accept", "args": None}]
return {"decisions": [{"type": "approve"}]}
with patch("langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_accept):
result = middleware.after_model(state, None)
@@ -848,9 +840,7 @@ def test_human_in_the_loop_middleware_interrupt_request_structure() -> None:
"""Test that interrupt requests are structured correctly."""
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"test_tool": {"allow_accept": True, "allow_edit": True, "allow_respond": True}
},
interrupt_on={"test_tool": {"allowed_decisions": ["approve", "edit", "reject"]}},
description_prefix="Custom prefix",
)
@@ -860,31 +850,34 @@ def test_human_in_the_loop_middleware_interrupt_request_structure() -> None:
)
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
captured_requests = []
captured_request = None
def mock_capture_requests(requests):
captured_requests.extend(requests)
return [{"type": "accept", "args": None}]
def mock_capture_requests(request):
nonlocal captured_request
captured_request = request
return {"decisions": [{"type": "approve"}]}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_capture_requests
):
middleware.after_model(state, None)
assert len(captured_requests) == 1
request = captured_requests[0]
assert captured_request is not None
assert "action_requests" in captured_request
assert "review_configs" in captured_request
assert "action_request" in request
assert "config" in request
assert "description" in request
assert len(captured_request["action_requests"]) == 1
action_request = captured_request["action_requests"][0]
assert action_request["name"] == "test_tool"
assert action_request["arguments"] == {"input": "test", "location": "SF"}
assert "Custom prefix" in action_request["description"]
assert "Tool: test_tool" in action_request["description"]
assert "Args: {'input': 'test', 'location': 'SF'}" in action_request["description"]
assert request["action_request"]["action"] == "test_tool"
assert request["action_request"]["args"] == {"input": "test", "location": "SF"}
expected_config = {"allow_accept": True, "allow_edit": True, "allow_respond": True}
assert request["config"] == expected_config
assert "Custom prefix" in request["description"]
assert "Tool: test_tool" in request["description"]
assert "Args: {'input': 'test', 'location': 'SF'}" in request["description"]
assert len(captured_request["review_configs"]) == 1
review_config = captured_request["review_configs"][0]
assert review_config["action_name"] == "test_tool"
assert review_config["allowed_decisions"] == ["approve", "edit", "reject"]
def test_human_in_the_loop_middleware_boolean_configs() -> None:
@@ -900,7 +893,7 @@ def test_human_in_the_loop_middleware_boolean_configs() -> None:
# Test accept
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt",
return_value=[{"type": "accept", "args": None}],
return_value={"decisions": [{"type": "approve"}]},
):
result = middleware.after_model(state, None)
assert result is not None
@@ -911,15 +904,17 @@ def test_human_in_the_loop_middleware_boolean_configs() -> None:
# Test edit
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt",
return_value=[
{
"type": "edit",
"args": ActionRequest(
action="test_tool",
args={"input": "edited"},
),
}
],
return_value={
"decisions": [
{
"type": "edit",
"edited_action": Action(
name="test_tool",
arguments={"input": "edited"},
),
}
]
},
):
result = middleware.after_model(state, None)
assert result is not None
@@ -947,25 +942,27 @@ def test_human_in_the_loop_middleware_sequence_mismatch() -> None:
# Test with too few responses
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt",
return_value=[], # No responses for 1 tool call
return_value={"decisions": []}, # No responses for 1 tool call
):
with pytest.raises(
ValueError,
match=r"Number of human responses \(0\) does not match number of hanging tool calls \(1\)\.",
match=r"Number of human decisions \(0\) does not match number of hanging tool calls \(1\)\.",
):
middleware.after_model(state, None)
# Test with too many responses
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt",
return_value=[
{"type": "accept", "args": None},
{"type": "accept", "args": None},
], # 2 responses for 1 tool call
return_value={
"decisions": [
{"type": "approve"},
{"type": "approve"},
]
}, # 2 responses for 1 tool call
):
with pytest.raises(
ValueError,
match=r"Number of human responses \(2\) does not match number of hanging tool calls \(1\)\.",
match=r"Number of human decisions \(2\) does not match number of hanging tool calls \(1\)\.",
):
middleware.after_model(state, None)
@@ -979,8 +976,14 @@ def test_human_in_the_loop_middleware_description_as_callable() -> None:
middleware = HumanInTheLoopMiddleware(
interrupt_on={
"tool_with_callable": {"allow_accept": True, "description": custom_description},
"tool_with_string": {"allow_accept": True, "description": "Static description"},
"tool_with_callable": {
"allowed_decisions": ["approve"],
"description": custom_description,
},
"tool_with_string": {
"allowed_decisions": ["approve"],
"description": "Static description",
},
}
)
@@ -993,26 +996,30 @@ def test_human_in_the_loop_middleware_description_as_callable() -> None:
)
state = {"messages": [HumanMessage(content="Hello"), ai_message]}
captured_requests = []
captured_request = None
def mock_capture_requests(requests):
captured_requests.extend(requests)
return [{"type": "accept"}, {"type": "accept"}]
def mock_capture_requests(request):
nonlocal captured_request
captured_request = request
return {"decisions": [{"type": "approve"}, {"type": "approve"}]}
with patch(
"langchain.agents.middleware.human_in_the_loop.interrupt", side_effect=mock_capture_requests
):
middleware.after_model(state, None)
assert len(captured_requests) == 2
assert captured_request is not None
assert "action_requests" in captured_request
assert len(captured_request["action_requests"]) == 2
# Check callable description
assert (
captured_requests[0]["description"] == "Custom: tool_with_callable with args {'x': 1}"
captured_request["action_requests"][0]["description"]
== "Custom: tool_with_callable with args {'x': 1}"
)
# Check string description
assert captured_requests[1]["description"] == "Static description"
assert captured_request["action_requests"][1]["description"] == "Static description"
# Tests for AnthropicPromptCachingMiddleware