From df03267edfa36859476b2b4fbfca5c4152827f76 Mon Sep 17 00:00:00 2001 From: Sergey Kozlov Date: Mon, 20 Nov 2023 08:45:43 +0600 Subject: [PATCH] Fix tool arguments formatting in StructuredChatAgent (#10480) In the `FORMAT_INSTRUCTIONS` template, 4 curly braces (escaping) are used to get single curly brace after formatting: ``` "{{{ ... }}}}" -> format_instructions.format() -> "{{ ... }}" -> template.format() -> "{ ... }". ``` Tool's `args_schema` string contains single braces `{ ... }`, and is also transformed to `{{{{ ... }}}}` form. But this is not really correct since there is only one `format()` call: ``` "{{{{ ... }}}}" -> template.format() -> "{{ ... }}". ``` As a result we get double curly braces in the prompt: ```` Respond to the human as helpfully and accurately as possible. You have access to the following tools: foo: Test tool FOO, args: {{'tool_input': {{'type': 'string'}}}} # <--- !!! ... Provide only ONE action per $JSON_BLOB, as shown: ``` { "action": $TOOL_NAME, "action_input": $INPUT } ``` ```` This PR fixes curly braces escaping in the `args_schema` to have single braces in the final prompt: ```` Respond to the human as helpfully and accurately as possible. You have access to the following tools: foo: Test tool FOO, args: {'tool_input': {'type': 'string'}} # <--- !!! ... Provide only ONE action per $JSON_BLOB, as shown: ``` { "action": $TOOL_NAME, "action_input": $INPUT } ``` ```` --------- Co-authored-by: Sergey Kozlov --- .../langchain/agents/structured_chat/base.py | 2 +- .../unit_tests/agents/test_structured_chat.py | 144 +++++++++++++++++- 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/libs/langchain/langchain/agents/structured_chat/base.py b/libs/langchain/langchain/agents/structured_chat/base.py index d325e3463ce..7c6fdd06993 100644 --- a/libs/langchain/langchain/agents/structured_chat/base.py +++ b/libs/langchain/langchain/agents/structured_chat/base.py @@ -81,7 +81,7 @@ class StructuredChatAgent(Agent): ) -> BasePromptTemplate: tool_strings = [] for tool in tools: - args_schema = re.sub("}", "}}}}", re.sub("{", "{{{{", str(tool.args))) + args_schema = re.sub("}", "}}", re.sub("{", "{{", str(tool.args))) tool_strings.append(f"{tool.name}: {tool.description}, args: {args_schema}") formatted_tools = "\n".join(tool_strings) tool_names = ", ".join([tool.name for tool in tools]) diff --git a/libs/langchain/tests/unit_tests/agents/test_structured_chat.py b/libs/langchain/tests/unit_tests/agents/test_structured_chat.py index 24020df0fe7..8e77f6be204 100644 --- a/libs/langchain/tests/unit_tests/agents/test_structured_chat.py +++ b/libs/langchain/tests/unit_tests/agents/test_structured_chat.py @@ -1,8 +1,16 @@ """Unittests for langchain.agents.chat package.""" -from typing import Tuple +from textwrap import dedent +from typing import Any, Tuple +from langchain.agents.structured_chat.base import StructuredChatAgent from langchain.agents.structured_chat.output_parser import StructuredChatOutputParser +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) from langchain.schema import AgentAction, AgentFinish +from langchain.tools import Tool output_parser = StructuredChatOutputParser() @@ -103,3 +111,137 @@ def test_parse_case_matched_and_final_answer() -> None: output, log = get_action_and_input(llm_output) assert output == "This is the final answer" assert log == llm_output + + +# TODO: add more tests. +# Test: StructuredChatAgent.create_prompt() method. +class TestCreatePrompt: + # Test: Output should be a ChatPromptTemplate with sys and human messages. + def test_create_prompt_output(self) -> None: + prompt = StructuredChatAgent.create_prompt( + [Tool(name="foo", description="Test tool FOO", func=lambda x: x)] + ) + + assert isinstance(prompt, ChatPromptTemplate) + assert len(prompt.messages) == 2 + assert isinstance(prompt.messages[0], SystemMessagePromptTemplate) + assert isinstance(prompt.messages[1], HumanMessagePromptTemplate) + + # Test: Format with a single tool. + def test_system_message_single_tool(self) -> None: + prompt: Any = StructuredChatAgent.create_prompt( + [Tool(name="foo", description="Test tool FOO", func=lambda x: x)] + ) + actual = prompt.messages[0].prompt.format() + + expected = dedent( + """ + Respond to the human as helpfully and accurately as possible. You have access to the following tools: + + foo: Test tool FOO, args: {'tool_input': {'type': 'string'}} + + Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input). + + Valid "action" values: "Final Answer" or foo + + Provide only ONE action per $JSON_BLOB, as shown: + + ``` + { + "action": $TOOL_NAME, + "action_input": $INPUT + } + ``` + + Follow this format: + + Question: input question to answer + Thought: consider previous and subsequent steps + Action: + ``` + $JSON_BLOB + ``` + Observation: action result + ... (repeat Thought/Action/Observation N times) + Thought: I know what to respond + Action: + ``` + { + "action": "Final Answer", + "action_input": "Final response to human" + } + ``` + + Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:. + Thought: + """ # noqa: E501 + ).strip() + + assert actual == expected + + # Test: Format with multiple tools. + # + # Check: + # + # You have access to the following tools: + # ... + # + # and + # + # Valid "action" values: "Final Answer" or ... + # + def test_system_message_multiple_tools(self) -> None: + prompt: Any = StructuredChatAgent.create_prompt( + [ + Tool(name="foo", description="Test tool FOO", func=lambda x: x), + Tool(name="bar", description="Test tool BAR", func=lambda x: x), + ] + ) + + actual = prompt.messages[0].prompt.format() + + expected = dedent( + """ + Respond to the human as helpfully and accurately as possible. You have access to the following tools: + + foo: Test tool FOO, args: {'tool_input': {'type': 'string'}} + bar: Test tool BAR, args: {'tool_input': {'type': 'string'}} + + Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input). + + Valid "action" values: "Final Answer" or foo, bar + + Provide only ONE action per $JSON_BLOB, as shown: + + ``` + { + "action": $TOOL_NAME, + "action_input": $INPUT + } + ``` + + Follow this format: + + Question: input question to answer + Thought: consider previous and subsequent steps + Action: + ``` + $JSON_BLOB + ``` + Observation: action result + ... (repeat Thought/Action/Observation N times) + Thought: I know what to respond + Action: + ``` + { + "action": "Final Answer", + "action_input": "Final response to human" + } + ``` + + Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:. + Thought: + """ # noqa: E501 + ).strip() + + assert actual == expected