From 27601a13c443d81951d8e77034462e0577d07940 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Sun, 11 Dec 2022 00:20:37 -0800 Subject: [PATCH] return agent intermediate steps --- docs/getting_started/agents.ipynb | 122 ++++++++++++++++++++++++------ langchain/agents/agent.py | 21 +++-- langchain/agents/input.py | 20 ++++- langchain/chains/base.py | 2 +- 4 files changed, 134 insertions(+), 31 deletions(-) diff --git a/docs/getting_started/agents.ipynb b/docs/getting_started/agents.ipynb index d60030d02cb..bad6a576d17 100644 --- a/docs/getting_started/agents.ipynb +++ b/docs/getting_started/agents.ipynb @@ -118,40 +118,40 @@ "name": "stdout", "output_type": "stream", "text": [ - "What is the age of Olivia Wilde's boyfriend raised to the 0.23 power?\n", - "Thought:\u001b[32;1m\u001b[1;3m I need to find the age of Olivia Wilde's boyfriend\n", - "Action: Search\n", - "Action Input: \"Olivia Wilde's boyfriend\"\u001b[0m\n", - "Observation: \u001b[36;1m\u001b[1;3mOlivia Wilde started dating Harry Styles after ending her years-long engagement to Jason Sudeikis — see their relationship timeline.\u001b[0m\n", - "Thought:\u001b[32;1m\u001b[1;3m I need to find the age of Harry Styles\n", - "Action: Search\n", - "Action Input: \"Harry Styles age\"\u001b[0m\n", - "Observation: \u001b[36;1m\u001b[1;3m28 years\u001b[0m\n", - "Thought:\u001b[32;1m\u001b[1;3m I need to calculate 28 to the 0.23 power\n", - "Action: Calculator\n", - "Action Input: 28^0.23\u001b[0m\n", "\n", - "\u001b[1m> Entering new chain...\u001b[0m\n", - "28^0.23\u001b[32;1m\u001b[1;3m\n", + "\n", + "\u001b[1m> Entering new ZeroShotAgent chain...\u001b[0m\n", + "How old is Olivia Wilde's boyfriend? What is that number raised to the 0.23 power?\n", + "Thought:\u001b[32;1m\u001b[1;3m I need to find out how old Olivia Wilde's boyfriend is, and then use a calculator to calculate the power.\n", + "Action: Search\n", + "Action Input: Olivia Wilde's boyfriend age\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3mWhile Wilde, 37, and Styles, 27, have both kept a low profile when it comes to talking about their relationship, Wilde did address their ...\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m Olivia Wilde's boyfriend is 27 years old.\n", + "Action: Calculator\n", + "Action Input: 27^0.23\u001b[0m\n", + "\n", + "\u001b[1m> Entering new LLMMathChain chain...\u001b[0m\n", + "27^0.23\u001b[32;1m\u001b[1;3m\n", "\n", "```python\n", - "print(28**0.23)\n", + "import math\n", + "print(math.pow(27, 0.23))\n", "```\n", "\u001b[0m\n", - "Answer: \u001b[33;1m\u001b[1;3m2.1520202182226886\n", + "Answer: \u001b[33;1m\u001b[1;3m2.1340945944237553\n", "\u001b[0m\n", - "\u001b[1m> Finished chain.\u001b[0m\n", + "\u001b[1m> Finished LLMMathChain chain.\u001b[0m\n", "\n", - "Observation: \u001b[33;1m\u001b[1;3mAnswer: 2.1520202182226886\n", + "Observation: \u001b[33;1m\u001b[1;3mAnswer: 2.1340945944237553\n", "\u001b[0m\n", - "Thought:\u001b[32;1m\u001b[1;3m I now know the final answer\n", - "Final Answer: 2.1520202182226886\u001b[0m" + "Thought:\n", + "\u001b[1m> Finished ZeroShotAgent chain.\u001b[0m\n" ] }, { "data": { "text/plain": [ - "'2.1520202182226886'" + "'2.1340945944237553'" ] }, "execution_count": 4, @@ -165,10 +165,86 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "2f0852ff", "metadata": {}, "outputs": [], + "source": [ + "# We can also return the intermediate steps\n", + "llm = OpenAI(temperature=0)\n", + "agent = initialize_agent(tools, llm, agent=\"zero-shot-react-description\", verbose=True, return_intermediate_steps=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "837211e8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new ZeroShotAgent chain...\u001b[0m\n", + "How old is Olivia Wilde's boyfriend? What is that number raised to the 0.23 power?\n", + "Thought:\u001b[32;1m\u001b[1;3m I need to find out how old Olivia Wilde's boyfriend is, and then use a calculator to calculate the power.\n", + "Action: Search\n", + "Action Input: Olivia Wilde's boyfriend age\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3mWhile Wilde, 37, and Styles, 27, have both kept a low profile when it comes to talking about their relationship, Wilde did address their ...\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m Olivia Wilde's boyfriend is 27 years old.\n", + "Action: Calculator\n", + "Action Input: 27^0.23\u001b[0m\n", + "\n", + "\u001b[1m> Entering new LLMMathChain chain...\u001b[0m\n", + "27^0.23\u001b[32;1m\u001b[1;3m\n", + "\n", + "```python\n", + "import math\n", + "print(math.pow(27, 0.23))\n", + "```\n", + "\u001b[0m\n", + "Answer: \u001b[33;1m\u001b[1;3m2.1340945944237553\n", + "\u001b[0m\n", + "\u001b[1m> Finished LLMMathChain chain.\u001b[0m\n", + "\n", + "Observation: \u001b[33;1m\u001b[1;3mAnswer: 2.1340945944237553\n", + "\u001b[0m\n", + "Thought:\n", + "\u001b[1m> Finished ZeroShotAgent chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "{'input': \"How old is Olivia Wilde's boyfriend? What is that number raised to the 0.23 power?\",\n", + " 'output': '2.1340945944237553',\n", + " 'intermediate_steps': [{'log': \" I need to find out how old Olivia Wilde's boyfriend is, and then use a calculator to calculate the power.\\nAction: Search\\nAction Input: Olivia Wilde's boyfriend age\",\n", + " 'tool': 'Search',\n", + " 'tool_input': \"Olivia Wilde's boyfriend age\",\n", + " 'observation': 'While Wilde, 37, and Styles, 27, have both kept a low profile when it comes to talking about their relationship, Wilde did address their ...'},\n", + " {'log': \" Olivia Wilde's boyfriend is 27 years old.\\nAction: Calculator\\nAction Input: 27^0.23\",\n", + " 'tool': 'Calculator',\n", + " 'tool_input': '27^0.23',\n", + " 'observation': 'Answer: 2.1340945944237553\\n'}]}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agent({\"input\":\"How old is Olivia Wilde's boyfriend? What is that number raised to the 0.23 power?\"})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9256ff6b", + "metadata": {}, + "outputs": [], "source": [] } ], @@ -188,7 +264,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.10.8" } }, "nbformat": 4, diff --git a/langchain/agents/agent.py b/langchain/agents/agent.py index fb60115ce53..1234d388859 100644 --- a/langchain/agents/agent.py +++ b/langchain/agents/agent.py @@ -22,6 +22,7 @@ class Agent(Chain, BaseModel, ABC): prompt: ClassVar[BasePromptTemplate] llm_chain: LLMChain tools: List[Tool] + return_intermediate_steps: bool = False input_key: str = "input" #: :meta private: output_key: str = "output" #: :meta private: @@ -39,7 +40,10 @@ class Agent(Chain, BaseModel, ABC): :meta private: """ - return [self.output_key] + if self.return_intermediate_steps: + return [self.output_key, "intermediate_steps"] + else: + return [self.output_key] @property @abstractmethod @@ -115,7 +119,7 @@ class Agent(Chain, BaseModel, ABC): tool, tool_input = parsed_output return AgentAction(tool, tool_input, full_output) - def _call(self, inputs: Dict[str, str]) -> Dict[str, str]: + def _call(self, inputs: Dict[str, str]) -> Dict[str, Any]: """Run text through and get agent response.""" text = inputs[self.input_key] # Do any preparation necessary when receiving a new input. @@ -138,12 +142,17 @@ class Agent(Chain, BaseModel, ABC): while True: # Call the LLM to see what to do. output = self.get_action(chained_input.input) - # Add the log to the Chained Input. - chained_input.add_action(output, color="green") # If the tool chosen is the finishing tool, then we end and return. if output.tool == self.finish_tool_name: - return {self.output_key: output.tool_input} - # Otherwise we lookup the tool + final_output: dict = {self.output_key: output.tool_input} + if self.return_intermediate_steps: + final_output[ + "intermediate_steps" + ] = chained_input.intermediate_steps + return final_output + # Other we add the log to the Chained Input. + chained_input.add_action(output, color="green") + # And then we lookup the tool if output.tool in name_to_tool_map: chain = name_to_tool_map[output.tool] # We then call the tool on the tool input to get an observation diff --git a/langchain/agents/input.py b/langchain/agents/input.py index 6659ad091c0..5e137882e73 100644 --- a/langchain/agents/input.py +++ b/langchain/agents/input.py @@ -1,5 +1,5 @@ """Input manager for agents.""" -from typing import Optional +from typing import List, Optional import langchain from langchain.schema import AgentAction @@ -14,12 +14,29 @@ class ChainedInput: if self._verbose: langchain.logger.log_agent_start(text) self._input = text + self._intermediate_actions: List[AgentAction] = [] + self._intermediate_observations: List[str] = [] + + @property + def intermediate_steps(self) -> List: + """Return intermediate steps the agent took.""" + steps = [] + for i, action in enumerate(self._intermediate_actions): + step = { + "log": action.log, + "tool": action.tool, + "tool_input": action.tool_input, + "observation": self._intermediate_observations[i], + } + steps.append(step) + return steps def add_action(self, action: AgentAction, color: Optional[str] = None) -> None: """Add text to input, print if in verbose mode.""" if self._verbose: langchain.logger.log_agent_action(action, color=color) self._input += action.log + self._intermediate_actions.append(action) def add_observation( self, @@ -37,6 +54,7 @@ class ChainedInput: llm_prefix=llm_prefix, ) self._input += f"\n{observation_prefix}{observation}\n{llm_prefix}" + self._intermediate_observations.append(observation) @property def input(self) -> str: diff --git a/langchain/chains/base.py b/langchain/chains/base.py index 44f0bdbf95b..f623c3ded3d 100644 --- a/langchain/chains/base.py +++ b/langchain/chains/base.py @@ -71,7 +71,7 @@ class Chain(BaseModel, ABC): def __call__( self, inputs: Dict[str, Any], return_only_outputs: bool = False - ) -> Dict[str, str]: + ) -> Dict[str, Any]: """Run the logic of this chain and add to output if desired. Args: