From 5c0c5fafb25461c0efbddbacbfa86fcaad0b150f Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Sun, 9 Apr 2023 12:29:16 -0700 Subject: [PATCH] Multi-Hop / Multi-Spec LLM Chain (#2549) Add a notebook showing how to make a chain that composes multiple OpenAPI Endpoint operations to accomplish tasks. --- .../chains/examples/multihop_openapi.ipynb | 292 ++++++++++++++++++ langchain/agents/agent_toolkits/__init__.py | 2 + .../agents/agent_toolkits/nla/__init__.py | 0 langchain/agents/agent_toolkits/nla/tool.py | 54 ++++ .../agents/agent_toolkits/nla/toolkit.py | 66 ++++ langchain/chains/api/openapi/chain.py | 9 +- langchain/chains/api/openapi/prompts.py | 2 +- langchain/tools/openapi/utils/api_models.py | 5 +- 8 files changed, 427 insertions(+), 3 deletions(-) create mode 100644 docs/modules/chains/examples/multihop_openapi.ipynb create mode 100644 langchain/agents/agent_toolkits/nla/__init__.py create mode 100644 langchain/agents/agent_toolkits/nla/tool.py create mode 100644 langchain/agents/agent_toolkits/nla/toolkit.py diff --git a/docs/modules/chains/examples/multihop_openapi.ipynb b/docs/modules/chains/examples/multihop_openapi.ipynb new file mode 100644 index 00000000000..ede030d55bf --- /dev/null +++ b/docs/modules/chains/examples/multihop_openapi.ipynb @@ -0,0 +1,292 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c7ad998d", + "metadata": {}, + "source": [ + "# Multi-hop Task Execution with the NLAToolkit\n", + "\n", + "Natural Language API Toolkits (NLAToolkits) permit LangChain Agents to efficiently plan and combine calls across endpoints. This notebook demonstrates a sample composition of the Speak, Klarna, and Spoonacluar APIs.\n", + "\n", + "For a detailed walkthrough of the OpenAPI chains wrapped within the NLAToolkit, see the [OpenAPI Operation Chain](openapi.ipynb) notebook.\n", + "\n", + "### First, import dependencies and load the LLM" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6593f793", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from typing import List, Optional\n", + "from langchain.chains import LLMChain\n", + "from langchain.llms import OpenAI\n", + "from langchain.prompts import PromptTemplate\n", + "from langchain.requests import Requests\n", + "from langchain.tools import APIOperation, OpenAPISpec\n", + "from langchain.agents import AgentType, Tool, initialize_agent\n", + "from langchain.agents.agent_toolkits import NLAToolkit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd720860", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Select the LLM to use. Here, we use text-davinci-003\n", + "llm = OpenAI(temperature=0, max_tokens=700) # You can swap between different core LLM's here." + ] + }, + { + "cell_type": "markdown", + "id": "4cadac9d", + "metadata": { + "tags": [] + }, + "source": [ + "### Next, load the Natural Language API Toolkits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b208ab0", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "speak_toolkit = NLAToolkit.from_llm_and_url(llm, \"https://api.speak.com/openapi.yaml\")\n", + "klarna_toolkit = NLAToolkit.from_llm_and_url(llm, \"https://www.klarna.com/us/shopping/public/openai/v0/api-docs/\")" + ] + }, + { + "cell_type": "markdown", + "id": "16c7336f", + "metadata": {}, + "source": [ + "### Create the Agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "730a0dc2-b4d0-46d5-a1e9-583803220973", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Slightly tweak the instructions from the default agent\n", + "openapi_format_instructions = \"\"\"Use the following format:\n", + "\n", + "Question: the input question you must answer\n", + "Thought: you should always think about what to do\n", + "Action: the action to take, should be one of [{tool_names}]\n", + "Action Input: what to instruct the AI Action representative.\n", + "Observation: The Agent's response\n", + "... (this Thought/Action/Action Input/Observation can repeat N times)\n", + "Thought: I now know the final answer. User can't see any of my observations, API responses, links, or tools.\n", + "Final Answer: the final answer to the original input question with the right amount of detail\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40a979c3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "natural_language_tools = speak_toolkit.get_tools() + klarna_toolkit.get_tools()\n", + "mrkl = initialize_agent(natural_language_tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, \n", + " verbose=True, agent_kwargs={\"format_instructions\":openapi_format_instructions})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "794380ba", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "mrkl.run(\"I have an end of year party for my Italian class and have to buy some Italian clothes for it\")" + ] + }, + { + "cell_type": "markdown", + "id": "c61d92a8", + "metadata": {}, + "source": [ + "### Using Auth + Adding more Endpoints\n", + "\n", + "Some endpoints may require user authentication via things like access tokens. Here we show how to pass in the authentication information via the `Requests` wrapper object.\n", + "\n", + "Since each NLATool exposes a concisee natural language interface to its wrapped API, the top level conversational agent has an easier job incorporating each endpoint to satisfy a user's request." + ] + }, + { + "cell_type": "markdown", + "id": "f0d132cc", + "metadata": {}, + "source": [ + "**Adding the Spoonacular endpoints.**\n", + "\n", + "1. Go to the [Spoonacular API Console](https://spoonacular.com/food-api/console#Profile) and make a free account.\n", + "2. Click on `Profile` and copy your API key below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2368b9c", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "spoonacular_api_key = \"\" # Copy from the API Console" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbd97c28-fef6-41b5-9600-a9611a32bfb3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "requests = Requests(headers={\"x-api-key\": spoonacular_api_key})\n", + "spoonacular_toolkit = NLAToolkit.from_llm_and_url(\n", + " llm, \n", + " \"https://spoonacular.com/application/frontend/downloads/spoonacular-openapi-3.json\",\n", + " requests=requests,\n", + " max_text_length=1800, # If you want to truncate the response text\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81a6edac", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "natural_language_api_tools = (speak_toolkit.get_tools() \n", + " + klarna_toolkit.get_tools() \n", + " + spoonacular_toolkit.get_tools()[:30]\n", + " )\n", + "print(f\"{len(natural_language_api_tools)} tools loaded.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "831f772d-5cd1-4467-b494-a3172af2ff48", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create an agent with the new tools\n", + "mrkl = initialize_agent(natural_language_api_tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, \n", + " verbose=True, agent_kwargs={\"format_instructions\":openapi_format_instructions})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0385e04b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Make the query more complex!\n", + "user_input = (\n", + " \"I'm learning Italian, and my language class is having an end of year party... \"\n", + " \" Could you help me find an Italian outfit to wear and\"\n", + " \" an appropriate recipe to prepare so I can present for the class in Italian?\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ebd3f55", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "mrkl.run(user_input)" + ] + }, + { + "cell_type": "markdown", + "id": "a2959462", + "metadata": {}, + "source": [ + "## Thank you!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fcda5f0", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "natural_language_api_tools[1].run(\"Tell the LangChain audience to 'enjoy the meal' in Italian, please!\")['output']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab366dc0", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/langchain/agents/agent_toolkits/__init__.py b/langchain/agents/agent_toolkits/__init__.py index 0b65e01d24d..d944a6b2e06 100644 --- a/langchain/agents/agent_toolkits/__init__.py +++ b/langchain/agents/agent_toolkits/__init__.py @@ -3,6 +3,7 @@ from langchain.agents.agent_toolkits.csv.base import create_csv_agent from langchain.agents.agent_toolkits.json.base import create_json_agent from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit +from langchain.agents.agent_toolkits.nla.toolkit import NLAToolkit from langchain.agents.agent_toolkits.openapi.base import create_openapi_agent from langchain.agents.agent_toolkits.openapi.toolkit import OpenAPIToolkit from langchain.agents.agent_toolkits.pandas.base import create_pandas_dataframe_agent @@ -28,6 +29,7 @@ __all__ = [ "create_vectorstore_agent", "JsonToolkit", "SQLDatabaseToolkit", + "NLAToolkit", "OpenAPIToolkit", "VectorStoreToolkit", "create_vectorstore_router_agent", diff --git a/langchain/agents/agent_toolkits/nla/__init__.py b/langchain/agents/agent_toolkits/nla/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/langchain/agents/agent_toolkits/nla/tool.py b/langchain/agents/agent_toolkits/nla/tool.py new file mode 100644 index 00000000000..d4d7e96a04a --- /dev/null +++ b/langchain/agents/agent_toolkits/nla/tool.py @@ -0,0 +1,54 @@ +"""Tool for interacting with a single API with natural language efinition.""" + + +from typing import Any, Optional + +from langchain.agents.tools import Tool +from langchain.chains.api.openapi.chain import OpenAPIEndpointChain +from langchain.llms.base import BaseLLM +from langchain.requests import Requests +from langchain.tools.openapi.utils.api_models import APIOperation +from langchain.tools.openapi.utils.openapi_utils import OpenAPISpec + + +class NLATool(Tool): + """Natural Language API Tool.""" + + @classmethod + def from_open_api_endpoint_chain( + cls, chain: OpenAPIEndpointChain, api_title: str + ) -> "NLATool": + """Convert an endpoint chain to an API endpoint tool.""" + expanded_name = ( + f'{api_title.replace(" ", "_")}.{chain.api_operation.operation_id}' + ) + description = ( + f"I'm an AI from {api_title}. Instruct what you want," + " and I'll assist via an API with description:" + f" {chain.api_operation.description}" + ) + return cls(name=expanded_name, func=chain.run, description=description) + + @classmethod + def from_llm_and_method( + cls, + llm: BaseLLM, + path: str, + method: str, + spec: OpenAPISpec, + requests: Optional[Requests] = None, + verbose: bool = False, + return_intermediate_steps: bool = False, + **kwargs: Any, + ) -> "NLATool": + """Instantiate the tool from the specified path and method.""" + api_operation = APIOperation.from_openapi_spec(spec, path, method) + chain = OpenAPIEndpointChain.from_api_operation( + api_operation, + llm, + requests=requests, + verbose=verbose, + return_intermediate_steps=return_intermediate_steps, + **kwargs, + ) + return cls.from_open_api_endpoint_chain(chain, spec.info.title) diff --git a/langchain/agents/agent_toolkits/nla/toolkit.py b/langchain/agents/agent_toolkits/nla/toolkit.py new file mode 100644 index 00000000000..d429611ba43 --- /dev/null +++ b/langchain/agents/agent_toolkits/nla/toolkit.py @@ -0,0 +1,66 @@ +"""Toolkit for interacting with API's using natural language.""" + + +from typing import Any, List, Optional, Sequence + +from pydantic import Field + +from langchain.agents.agent_toolkits.base import BaseToolkit +from langchain.agents.agent_toolkits.nla.tool import NLATool +from langchain.llms.base import BaseLLM +from langchain.requests import Requests +from langchain.tools.base import BaseTool +from langchain.tools.openapi.utils.openapi_utils import OpenAPISpec + + +class NLAToolkit(BaseToolkit): + """Natural Language API Toolkit Definition.""" + + nla_tools: Sequence[NLATool] = Field(...) + """List of API Endpoint Tools.""" + + def get_tools(self) -> List[BaseTool]: + """Get the tools for all the API operations.""" + return list(self.nla_tools) + + @classmethod + def from_llm_and_spec( + cls, + llm: BaseLLM, + spec: OpenAPISpec, + requests: Optional[Requests] = None, + verbose: bool = False, + **kwargs: Any + ) -> "NLAToolkit": + """Instantiate the toolkit by creating tools for each operation.""" + http_operation_tools: List[NLATool] = [] + if not spec.paths: + return cls(nla_tools=http_operation_tools) + for path in spec.paths: + for method in spec.get_methods_for_path(path): + endpoint_tool = NLATool.from_llm_and_method( + llm=llm, + path=path, + method=method, + spec=spec, + requests=requests, + verbose=verbose, + **kwargs + ) + http_operation_tools.append(endpoint_tool) + return cls(nla_tools=http_operation_tools) + + @classmethod + def from_llm_and_url( + cls, + llm: BaseLLM, + open_api_url: str, + requests: Optional[Requests] = None, + verbose: bool = False, + **kwargs: Any + ) -> "NLAToolkit": + """Instantiate the toolkit from an OpenAPI Spec URL""" + spec = OpenAPISpec.from_url(open_api_url) + return cls.from_llm_and_spec( + llm=llm, spec=spec, requests=requests, verbose=verbose, **kwargs + ) diff --git a/langchain/chains/api/openapi/chain.py b/langchain/chains/api/openapi/chain.py index dbbdfd0376e..f9e01aa2d94 100644 --- a/langchain/chains/api/openapi/chain.py +++ b/langchain/chains/api/openapi/chain.py @@ -2,7 +2,7 @@ from __future__ import annotations import json -from typing import Dict, List, NamedTuple, Optional, cast +from typing import Any, Dict, List, NamedTuple, Optional, cast from pydantic import BaseModel, Field from requests import Response @@ -35,6 +35,7 @@ class OpenAPIEndpointChain(Chain, BaseModel): return_intermediate_steps: bool = False instructions_key: str = "instructions" #: :meta private: output_key: str = "output" #: :meta private: + max_text_length: Optional[int] = Field(ge=0) #: :meta private: @property def input_keys(self) -> List[str]: @@ -108,6 +109,7 @@ class OpenAPIEndpointChain(Chain, BaseModel): def _call(self, inputs: Dict[str, str]) -> Dict[str, str]: intermediate_steps = {} instructions = inputs[self.instructions_key] + instructions = instructions[: self.max_text_length] _api_arguments = self.api_request_chain.predict_and_parse( instructions=instructions ) @@ -137,6 +139,7 @@ class OpenAPIEndpointChain(Chain, BaseModel): response_text = api_response.text except Exception as e: response_text = f"Error with message {str(e)}" + response_text = response_text[: self.max_text_length] intermediate_steps["response_text"] = response_text self.callback_manager.on_text( response_text, color="blue", end="\n", verbose=self.verbose @@ -160,6 +163,7 @@ class OpenAPIEndpointChain(Chain, BaseModel): llm: BaseLLM, requests: Optional[Requests] = None, return_intermediate_steps: bool = False, + **kwargs: Any # TODO: Handle async ) -> "OpenAPIEndpointChain": """Create an OpenAPIEndpoint from a spec at the specified url.""" @@ -169,6 +173,7 @@ class OpenAPIEndpointChain(Chain, BaseModel): requests=requests, llm=llm, return_intermediate_steps=return_intermediate_steps, + **kwargs, ) @classmethod @@ -179,6 +184,7 @@ class OpenAPIEndpointChain(Chain, BaseModel): requests: Optional[Requests] = None, verbose: bool = False, return_intermediate_steps: bool = False, + **kwargs: Any # TODO: Handle async ) -> "OpenAPIEndpointChain": """Create an OpenAPIEndpointChain from an operation and a spec.""" @@ -200,4 +206,5 @@ class OpenAPIEndpointChain(Chain, BaseModel): param_mapping=param_mapping, verbose=verbose, return_intermediate_steps=return_intermediate_steps, + **kwargs, ) diff --git a/langchain/chains/api/openapi/prompts.py b/langchain/chains/api/openapi/prompts.py index 2eca5a1a672..0756ae36563 100644 --- a/langchain/chains/api/openapi/prompts.py +++ b/langchain/chains/api/openapi/prompts.py @@ -50,7 +50,7 @@ Response Error: ```json {{"response": "What you did and a concise statement of the resulting error. If it can be easily fixed, provide a suggestion."}} ``` -You MUST respond as a markdown json code block. +You MUST respond as a markdown json code block. API_RESPONSE and other information is not visible to the user. Begin: --- diff --git a/langchain/tools/openapi/utils/api_models.py b/langchain/tools/openapi/utils/api_models.py index 3df6f72fad5..cee38658123 100644 --- a/langchain/tools/openapi/utils/api_models.py +++ b/langchain/tools/openapi/utils/api_models.py @@ -475,9 +475,12 @@ class APIOperation(BaseModel): if request_body is not None else None ) + description = operation.description or operation.summary + if not description and spec.paths is not None: + description = spec.paths[path].description or spec.paths[path].summary return cls( operation_id=operation_id, - description=operation.description, + description=description, base_url=spec.base_url, path=path, method=method,