From 9f9afbb6a8bde114b7d7a02c641cfd4056184372 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Fri, 13 Jan 2023 06:28:51 -0800 Subject: [PATCH] add custom prompt for LLMMathChain and SQLDatabase chain (#605) --- docs/modules/chains/examples/llm_math.ipynb | 103 +++++++++++++++++++- docs/modules/chains/examples/sqlite.ipynb | 87 ++++++++++++++++- langchain/chains/llm_math/base.py | 5 +- langchain/chains/sql_database/base.py | 5 +- 4 files changed, 192 insertions(+), 8 deletions(-) diff --git a/docs/modules/chains/examples/llm_math.ipynb b/docs/modules/chains/examples/llm_math.ipynb index d906d0e9c0e..29eaaea1c2f 100644 --- a/docs/modules/chains/examples/llm_math.ipynb +++ b/docs/modules/chains/examples/llm_math.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 1, "id": "44e9ba31", "metadata": {}, "outputs": [ @@ -31,7 +31,7 @@ "\u001b[0m\n", "Answer: \u001b[33;1m\u001b[1;3m2.4116004626599237\n", "\u001b[0m\n", - "\u001b[1m> Finished LLMMathChain chain.\u001b[0m\n" + "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { @@ -40,7 +40,7 @@ "'Answer: 2.4116004626599237\\n'" ] }, - "execution_count": 5, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -54,10 +54,105 @@ "llm_math.run(\"What is 13 raised to the .3432 power?\")" ] }, + { + "cell_type": "markdown", + "id": "2bdd5fc6", + "metadata": {}, + "source": [ + "## Customize Prompt\n", + "You can also customize the prompt that is used. Here is an example prompting it to use numpy" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "76be17b0", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.prompt import PromptTemplate\n", + "\n", + "_PROMPT_TEMPLATE = \"\"\"You are GPT-3, and you can't do math.\n", + "\n", + "You can do basic math, and your memorization abilities are impressive, but you can't do any complex calculations that a human could not do in their head. You also have an annoying tendency to just make up highly specific, but wrong, answers.\n", + "\n", + "So we hooked you up to a Python 3 kernel, and now you can execute code. If you execute code, you must print out the final answer using the print function. You MUST use the python package numpy to answer your question. You must import numpy as np.\n", + "\n", + "\n", + "Question: ${{Question with hard calculation.}}\n", + "```python\n", + "${{Code that prints what you need to know}}\n", + "print(${{code}})\n", + "```\n", + "```output\n", + "${{Output of your code}}\n", + "```\n", + "Answer: ${{Answer}}\n", + "\n", + "Begin.\n", + "\n", + "Question: What is 37593 * 67?\n", + "\n", + "```python\n", + "import numpy as np\n", + "print(np.multiply(37593, 67))\n", + "```\n", + "```output\n", + "2518731\n", + "```\n", + "Answer: 2518731\n", + "\n", + "Question: {question}\"\"\"\n", + "\n", + "PROMPT = PromptTemplate(input_variables=[\"question\"], template=_PROMPT_TEMPLATE)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "0c42faa0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new LLMMathChain chain...\u001b[0m\n", + "What is 13 raised to the .3432 power?\u001b[32;1m\u001b[1;3m\n", + "\n", + "```python\n", + "import numpy as np\n", + "print(np.power(13, .3432))\n", + "```\n", + "\u001b[0m\n", + "Answer: \u001b[33;1m\u001b[1;3m2.4116004626599237\n", + "\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "'Answer: 2.4116004626599237\\n'" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "llm_math = LLMMathChain(llm=llm, prompt=PROMPT, verbose=True)\n", + "\n", + "llm_math.run(\"What is 13 raised to the .3432 power?\")" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "f62f0c75", + "id": "0c62951b", "metadata": {}, "outputs": [], "source": [] diff --git a/docs/modules/chains/examples/sqlite.ipynb b/docs/modules/chains/examples/sqlite.ipynb index 4a2d266d2c5..66a4ba8ea19 100644 --- a/docs/modules/chains/examples/sqlite.ipynb +++ b/docs/modules/chains/examples/sqlite.ipynb @@ -78,7 +78,7 @@ "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT COUNT(*) FROM Employee;\u001b[0m\n", "SQLResult: \u001b[33;1m\u001b[1;3m[(9,)]\u001b[0m\n", "Answer:\u001b[32;1m\u001b[1;3m There are 9 employees.\u001b[0m\n", - "\u001b[1m> Finished SQLDatabaseChain chain.\u001b[0m\n" + "\u001b[1m> Finished chain.\u001b[0m\n" ] }, { @@ -96,10 +96,93 @@ "db_chain.run(\"How many employees are there?\")" ] }, + { + "cell_type": "markdown", + "id": "aad2cba6", + "metadata": {}, + "source": [ + "## Customize Prompt\n", + "You can also customize the prompt that is used. Here is an example prompting it to understand that foobar is the same as the Employee table" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8ca7bafb", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.prompt import PromptTemplate\n", + "\n", + "_DEFAULT_TEMPLATE = \"\"\"Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer.\n", + "Use the following format:\n", + "\n", + "Question: \"Question here\"\n", + "SQLQuery: \"SQL Query to run\"\n", + "SQLResult: \"Result of the SQLQuery\"\n", + "Answer: \"Final answer here\"\n", + "\n", + "Only use the following tables:\n", + "\n", + "{table_info}\n", + "\n", + "If someone asks for the table foobar, they really mean the employee table.\n", + "\n", + "Question: {input}\"\"\"\n", + "PROMPT = PromptTemplate(\n", + " input_variables=[\"input\", \"table_info\", \"dialect\"], template=_DEFAULT_TEMPLATE\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ec47a2bf", + "metadata": {}, + "outputs": [], + "source": [ + "db_chain = SQLDatabaseChain(llm=llm, database=db, prompt=PROMPT, verbose=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ebb0674e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new SQLDatabaseChain chain...\u001b[0m\n", + "How many employees are there in the foobar table? \n", + "SQLQuery:\u001b[32;1m\u001b[1;3m SELECT COUNT(*) FROM Employee;\u001b[0m\n", + "SQLResult: \u001b[33;1m\u001b[1;3m[(9,)]\u001b[0m\n", + "Answer:\u001b[32;1m\u001b[1;3m There are 9 employees in the foobar table.\u001b[0m\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "' There are 9 employees in the foobar table.'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db_chain.run(\"How many employees are there in the foobar table?\")" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "61d91b85", + "id": "e59a4740", "metadata": {}, "outputs": [], "source": [] diff --git a/langchain/chains/llm_math/base.py b/langchain/chains/llm_math/base.py index 383bcac96f3..1b6628f4616 100644 --- a/langchain/chains/llm_math/base.py +++ b/langchain/chains/llm_math/base.py @@ -7,6 +7,7 @@ from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.llm_math.prompt import PROMPT from langchain.llms.base import BaseLLM +from langchain.prompts.base import BasePromptTemplate from langchain.python import PythonREPL @@ -22,6 +23,8 @@ class LLMMathChain(Chain, BaseModel): llm: BaseLLM """LLM wrapper to use.""" + prompt: BasePromptTemplate = PROMPT + """Prompt to use to translate to python if neccessary.""" input_key: str = "question" #: :meta private: output_key: str = "answer" #: :meta private: @@ -48,7 +51,7 @@ class LLMMathChain(Chain, BaseModel): return [self.output_key] def _call(self, inputs: Dict[str, str]) -> Dict[str, str]: - llm_executor = LLMChain(prompt=PROMPT, llm=self.llm) + llm_executor = LLMChain(prompt=self.prompt, llm=self.llm) python_executor = PythonREPL() if self.verbose: self.callback_manager.on_text(inputs[self.input_key]) diff --git a/langchain/chains/sql_database/base.py b/langchain/chains/sql_database/base.py index 3c56cac7d69..9fbc0ab2f1c 100644 --- a/langchain/chains/sql_database/base.py +++ b/langchain/chains/sql_database/base.py @@ -7,6 +7,7 @@ from langchain.chains.base import Chain from langchain.chains.llm import LLMChain from langchain.chains.sql_database.prompt import PROMPT from langchain.llms.base import BaseLLM +from langchain.prompts.base import BasePromptTemplate from langchain.sql_database import SQLDatabase @@ -25,6 +26,8 @@ class SQLDatabaseChain(Chain, BaseModel): """LLM wrapper to use.""" database: SQLDatabase """SQL Database to connect to.""" + prompt: BasePromptTemplate = PROMPT + """Prompt to use to translate natural language to SQL.""" input_key: str = "query" #: :meta private: output_key: str = "result" #: :meta private: @@ -51,7 +54,7 @@ class SQLDatabaseChain(Chain, BaseModel): return [self.output_key] def _call(self, inputs: Dict[str, str]) -> Dict[str, str]: - llm_chain = LLMChain(llm=self.llm, prompt=PROMPT) + llm_chain = LLMChain(llm=self.llm, prompt=self.prompt) input_text = f"{inputs[self.input_key]} \nSQLQuery:" if self.verbose: self.callback_manager.on_text(input_text)