github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-10-09 16:08:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

RFC: add cache override to LLM class (#379)

Browse Source
This commit is contained in:
Harrison Chase
2022-12-19 17:36:14 -05:00
committed by GitHub
parent 46c428234f
commit 6be5747466
7 changed files with 522 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
docs/examples/prompts/llm_functionality.ipynb
View File

@@ -198,194 +198,11 @@
"source": [
"llm.get_num_tokens(\"what a joke\")"
]
},
{
"cell_type": "markdown",
"id": "ee6fcf8d",
"metadata": {},
"source": [
"### Caching\n",
"With LangChain, you can also enable caching of LLM calls. Note that currently this only applies for individual LLM calls."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "2626ca48",
"metadata": {},
"outputs": [],
"source": [
"import langchain\n",
"from langchain.cache import InMemoryCache\n",
"langchain.llm_cache = InMemoryCache()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "97762272",
"metadata": {},
"outputs": [],
"source": [
"# To make the caching really obvious, lets use a slower model.\n",
"llm = OpenAI(model_name=\"text-davinci-002\", n=2, best_of=2)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "e80c65e4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 31.2 ms, sys: 11.8 ms, total: 43.1 ms\n",
"Wall time: 1.75 s\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side!'"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The first time, it is not yet in cache, so it should take longer\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "678408ec",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 51 µs, sys: 1 µs, total: 52 µs\n",
"Wall time: 67.2 µs\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side!'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The second time it is, so it goes faster\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "3f0ac8d2",
"metadata": {},
"outputs": [],
"source": [
"# We can do the same thing with a SQLite cache\n",
"from langchain.cache import SQLiteCache\n",
"langchain.llm_cache = SQLiteCache(database_path=\".langchain.db\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "0e1dcce3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 26.6 ms, sys: 11.2 ms, total: 37.7 ms\n",
"Wall time: 1.89 s\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The first time, it is not yet in cache, so it should take longer\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "efadd750",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 2.69 ms, sys: 1.57 ms, total: 4.27 ms\n",
"Wall time: 2.73 ms\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The second time it is, so it goes faster\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6053408b",
"metadata": {},
"outputs": [],
"source": [
"# You can use SQLAlchemyCache to cache with any SQL database supported by SQLAlchemy.\n",
"from langchain.cache import SQLAlchemyCache\n",
"from sqlalchemy import create_engine\n",
"\n",
"engine = create_engine(\"postgresql://postgres:postgres@localhost:5432/postgres\")\n",
"langchain.llm_cache = SQLAlchemyCache(engine)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -399,7 +216,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.12 (main, Jun 1 2022, 06:34:44) \n[Clang 12.0.0 ]"
"version": "3.10.8"
},
"vscode": {
"interpreter": {