Harrison/gpt cache (#2744)

Co-authored-by: SimFG <bang.fu@zilliz.com>

docs/modules/models/llms/examples/llm_caching.ipynb

@@ -60,14 +60,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 30.7 ms, sys: 18.6 ms, total: 49.3 ms\n",
-      "Wall time: 791 ms\n"
+      "CPU times: user 14.2 ms, sys: 4.9 ms, total: 19.1 ms\n",
+      "Wall time: 1.1 s\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "\"\\n\\nWhy couldn't the bicycle stand up by itself? Because it was...two tired!\""
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
       ]
      },
      "execution_count": 4,
@@ -91,14 +91,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 80 µs, sys: 0 ns, total: 80 µs\n",
-      "Wall time: 83.9 µs\n"
+      "CPU times: user 162 µs, sys: 7 µs, total: 169 µs\n",
+      "Wall time: 175 µs\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "\"\\n\\nWhy couldn't the bicycle stand up by itself? Because it was...two tired!\""
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
       ]
      },
      "execution_count": 5,
@@ -252,6 +252,249 @@
     "llm(\"Tell me a joke\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "684eab55",
+   "metadata": {},
+   "source": [
+    "## GPTCache\n",
+    "\n",
+    "We can use [GPTCache](https://github.com/zilliztech/GPTCache) for exact match caching OR to cache results based on semantic similarity\n",
+    "\n",
+    "Let's first start with an example of exact match"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "14a82124",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gptcache\n",
+    "from gptcache.processor.pre import get_prompt\n",
+    "from gptcache.manager.factory import get_data_manager\n",
+    "from langchain.cache import GPTCache\n",
+    "\n",
+    "# Avoid multiple caches using the same file, causing different llm model caches to affect each other\n",
+    "i = 0\n",
+    "file_prefix = \"data_map\"\n",
+    "\n",
+    "def init_gptcache_map(cache_obj: gptcache.Cache):\n",
+    "    global i\n",
+    "    cache_path = f'{file_prefix}_{i}.txt'\n",
+    "    cache_obj.init(\n",
+    "        pre_embedding_func=get_prompt,\n",
+    "        data_manager=get_data_manager(data_path=cache_path),\n",
+    "    )\n",
+    "    i += 1\n",
+    "\n",
+    "langchain.llm_cache = GPTCache(init_gptcache_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "9e4ecfd1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 8.6 ms, sys: 3.82 ms, total: 12.4 ms\n",
+      "Wall time: 881 ms\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "# The first time, it is not yet in cache, so it should take longer\n",
+    "llm(\"Tell me a joke\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "c98bbe3b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 286 µs, sys: 21 µs, total: 307 µs\n",
+      "Wall time: 316 µs\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "# The second time it is, so it goes faster\n",
+    "llm(\"Tell me a joke\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "502b6076",
+   "metadata": {},
+   "source": [
+    "Let's now show an example of similarity caching"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "b3c663bb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gptcache\n",
+    "from gptcache.processor.pre import get_prompt\n",
+    "from gptcache.manager.factory import get_data_manager\n",
+    "from langchain.cache import GPTCache\n",
+    "from gptcache.manager import get_data_manager, CacheBase, VectorBase\n",
+    "from gptcache import Cache\n",
+    "from gptcache.embedding import Onnx\n",
+    "from gptcache.similarity_evaluation.distance import SearchDistanceEvaluation\n",
+    "\n",
+    "# Avoid multiple caches using the same file, causing different llm model caches to affect each other\n",
+    "i = 0\n",
+    "file_prefix = \"data_map\"\n",
+    "llm_cache = Cache()\n",
+    "\n",
+    "\n",
+    "def init_gptcache_map(cache_obj: gptcache.Cache):\n",
+    "    global i\n",
+    "    cache_path = f'{file_prefix}_{i}.txt'\n",
+    "    onnx = Onnx()\n",
+    "    cache_base = CacheBase('sqlite')\n",
+    "    vector_base = VectorBase('faiss', dimension=onnx.dimension)\n",
+    "    data_manager = get_data_manager(cache_base, vector_base, max_size=10, clean_size=2)\n",
+    "    cache_obj.init(\n",
+    "        pre_embedding_func=get_prompt,\n",
+    "        embedding_func=onnx.to_embeddings,\n",
+    "        data_manager=data_manager,\n",
+    "        similarity_evaluation=SearchDistanceEvaluation(),\n",
+    "    )\n",
+    "    i += 1\n",
+    "\n",
+    "langchain.llm_cache = GPTCache(init_gptcache_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8c273ced",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1.01 s, sys: 153 ms, total: 1.16 s\n",
+      "Wall time: 2.49 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "# The first time, it is not yet in cache, so it should take longer\n",
+    "llm(\"Tell me a joke\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "93e21a5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 745 ms, sys: 13.2 ms, total: 758 ms\n",
+      "Wall time: 136 ms\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "# This is an exact match, so it finds it in the cache\n",
+    "llm(\"Tell me a joke\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "c4bb024b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 737 ms, sys: 7.79 ms, total: 745 ms\n",
+      "Wall time: 135 ms\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "# This is not an exact match, but semantically within distance so it hits!\n",
+    "llm(\"Tell me joke\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "934943dc",