diff --git a/docs/extras/integrations/document_loaders/google_drive.ipynb b/docs/extras/integrations/document_loaders/google_drive.ipynb index e7cda8f0617..9d17e5df97a 100644 --- a/docs/extras/integrations/document_loaders/google_drive.ipynb +++ b/docs/extras/integrations/document_loaders/google_drive.ipynb @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "878928a6-a5ae-4f74-b351-64e3b01733fe", "metadata": { "tags": [] @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "2216c83f-68e4-4d2f-8ea2-5878fb18bbe7", "metadata": { "tags": [] @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "8f3b6aa0-b45d-4e37-8c50-5bebe70fdb9d", "metadata": { "tags": [] @@ -93,7 +93,7 @@ "source": [ "loader = GoogleDriveLoader(\n", " folder_id=\"1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5\",\n", - " file_types=[\"document\", \"sheet\"]\n", + " file_types=[\"document\", \"sheet\"],\n", " recursive=False\n", ")" ] @@ -110,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "94207e39", "metadata": {}, "outputs": [], @@ -121,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "a15fbee0", "metadata": {}, "outputs": [], @@ -136,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "98410bda", "metadata": {}, "outputs": [], @@ -146,21 +146,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "e3e72221", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Document(page_content='\\n \\n \\n Team\\n Location\\n Stanley Cups\\n \\n \\n Blues\\n STL\\n 1\\n \\n \\n Flyers\\n PHI\\n 2\\n \\n \\n Maple Leafs\\n TOR\\n 13\\n \\n \\n', metadata={'filetype': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'page_number': 1, 'page_name': 'Stanley Cups', 'text_as_html': '\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
TeamLocationStanley Cups
BluesSTL1
FlyersPHI2
Maple LeafsTOR13
', 'category': 'Table', 'source': 'https://drive.google.com/file/d/1aA6L2AR3g0CR-PW03HEZZo4NaVlKpaP7/view'})" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "docs[0]" ] @@ -175,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "0e2d093f", "metadata": {}, "outputs": [], @@ -190,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "b35ddcc6", "metadata": {}, "outputs": [], @@ -200,21 +189,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "3cc141e0", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Document(page_content='\\n \\n \\n Team\\n Location\\n Stanley Cups\\n \\n \\n Blues\\n STL\\n 1\\n \\n \\n Flyers\\n PHI\\n 2\\n \\n \\n Maple Leafs\\n TOR\\n 13\\n \\n \\n', metadata={'filetype': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'page_number': 1, 'page_name': 'Stanley Cups', 'text_as_html': '\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
TeamLocationStanley Cups
BluesSTL1
FlyersPHI2
Maple LeafsTOR13
', 'category': 'Table', 'source': 'https://drive.google.com/file/d/1aA6L2AR3g0CR-PW03HEZZo4NaVlKpaP7/view'})" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "docs[0]" ] @@ -226,6 +204,309 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "markdown", + "id": "83ac576b-48c9-4aad-a35e-e978ea32f746", + "metadata": {}, + "source": [ + "# Extended usage\n", + "An external component can manage the complexity of Google Drive : `langchain-googledrive`\n", + "It's compatible with the ̀`langchain.document_loaders.GoogleDriveLoader` and can be used\n", + "in its place.\n", + "\n", + "To be compatible with containers, the authentication uses an environment variable ̀GOOGLE_ACCOUNT_FILE` to credential file (for user or service)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b94f7119-bc1e-4ca3-907f-9d81e837ac59", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install langchain-googledrive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4c7474e-49cb-48a1-b3a0-77fba8e2dd70", + "metadata": {}, + "outputs": [], + "source": [ + "folder_id='root'\n", + "#folder_id='1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8357f7f1-e2b1-41ef-8e38-48fcc3897dba", + "metadata": {}, + "outputs": [], + "source": [ + "# Use the advanced version.\n", + "from langchain_googledrive.document_loaders import GoogleDriveLoader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16ab9d3d-1782-4cb9-ab56-d87edbb25a18", + "metadata": {}, + "outputs": [], + "source": [ + "loader = GoogleDriveLoader(\n", + " folder_id=folder_id,\n", + " recursive=False,\n", + " num_results=2, # Maximum number of file to load\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ebac43aa-dd64-4964-802a-a90172415fd1", + "metadata": {}, + "source": [ + "By default, all files with these mime-type can be converted to `Document`.\n", + "- text/text\n", + "- text/plain\n", + "- text/html\n", + "- text/csv\n", + "- text/markdown\n", + "- image/png\n", + "- image/jpeg\n", + "- application/epub+zip\n", + "- application/pdf\n", + "- application/rtf\n", + "- application/vnd.google-apps.document (GDoc)\n", + "- application/vnd.google-apps.presentation (GSlide)\n", + "- application/vnd.google-apps.spreadsheet (GSheet)\n", + "- application/vnd.google.colaboratory (Notebook colab)\n", + "- application/vnd.openxmlformats-officedocument.presentationml.presentation (PPTX)\n", + "- application/vnd.openxmlformats-officedocument.wordprocessingml.document (DOCX)\n", + "\n", + "It's possible to update or customize this. See the documentation of `GDriveLoader`.\n", + "\n", + "But, the corresponding packages must be installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4560f35-a37d-44e2-be0b-adaa245b3b3d", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install unstructured" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6cb08da3-27df-46de-b60e-583bb7e31af4", + "metadata": {}, + "outputs": [], + "source": [ + "for doc in loader.load():\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "cd13d7d1-db7a-498d-ac98-76ccd9ad9019", + "metadata": {}, + "source": [ + "## Customize the search pattern\n", + "\n", + "All parameter compatible with Google [`list()`](https://developers.google.com/drive/api/v3/reference/files/list)\n", + "API can be set.\n", + "\n", + "To specify the new pattern of the Google request, you can use a `PromptTemplate()`.\n", + "The variables for the prompt can be set with `kwargs` in the constructor.\n", + "Some pre-formated request are proposed (use `{query}`, `{folder_id}` and/or `{mime_type}`):\n", + "\n", + "You can customize the criteria to select the files. A set of predefined filter are proposed:\n", + "| template | description |\n", + "| -------------------------------------- | --------------------------------------------------------------------- |\n", + "| gdrive-all-in-folder | Return all compatible files from a `folder_id` |\n", + "| gdrive-query | Search `query` in all drives |\n", + "| gdrive-by-name | Search file with name `query` |\n", + "| gdrive-query-in-folder | Search `query` in `folder_id` (and sub-folders if `recursive=true`) |\n", + "| gdrive-mime-type | Search a specific `mime_type` |\n", + "| gdrive-mime-type-in-folder | Search a specific `mime_type` in `folder_id` |\n", + "| gdrive-query-with-mime-type | Search `query` with a specific `mime_type` |\n", + "| gdrive-query-with-mime-type-and-folder | Search `query` with a specific `mime_type` and in `folder_id` |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81348d59-8fd6-45d4-9de3-5df5cff5c7e2", + "metadata": {}, + "outputs": [], + "source": [ + "loader = GoogleDriveLoader(\n", + " folder_id=folder_id,\n", + " recursive=False,\n", + " template=\"gdrive-query\", # Default template to use\n", + " query=\"machine learning\",\n", + " num_results=2, # Maximum number of file to load\n", + " supportsAllDrives=False, # GDrive `list()` parameter\n", + ")\n", + "for doc in loader.load():\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "46c6ba5b-d4b1-4f0f-9801-5c1314021605", + "metadata": {}, + "source": [ + "You can customize your pattern." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a5a323b-8d96-46b7-b46a-fd69bd2c8e04", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.prompts.prompt import PromptTemplate\n", + "loader = GoogleDriveLoader(\n", + " folder_id=folder_id,\n", + " recursive=False,\n", + " template=PromptTemplate(\n", + " input_variables=[\"query\", \"query_name\"],\n", + " template=\"fullText contains '{query}' and name contains '{query_name}' and trashed=false\",\n", + " ), # Default template to use\n", + " query=\"machine learning\",\n", + " query_name=\"ML\", \n", + " num_results=2, # Maximum number of file to load\n", + ")\n", + "for doc in loader.load():\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "375bb465-8f69-407b-94bd-ffa3718ef500", + "metadata": {}, + "source": [ + "### Modes for GSlide and GSheet\n", + "The parameter mode accepts different values:\n", + "\n", + "- \"document\": return the body of each document\n", + "- \"snippets\": return the description of each file (set in metadata of Google Drive files).\n", + "\n", + "\n", + "The conversion can manage in Markdown format:\n", + "- bullet\n", + "- link\n", + "- table\n", + "- titles\n", + "\n", + "The parameter `gslide_mode` accepts different values:\n", + "\n", + "- \"single\" : one document with <PAGE BREAK>\n", + "- \"slide\" : one document by slide\n", + "- \"elements\" : one document for each elements.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7493d7b0-0600-49af-8107-7f4597c92de7", + "metadata": {}, + "outputs": [], + "source": [ + "loader = GoogleDriveLoader(\n", + " template=\"gdrive-mime-type\",\n", + " mime_type=\"application/vnd.google-apps.presentation\", # Only GSlide files\n", + " gslide_mode=\"slide\",\n", + " num_results=2, # Maximum number of file to load\n", + ")\n", + "for doc in loader.load():\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "9bf338fb-02d7-452f-8679-c50419b13464", + "metadata": {}, + "source": [ + "The parameter `gsheet_mode` accepts different values:\n", + "- `\"single\"`: Generate one document by line\n", + "- `\"elements\"` : one document with markdown array and <PAGE BREAK> tags." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "469f5af0-67db-4f15-8aee-88cde480729b", + "metadata": {}, + "outputs": [], + "source": [ + "loader = GoogleDriveLoader(\n", + " template=\"gdrive-mime-type\",\n", + " mime_type=\"application/vnd.google-apps.spreadsheet\", # Only GSheet files\n", + " gsheet_mode=\"elements\",\n", + " num_results=2, # Maximum number of file to load\n", + ")\n", + "for doc in loader.load():\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "09acb864-e919-4add-9e06-deba6f7f0cd8", + "metadata": {}, + "source": [ + "## Advanced usage\n", + "All Google File have a 'description' in the metadata. This field can be used to memorize a summary of the document or others indexed tags (See method `lazy_update_description_with_summary()`).\n", + "\n", + "If you use the `mode=\"snippet\"`, only the description will be used for the body. Else, the `metadata['summary']` has the field.\n", + "\n", + "Sometime, a specific filter can be used to extract some information from the filename, to select some files with specific criteria. You can use a filter.\n", + "\n", + "Sometimes, many documents are returned. It's not necessary to have all documents in memory at the same time. You can use the lazy versions of methods, to get one document at a time. It's better to use a complex query in place of a recursive search. For each folder, a query must be applied if you activate `recursive=True`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5e9c8eb-a266-4ae6-a760-d7826a0aa7c5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "loader = GoogleDriveLoader(\n", + " gdrive_api_file=os.environ[\"GOOGLE_ACCOUNT_FILE\"],\n", + " num_results=2,\n", + " template=\"gdrive-query\",\n", + " filter=lambda search, file: \"#test\" not in file.get('description',''),\n", + " query='machine learning',\n", + " supportsAllDrives=False,\n", + " )\n", + "for doc in loader.load():\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51efa73a-4e2d-4f9c-abaf-6c9bde2ff69d", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -244,7 +525,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.1" } }, "nbformat": 4, diff --git a/docs/extras/integrations/retrievers/google_drive.ipynb b/docs/extras/integrations/retrievers/google_drive.ipynb new file mode 100644 index 00000000000..3acb14cbc1a --- /dev/null +++ b/docs/extras/integrations/retrievers/google_drive.ipynb @@ -0,0 +1,279 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b0ed136e-6983-4893-ae1b-b75753af05f8", + "metadata": {}, + "source": [ + "# Google Drive Retriever\n", + "This notebook covers how to retrieve documents from Google Drive.\n", + "\n", + "## Prerequisites\n", + "\n", + "1. Create a Google Cloud project or use an existing project\n", + "1. Enable the [Google Drive API](https://console.cloud.google.com/flows/enableapi?apiid=drive.googleapis.com)\n", + "1. [Authorize credentials for desktop app](https://developers.google.com/drive/api/quickstart/python#authorize_credentials_for_a_desktop_application)\n", + "1. `pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib`\n", + "\n", + "## Instructions for retrieving your Google Docs data\n", + "By default, the `GoogleDriveRetriever` expects the `credentials.json` file to be `~/.credentials/credentials.json`, but this is configurable using the `GOOGLE_ACCOUNT_FILE` environment variable. \n", + "The location of `token.json` use the same directory (or use the parameter `token_path`). Note that `token.json` will be created automatically the first time you use the retriever.\n", + "\n", + "`GoogleDriveRetriever` can retrieve a selection of files with some requests. \n", + "\n", + "By default, If you use a `folder_id`, all the files inside this folder can be retrieved to `Document`.\n" + ] + }, + { + "cell_type": "markdown", + "id": "35b94a93-97de-4af8-9cca-de9ffb7930c3", + "metadata": {}, + "source": [ + "You can obtain your folder and document id from the URL:\n", + "* Folder: https://drive.google.com/drive/u/0/folders/1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5 -> folder id is `\"1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5\"`\n", + "* Document: https://docs.google.com/document/d/1bfaMQ18_i56204VaQDVeAFpqEijJTgvurupdEDiaUQw/edit -> document id is `\"1bfaMQ18_i56204VaQDVeAFpqEijJTgvurupdEDiaUQw\"`\n", + "\n", + "The special value `root` is for your personal home." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c9665c9-a023-4078-9d95-e43021cecb6f", + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "878928a6-a5ae-4f74-b351-64e3b01733fe", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-09T10:45:59.438650905Z", + "start_time": "2023-05-09T10:45:57.955900302Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.retrievers import GoogleDriveRetriever" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "755907c2-145d-4f0f-9b15-07a628a2d2d2", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-09T10:45:59.442890834Z", + "start_time": "2023-05-09T10:45:59.440941528Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "folder_id=\"root\"\n", + "#folder_id='1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2216c83f-68e4-4d2f-8ea2-5878fb18bbe7", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-09T10:45:59.795842403Z", + "start_time": "2023-05-09T10:45:59.445262457Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "retriever = GoogleDriveRetriever(\n", + " num_results=2,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fa339ca0-f478-440c-ba80-0e5f41a19ce1", + "metadata": {}, + "source": [ + "By default, all files with these mime-type can be converted to `Document`.\n", + "- text/text\n", + "- text/plain\n", + "- text/html\n", + "- text/csv\n", + "- text/markdown\n", + "- image/png\n", + "- image/jpeg\n", + "- application/epub+zip\n", + "- application/pdf\n", + "- application/rtf\n", + "- application/vnd.google-apps.document (GDoc)\n", + "- application/vnd.google-apps.presentation (GSlide)\n", + "- application/vnd.google-apps.spreadsheet (GSheet)\n", + "- application/vnd.google.colaboratory (Notebook colab)\n", + "- application/vnd.openxmlformats-officedocument.presentationml.presentation (PPTX)\n", + "- application/vnd.openxmlformats-officedocument.wordprocessingml.document (DOCX)\n", + "\n", + "It's possible to update or customize this. See the documentation of `GDriveRetriever`.\n", + "\n", + "But, the corresponding packages must be installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9dadec48", + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install unstructured" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f3b6aa0-b45d-4e37-8c50-5bebe70fdb9d", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-09T10:46:00.990310466Z", + "start_time": "2023-05-09T10:45:59.798774595Z" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "retriever.get_relevant_documents(\"machine learning\")" + ] + }, + { + "cell_type": "markdown", + "id": "8ff33817-8619-4897-8742-2216b9934d2a", + "metadata": {}, + "source": [ + "You can customize the criteria to select the files. A set of predefined filter are proposed:\n", + "| template | description |\n", + "| -------------------------------------- | --------------------------------------------------------------------- |\n", + "| gdrive-all-in-folder | Return all compatible files from a `folder_id` |\n", + "| gdrive-query | Search `query` in all drives |\n", + "| gdrive-by-name | Search file with name `query`) |\n", + "| gdrive-query-in-folder | Search `query` in `folder_id` (and sub-folders in `_recursive=true`) |\n", + "| gdrive-mime-type | Search a specific `mime_type` |\n", + "| gdrive-mime-type-in-folder | Search a specific `mime_type` in `folder_id` |\n", + "| gdrive-query-with-mime-type | Search `query` with a specific `mime_type` |\n", + "| gdrive-query-with-mime-type-and-folder | Search `query` with a specific `mime_type` and in `folder_id` |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9977c712-9659-4959-b508-f59cc7d49d44", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "retriever = GoogleDriveRetriever(\n", + " template=\"gdrive-query\", # Search everywhere\n", + " num_results=2, # But take only 2 documents\n", + ")\n", + "for doc in retriever.get_relevant_documents(\"machine learning\"):\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "a5a0f3ef-26fb-4a5c-85f0-5aba90b682b1", + "metadata": {}, + "source": [ + "Else, you can customize the prompt with a specialized `PromptTemplate`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0bbebde-0487-4d20-9d77-8070e4f0e0d6", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain import PromptTemplate\n", + "retriever = GoogleDriveRetriever(\n", + " template=PromptTemplate(input_variables=['query'],\n", + " # See https://developers.google.com/drive/api/guides/search-files\n", + " template=\"(fullText contains '{query}') \"\n", + " \"and mimeType='application/vnd.google-apps.document' \"\n", + " \"and modifiedTime > '2000-01-01T00:00:00' \"\n", + " \"and trashed=false\"),\n", + " num_results=2,\n", + " # See https://developers.google.com/drive/api/v3/reference/files/list\n", + " includeItemsFromAllDrives=False,\n", + " supportsAllDrives=False,\n", + ")\n", + "for doc in retriever.get_relevant_documents(\"machine learning\"):\n", + " print(f\"{doc.metadata['name']}:\")\n", + " print(\"---\")\n", + " print(doc.page_content.strip()[:60]+\"...\")" + ] + }, + { + "cell_type": "markdown", + "id": "9b6fed29-1666-452e-b677-401613270388", + "metadata": {}, + "source": [ + "# Use GDrive 'description' metadata\n", + "Each Google Drive has a `description` field in metadata (see the *details of a file*).\n", + "Use the `snippets` mode to return the description of selected files.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "342dbe12-ed83-40f4-8957-0cc8c4609542", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "retriever = GoogleDriveRetriever(\n", + " template='gdrive-mime-type-in-folder',\n", + " folder_id=folder_id,\n", + " mime_type='application/vnd.google-apps.document', # Only Google Docs\n", + " num_results=2,\n", + " mode='snippets',\n", + " includeItemsFromAllDrives=False,\n", + " supportsAllDrives=False,\n", + ")\n", + "retriever.get_relevant_documents(\"machine learning\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/extras/integrations/toolkits/google_drive.ipynb b/docs/extras/integrations/toolkits/google_drive.ipynb new file mode 100644 index 00000000000..6cf1cb56d46 --- /dev/null +++ b/docs/extras/integrations/toolkits/google_drive.ipynb @@ -0,0 +1,215 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Google Drive tool\n", + "\n", + "This notebook walks through connecting a LangChain to the Google Drive API.\n", + "\n", + "## Prerequisites\n", + "\n", + "1. Create a Google Cloud project or use an existing project\n", + "1. Enable the [Google Drive API](https://console.cloud.google.com/flows/enableapi?apiid=drive.googleapis.com)\n", + "1. [Authorize credentials for desktop app](https://developers.google.com/drive/api/quickstart/python#authorize_credentials_for_a_desktop_application)\n", + "1. `pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib`\n", + "\n", + "## Instructions for retrieving your Google Docs data\n", + "By default, the `GoogleDriveTools` and `GoogleDriveWrapper` expects the `credentials.json` file to be `~/.credentials/credentials.json`, but this is configurable using the `GOOGLE_ACCOUNT_FILE` environment variable. \n", + "The location of `token.json` use the same directory (or use the parameter `token_path`). Note that `token.json` will be created automatically the first time you use the tool.\n", + "\n", + "`GoogleDriveSearchTool` can retrieve a selection of files with some requests. \n", + "\n", + "By default, If you use a `folder_id`, all the files inside this folder can be retrieved to `Document`, if the name match the query.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can obtain your folder and document id from the URL:\n", + "* Folder: https://drive.google.com/drive/u/0/folders/1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5 -> folder id is `\"1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5\"`\n", + "* Document: https://docs.google.com/document/d/1bfaMQ18_i56204VaQDVeAFpqEijJTgvurupdEDiaUQw/edit -> document id is `\"1bfaMQ18_i56204VaQDVeAFpqEijJTgvurupdEDiaUQw\"`\n", + "\n", + "The special value `root` is for your personal home." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "folder_id=\"root\"\n", + "#folder_id='1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, all files with these mime-type can be converted to `Document`.\n", + "- text/text\n", + "- text/plain\n", + "- text/html\n", + "- text/csv\n", + "- text/markdown\n", + "- image/png\n", + "- image/jpeg\n", + "- application/epub+zip\n", + "- application/pdf\n", + "- application/rtf\n", + "- application/vnd.google-apps.document (GDoc)\n", + "- application/vnd.google-apps.presentation (GSlide)\n", + "- application/vnd.google-apps.spreadsheet (GSheet)\n", + "- application/vnd.google.colaboratory (Notebook colab)\n", + "- application/vnd.openxmlformats-officedocument.presentationml.presentation (PPTX)\n", + "- application/vnd.openxmlformats-officedocument.wordprocessingml.document (DOCX)\n", + "\n", + "It's possible to update or customize this. See the documentation of `GoogleDriveAPIWrapper`.\n", + "\n", + "But, the corresponding packages must installed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install unstructured" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.utilities.google_drive import GoogleDriveAPIWrapper\n", + "from langchain.tools.google_drive.tool import GoogleDriveSearchTool\n", + "\n", + "# By default, search only in the filename.\n", + "tool = GoogleDriveSearchTool(\n", + " api_wrapper=GoogleDriveAPIWrapper(\n", + " folder_id=folder_id,\n", + " num_results=2,\n", + " template=\"gdrive-query-in-folder\", # Search in the body of documents\n", + " )\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "logging.basicConfig(level=logging.INFO)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool.run(\"machine learning\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool.description" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.agents import load_tools\n", + "tools = load_tools([\"google-drive-search\"],\n", + " folder_id=folder_id,\n", + " template=\"gdrive-query-in-folder\",\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use within an Agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain import OpenAI\n", + "from langchain.agents import initialize_agent, AgentType\n", + "llm = OpenAI(temperature=0)\n", + "agent = initialize_agent(\n", + " tools=tools,\n", + " llm=llm,\n", + " agent=AgentType.STRUCTURED_CHAT_ZERO_SHOT_REACT_DESCRIPTION,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "agent.run(\n", + " \"Search in google drive, who is 'Yann LeCun' ?\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}