diff --git a/docs/docs/integrations/document_loaders/pypdfdirectory.ipynb b/docs/docs/integrations/document_loaders/pypdfdirectory.ipynb index 69c49aca36d..534adda7fda 100644 --- a/docs/docs/integrations/document_loaders/pypdfdirectory.ipynb +++ b/docs/docs/integrations/document_loaders/pypdfdirectory.ipynb @@ -36,13 +36,18 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:00:08.878423Z", + "start_time": "2025-01-21T08:00:08.876042Z" + } + }, "source": [ "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n", "# os.environ[\"LANGSMITH_TRACING\"] = \"true\"" - ] + ], + "outputs": [], + "execution_count": 1 }, { "cell_type": "markdown", @@ -54,17 +59,28 @@ ] }, { + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:00:12.003718Z", + "start_time": "2025-01-21T08:00:10.291617Z" + } + }, "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -qU langchain_community" - ] + "source": "%pip install -qU langchain_community pypdf pillow", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 2 }, { - "cell_type": "markdown", "metadata": {}, + "cell_type": "markdown", "source": [ "## Initialization\n", "\n", @@ -72,10 +88,13 @@ ] }, { + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:00:18.512061Z", + "start_time": "2025-01-21T08:00:17.313969Z" + } + }, "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], "source": [ "from langchain_community.document_loaders import PyPDFDirectoryLoader\n", "\n", @@ -83,7 +102,9 @@ " \"../../docs/integrations/document_loaders/example_data/layout-parser-paper.pdf\"\n", ")\n", "loader = PyPDFDirectoryLoader(\"example_data/\")" - ] + ], + "outputs": [], + "execution_count": 3 }, { "cell_type": "markdown", @@ -94,41 +115,51 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:00:23.549752Z", + "start_time": "2025-01-21T08:00:23.129010Z" + } + }, + "source": [ + "docs = loader.load()\n", + "docs[0]" + ], "outputs": [ { "data": { "text/plain": [ - "Document(metadata={'source': 'example_data/layout-parser-paper.pdf', 'page': 0}, page_content='LayoutParser : A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1( \\x00), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1Allen Institute for AI\\nshannons@allenai.org\\n2Brown University\\nruochen zhang@brown.edu\\n3Harvard University\\n{melissadell,jacob carlson }@fas.harvard.edu\\n4University of Washington\\nbcgl@cs.washington.edu\\n5University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser , an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io .\\nKeywords: Document Image Analysis ·Deep Learning ·Layout Analysis\\n·Character Recognition ·Open Source library ·Toolkit.\\n1 Introduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [ 11,arXiv:2103.15348v2 [cs.CV] 21 Jun 2021')" + "Document(metadata={'producer': 'pdfTeX-1.40.21', 'creator': 'LaTeX with hyperref', 'creationdate': '2021-06-22T01:27:10+00:00', 'author': '', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'subject': '', 'title': '', 'trapped': '/False', 'source': 'example_data/layout-parser-paper.pdf', 'total_pages': 16, 'page': 0, 'page_label': '1'}, page_content='LayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1 (\\x00 ), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\nKeywords: Document Image Analysis · Deep Learning · Layout Analysis\\n· Character Recognition · Open Source library · Toolkit.\\n1 Introduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [11,\\narXiv:2103.15348v2 [cs.CV] 21 Jun 2021')" ] }, - "execution_count": 2, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], - "source": [ - "docs = loader.load()\n", - "docs[0]" - ] + "execution_count": 4 }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:00:26.612346Z", + "start_time": "2025-01-21T08:00:26.609051Z" + } + }, + "source": [ + "print(docs[0].metadata)" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'source': 'example_data/layout-parser-paper.pdf', 'page': 0}\n" + "{'producer': 'pdfTeX-1.40.21', 'creator': 'LaTeX with hyperref', 'creationdate': '2021-06-22T01:27:10+00:00', 'author': '', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'subject': '', 'title': '', 'trapped': '/False', 'source': 'example_data/layout-parser-paper.pdf', 'total_pages': 16, 'page': 0, 'page_label': '1'}\n" ] } ], - "source": [ - "print(docs[0].metadata)" - ] + "execution_count": 5 }, { "cell_type": "markdown", @@ -139,9 +170,12 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:00:30.251598Z", + "start_time": "2025-01-21T08:00:29.972141Z" + } + }, "source": [ "page = []\n", "for doc in loader.lazy_load():\n", @@ -151,7 +185,9 @@ " # index.upsert(page)\n", "\n", " page = []" - ] + ], + "outputs": [], + "execution_count": 6 }, { "cell_type": "markdown", @@ -161,6 +197,13 @@ "\n", "For detailed documentation of all PyPDFDirectoryLoader features and configurations head to the API reference: https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyPDFDirectoryLoader.html" ] + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "" } ], "metadata": { diff --git a/docs/docs/integrations/document_loaders/pypdfloader.ipynb b/docs/docs/integrations/document_loaders/pypdfloader.ipynb index b0cc79d92de..70b4a52258f 100644 --- a/docs/docs/integrations/document_loaders/pypdfloader.ipynb +++ b/docs/docs/integrations/document_loaders/pypdfloader.ipynb @@ -8,18 +8,24 @@ "\n", "This notebook provides a quick overview for getting started with `PyPDF` [document loader](https://python.langchain.com/docs/concepts/document_loaders). For detailed documentation of all DocumentLoader features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyPDFLoader.html).\n", "\n", + " \n", "\n", "## Overview\n", "### Integration details\n", "\n", - "\n", "| Class | Package | Local | Serializable | JS support|\n", "| :--- | :--- | :---: | :---: | :---: |\n", - "| [PyPDFLoader](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyPDFLoader.html) | [langchain_community](https://python.langchain.com/api_reference/community/index.html) | ✅ | ❌ | ❌ | \n", + "| [PyPDFLoader](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.pdf.PyPDFLoader.html) | [langchain_community](https://python.langchain.com/api_reference/community/index.html) | ✅ | ❌ | ❌ | \n", + " \n", + "--------- \n", + "\n", "### Loader features\n", - "| Source | Document Lazy Loading | Native Async Support\n", - "| :---: | :---: | :---: | \n", - "| PyPDFLoader | ✅ | ❌ | \n", + "\n", + "| Source | Document Lazy Loading | Native Async Support | Extract Images | Extract Tables |\n", + "|:-----------:| :---: | :---: | :---: |:---: |\n", + "| PyPDFLoader | ✅ | ❌ | ✅ | ❌ |\n", + "\n", + " \n", "\n", "## Setup\n", "\n", @@ -28,20 +34,53 @@ "No credentials are required to use `PyPDFLoader`." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "If you want to get automated best in-class tracing of your model calls you can also set your [LangSmith](https://docs.smith.langchain.com/) API key by uncommenting below:" + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:08.825630Z", + "start_time": "2025-01-21T08:01:08.823315Z" + } + }, + "outputs": [], + "source": [ + "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n", + "# os.environ[\"LANGSMITH_TRACING\"] = \"true\"" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Installation\n", "\n", - "To use `PyPDFLoader` you need to have the `langchain-community` python package downloaded:" + "Install **langchain_community** and **pypdf**." ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:10.997240Z", + "start_time": "2025-01-21T08:01:09.529997Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -qU langchain_community pypdf" ] @@ -57,15 +96,19 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": {}, + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:14.685958Z", + "start_time": "2025-01-21T08:01:13.653438Z" + } + }, "outputs": [], "source": [ "from langchain_community.document_loaders import PyPDFLoader\n", "\n", - "loader = PyPDFLoader(\n", - " \"./example_data/layout-parser-paper.pdf\",\n", - ")" + "file_path = \"./example_data/layout-parser-paper.pdf\"\n", + "loader = PyPDFLoader(file_path)" ] }, { @@ -77,16 +120,21 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:17.433959Z", + "start_time": "2025-01-21T08:01:17.080724Z" + } + }, "outputs": [ { "data": { "text/plain": [ - "Document(metadata={'source': './example_data/layout-parser-paper.pdf', 'page': 0}, page_content='LayoutParser : A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1( \\x00), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1Allen Institute for AI\\nshannons@allenai.org\\n2Brown University\\nruochen zhang@brown.edu\\n3Harvard University\\n{melissadell,jacob carlson }@fas.harvard.edu\\n4University of Washington\\nbcgl@cs.washington.edu\\n5University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser , an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io .\\nKeywords: Document Image Analysis ·Deep Learning ·Layout Analysis\\n·Character Recognition ·Open Source library ·Toolkit.\\n1 Introduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [ 11,arXiv:2103.15348v2 [cs.CV] 21 Jun 2021')" + "Document(metadata={'producer': 'pdfTeX-1.40.21', 'creator': 'LaTeX with hyperref', 'creationdate': '2021-06-22T01:27:10+00:00', 'author': '', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'subject': '', 'title': '', 'trapped': '/False', 'source': './example_data/layout-parser-paper.pdf', 'total_pages': 16, 'page': 0, 'page_label': '1'}, page_content='LayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\nZejiang Shen1 (\\x00 ), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\nKeywords: Document Image Analysis · Deep Learning · Layout Analysis\\n· Character Recognition · Open Source library · Toolkit.\\n1 Introduction\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [11,\\narXiv:2103.15348v2 [cs.CV] 21 Jun 2021')" ] }, - "execution_count": 2, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -98,19 +146,40 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:18.619845Z", + "start_time": "2025-01-21T08:01:18.615643Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'source': './example_data/layout-parser-paper.pdf', 'page': 0}\n" + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'author': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n", + " '2020) kpathsea version 6.3.2',\n", + " 'subject': '',\n", + " 'title': '',\n", + " 'trapped': '/False',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'page': 0,\n", + " 'page_label': '1'}\n" ] } ], "source": [ - "print(docs[0].metadata)" + "import pprint\n", + "\n", + "pprint.pp(docs[0].metadata)" ] }, { @@ -122,8 +191,13 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:20.450806Z", + "start_time": "2025-01-21T08:01:20.176333Z" + } + }, "outputs": [ { "data": { @@ -131,7 +205,7 @@ "6" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -150,22 +224,867 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:21.267444Z", + "start_time": "2025-01-21T08:01:21.263726Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "LayoutParser : A Unified Toolkit for DL-Based DIA 11\n", - "focuses on precision, efficiency, and robustness. \n", - "{'source': './example_data/layout-parser-paper.pdf', 'page': 10}\n" + "LayoutParser: A Unified Toolkit for DL-Based DIA 11\n", + "focuses on precision, efficiency, and robustness. T\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'author': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n", + " '2020) kpathsea version 6.3.2',\n", + " 'subject': '',\n", + " 'title': '',\n", + " 'trapped': '/False',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'page': 10,\n", + " 'page_label': '11'}\n" ] } ], "source": [ "print(pages[0].page_content[:100])\n", - "print(pages[0].metadata)" + "pprint.pp(pages[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The metadata attribute contains at least the following keys:\n", + "- source\n", + "- page (if in mode *page*)\n", + "- total_page\n", + "- creationdate\n", + "- creator\n", + "- producer\n", + "\n", + "Additional metadata are specific to each parser.\n", + "These pieces of information can be helpful (to categorize your PDFs for example)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Splitting mode & custom pages delimiter" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When loading the PDF file you can split it in two different ways:\n", + "- By page\n", + "- As a single text flow\n", + "\n", + "By default PyPDFLoader will split the PDF as a single text flow." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Extract the PDF by page. Each page is extracted as a langchain Document object:" + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:28.128153Z", + "start_time": "2025-01-21T08:01:27.823798Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'author': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n", + " '2020) kpathsea version 6.3.2',\n", + " 'subject': '',\n", + " 'title': '',\n", + " 'trapped': '/False',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'page': 0,\n", + " 'page_label': '1'}\n" + ] + } + ], + "source": [ + "loader = PyPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + ")\n", + "docs = loader.load()\n", + "print(len(docs))\n", + "pprint.pp(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "In this mode the pdf is split by pages and the resulting Documents metadata contains the page number. But in some cases we could want to process the pdf as a single text flow (so we don't cut some paragraphs in half). In this case you can use the *single* mode :" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Extract the whole PDF as a single langchain Document object:" + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:31.794895Z", + "start_time": "2025-01-21T08:01:31.470806Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'author': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n", + " '2020) kpathsea version 6.3.2',\n", + " 'subject': '',\n", + " 'title': '',\n", + " 'trapped': '/False',\n", + " 'source': './example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16}\n" + ] + } + ], + "source": [ + "loader = PyPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"single\",\n", + ")\n", + "docs = loader.load()\n", + "print(len(docs))\n", + "pprint.pp(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Logically, in this mode, the ‘page_number’ metadata disappears. Here's how to clearly identify where pages end in the text flow :" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Add a custom *pages_delimiter* to identify where are ends of pages in *single* mode:" + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:35.848808Z", + "start_time": "2025-01-21T08:01:35.575903Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LayoutParser: A Unified Toolkit for Deep\n", + "Learning Based Document Image Analysis\n", + "Zejiang Shen1 (\u0000 ), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\n", + "Lee4, Jacob Carlson3, and Weining Li5\n", + "1 Allen Institute for AI\n", + "shannons@allenai.org\n", + "2 Brown University\n", + "ruochen zhang@brown.edu\n", + "3 Harvard University\n", + "{melissadell,jacob carlson}@fas.harvard.edu\n", + "4 University of Washington\n", + "bcgl@cs.washington.edu\n", + "5 University of Waterloo\n", + "w422li@uwaterloo.ca\n", + "Abstract. Recent advances in document image analysis (DIA) have been\n", + "primarily driven by the application of neural networks. Ideally, research\n", + "outcomes could be easily deployed in production and extended for further\n", + "investigation. However, various factors like loosely organized codebases\n", + "and sophisticated model configurations complicate the easy reuse of im-\n", + "portant innovations by a wide audience. Though there have been on-going\n", + "efforts to improve reusability and simplify deep learning (DL) model\n", + "development in disciplines like natural language processing and computer\n", + "vision, none of them are optimized for challenges in the domain of DIA.\n", + "This represents a major gap in the existing toolkit, as DIA is central to\n", + "academic research across a wide range of disciplines in the social sciences\n", + "and humanities. This paper introduces LayoutParser, an open-source\n", + "library for streamlining the usage of DL in DIA research and applica-\n", + "tions. The core LayoutParser library comes with a set of simple and\n", + "intuitive interfaces for applying and customizing DL models for layout de-\n", + "tection, character recognition, and many other document processing tasks.\n", + "To promote extensibility, LayoutParser also incorporates a community\n", + "platform for sharing both pre-trained models and full document digiti-\n", + "zation pipelines. We demonstrate that LayoutParser is helpful for both\n", + "lightweight and large-scale digitization pipelines in real-word use cases.\n", + "The library is publicly available at https://layout-parser.github.io.\n", + "Keywords: Document Image Analysis · Deep Learning · Layout Analysis\n", + "· Character Recognition · Open Source library · Toolkit.\n", + "1 Introduction\n", + "Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of\n", + "document image analysis (DIA) tasks including document image classification [11,\n", + "arXiv:2103.15348v2 [cs.CV] 21 Jun 2021\n", + "-------THIS IS A CUSTOM END OF PAGE-------\n", + "2 Z. Shen et al.\n", + "37], layout detection [38, 22], table detection [ 26], and scene text detection [ 4].\n", + "A generalized learning-based framework dramatically reduces the need for the\n", + "manual specification of complicated rules, which is the status quo with traditional\n", + "methods. DL has the potential to transform DIA pipelines and benefit a broad\n", + "spectrum of large-scale document digitization projects.\n", + "However, there are several practical difficulties for taking advantages of re-\n", + "cent advances in DL-based methods: 1) DL models are notoriously convoluted\n", + "for reuse and extension. Existing models are developed using distinct frame-\n", + "works like TensorFlow [1] or PyTorch [ 24], and the high-level parameters can\n", + "be obfuscated by implementation details [ 8]. It can be a time-consuming and\n", + "frustrating experience to debug, reproduce, and adapt existing models for DIA,\n", + "and many researchers who would benefit the most from using these methods lack\n", + "the technical background to implement them from scratch. 2) Document images\n", + "contain diverse and disparate patterns across domains, and customized training\n", + "is often required to achieve a desirable detection accuracy. Currently there is no\n", + "full-fledged infrastructure for easily curating the target document image datasets\n", + "and fine-tuning or re-training the models. 3) DIA usually requires a sequence of\n", + "models and other processing to obtain the final outputs. Often research teams use\n", + "DL models and then perform further document analyses in separate processes,\n", + "and these pipelines are not documented in any central location (and often not\n", + "documented at all). This makes it difficult for research teams to learn about how\n", + "full pipelines are implemented and leads them to invest significant resources in\n", + "reinventing the DIA wheel .\n", + "LayoutParser provides a unified toolkit to support DL-based document image\n", + "analysis and processing. To address the aforementioned challenges,LayoutParser\n", + "is built with the following components:\n", + "1. An off-the-shelf toolkit for applying DL models for layout detection, character\n", + "recognition, and other DIA tasks (Section 3)\n", + "2. A rich repository of pre-trained neural network models (Model Zoo) that\n", + "underlies the off-the-shelf usage\n", + "3. Comprehensive tools for efficient document image data annotation and model\n", + "tuning to support different levels of customization\n", + "4. A DL model hub and community platform for the easy sharing, distribu-\n", + "tion, and discussion of DIA models and pipelines, to promote reusability,\n", + "reproducibility, and extensibility (Section 4)\n", + "The library implements simple and intuitive Python APIs without sacrificing\n", + "generalizability and versatility, and can be easily installed via pip. Its convenient\n", + "functions for handling document image data can be seamlessly integrated with\n", + "existing DIA pipelines. With detailed documentations and carefully curated\n", + "tutorials, we hope this tool will benefit a variety of end-users, and will lead to\n", + "advances in applications in both industry and academic research.\n", + "LayoutParser is well aligned with recent efforts for improving DL model\n", + "reusability in other disciplines like natural language processing [ 8, 34] and com-\n", + "puter vision [ 35], but with a focus on unique challenges in DIA. We show\n", + "LayoutParser can be applied in sophisticated and large-scale digitization projects\n", + "-------THIS IS A CUSTOM END OF PAGE-------\n", + "LayoutParser: A Unified Toolkit for DL-Based DIA 3\n", + "that require precision, efficiency, and robustness, as well as simple and light\n" + ] + } + ], + "source": [ + "loader = PyPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"single\",\n", + " pages_delimiter=\"\\n-------THIS IS A CUSTOM END OF PAGE-------\\n\",\n", + ")\n", + "docs = loader.load()\n", + "print(docs[0].page_content[:5780])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "This could simply be \\n, or \\f to clearly indicate a page change, or \\ for seamless injection in a Markdown viewer without a visual effect." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "# Extract images from the PDF" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can extract images from your PDFs with a choice of three different solutions:\n", + "- rapidOCR (lightweight Optical Character Recognition tool)\n", + "- Tesseract (OCR tool with high precision)\n", + "- Multimodal language model\n", + "\n", + "You can tune these functions to choose the output format of the extracted images among *html*, *markdown* or *text*\n", + "\n", + "The result is inserted between the last and the second-to-last paragraphs of text of the page." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Extract images from the PDF with rapidOCR:" + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:01:40.692855Z", + "start_time": "2025-01-21T08:01:39.293791Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU rapidocr-onnxruntime" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:02:07.273962Z", + "start_time": "2025-01-21T08:01:42.848244Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6 Z. Shen et al.\n", + "Fig. 2: The relationship between the three types of layout data structures.\n", + "Coordinate supports three kinds of variation; TextBlock consists of the co-\n", + "ordinate information and extra features like block text, types, and reading orders;\n", + "a Layout object is a list of all possible layout elements, including other Layout\n", + "objects. They all support the same set of transformation and operation APIs for\n", + "maximum flexibility.\n", + "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n", + "on 5 different datasets. Description of the training dataset is provided alongside\n", + "with the trained models such that users can quickly identify the most suitable\n", + "models for their tasks. Additionally, when such a model is not readily available,\n", + "LayoutParser also supports training customized layout models and community\n", + "sharing of the models (detailed in Section 3.5).\n", + "3.2 Layout Data Structures\n", + "A critical feature of LayoutParser is the implementation of a series of data\n", + "structures and operations that can be used to efficiently process and manipulate\n", + "the layout elements. In document image analysis pipelines, various post-processing\n", + "on the layout analysis model outputs is usually required to obtain the final\n", + "outputs. Traditionally, this requires exporting DL model outputs and then loading\n", + "the results into other pipelines. All model outputs from LayoutParser will be\n", + "stored in carefully engineered data types optimized for further processing, which\n", + "makes it possible to build an end-to-end document digitization pipeline within\n", + "LayoutParser. There are three key components in the data structure, namely\n", + "the Coordinate system, the TextBlock, and the Layout. They provide different\n", + "levels of abstraction for the layout data, and a set of APIs are supported for\n", + "transformations or operations on these classes.\n", + "\n", + "\n", + "\n", + "![Coordinate\n", + "(x1, y1)\n", + "(X1, y1)\n", + "(x2,y2)\n", + "APIS\n", + "x-interval\n", + "tart\n", + "end\n", + "Quadrilateral\n", + "operation\n", + "Rectangle\n", + "y-interval\n", + "ena\n", + "(x2, y2)\n", + "(x4, y4)\n", + "(x3, y3)\n", + "and\n", + "textblock\n", + "Coordinate\n", + "transformation\n", + "+\n", + "Block\n", + "Block\n", + "Reading\n", + "Extra features\n", + "Text\n", + "Type\n", + "Order\n", + "coordinatel\n", + "textblockl\n", + "layout\n", + " same\n", + "textblock2\n", + "layoutl\n", + "The\n", + "A list of the layout elements](#)\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders.parsers import RapidOCRBlobParser\n", + "\n", + "loader = PyPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " images_inner_format=\"markdown-img\",\n", + " images_parser=RapidOCRBlobParser(),\n", + ")\n", + "docs = loader.load()\n", + "\n", + "print(docs[5].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Be careful, RapidOCR is designed to work with Chinese and English, not other languages." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Extract images from the PDF with Tesseract:" + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:02:12.070378Z", + "start_time": "2025-01-21T08:02:10.696635Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU pytesseract" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:02:21.712219Z", + "start_time": "2025-01-21T08:02:12.081700Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6 Z. Shen et al.\n", + "Fig. 2: The relationship between the three types of layout data structures.\n", + "Coordinate supports three kinds of variation; TextBlock consists of the co-\n", + "ordinate information and extra features like block text, types, and reading orders;\n", + "a Layout object is a list of all possible layout elements, including other Layout\n", + "objects. They all support the same set of transformation and operation APIs for\n", + "maximum flexibility.\n", + "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n", + "on 5 different datasets. Description of the training dataset is provided alongside\n", + "with the trained models such that users can quickly identify the most suitable\n", + "models for their tasks. Additionally, when such a model is not readily available,\n", + "LayoutParser also supports training customized layout models and community\n", + "sharing of the models (detailed in Section 3.5).\n", + "3.2 Layout Data Structures\n", + "A critical feature of LayoutParser is the implementation of a series of data\n", + "structures and operations that can be used to efficiently process and manipulate\n", + "the layout elements. In document image analysis pipelines, various post-processing\n", + "on the layout analysis model outputs is usually required to obtain the final\n", + "outputs. Traditionally, this requires exporting DL model outputs and then loading\n", + "the results into other pipelines. All model outputs from LayoutParser will be\n", + "stored in carefully engineered data types optimized for further processing, which\n", + "makes it possible to build an end-to-end document digitization pipeline within\n", + "LayoutParser. There are three key components in the data structure, namely\n", + "the Coordinate system, the TextBlock, and the Layout. They provide different\n", + "levels of abstraction for the layout data, and a set of APIs are supported for\n", + "transformations or operations on these classes.\n", + "\n", + "\n", + "\n", + "\"Coordinate\n",\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders.parsers import TesseractBlobParser\n", + "\n", + "loader = PyPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " images_inner_format=\"html-img\",\n", + " images_parser=TesseractBlobParser(),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[5].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "### Extract images from the PDF with multimodal model:" + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:02:25.912928Z", + "start_time": "2025-01-21T08:02:24.324014Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain_openai" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:02:28.494996Z", + "start_time": "2025-01-21T08:02:28.468181Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:02:29.318093Z", + "start_time": "2025-01-21T08:02:29.314654Z" + } + }, + "outputs": [], + "source": [ + "from getpass import getpass\n", + "\n", + "if not os.environ.get(\"OPENAI_API_KEY\"):\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:05:00.352337Z", + "start_time": "2025-01-21T08:02:30.723099Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6 Z. Shen et al.\n", + "Fig. 2: The relationship between the three types of layout data structures.\n", + "Coordinate supports three kinds of variation; TextBlock consists of the co-\n", + "ordinate information and extra features like block text, types, and reading orders;\n", + "a Layout object is a list of all possible layout elements, including other Layout\n", + "objects. They all support the same set of transformation and operation APIs for\n", + "maximum flexibility.\n", + "Shown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\n", + "on 5 different datasets. Description of the training dataset is provided alongside\n", + "with the trained models such that users can quickly identify the most suitable\n", + "models for their tasks. Additionally, when such a model is not readily available,\n", + "LayoutParser also supports training customized layout models and community\n", + "sharing of the models (detailed in Section 3.5).\n", + "3.2 Layout Data Structures\n", + "A critical feature of LayoutParser is the implementation of a series of data\n", + "structures and operations that can be used to efficiently process and manipulate\n", + "the layout elements. In document image analysis pipelines, various post-processing\n", + "on the layout analysis model outputs is usually required to obtain the final\n", + "outputs. Traditionally, this requires exporting DL model outputs and then loading\n", + "the results into other pipelines. All model outputs from LayoutParser will be\n", + "stored in carefully engineered data types optimized for further processing, which\n", + "makes it possible to build an end-to-end document digitization pipeline within\n", + "LayoutParser. There are three key components in the data structure, namely\n", + "the Coordinate system, the TextBlock, and the Layout. They provide different\n", + "levels of abstraction for the layout data, and a set of APIs are supported for\n", + "transformations or operations on these classes.\n", + "\n", + "\n", + "\n", + "![**Image Summary:** \n", + "Diagram explaining coordinate systems and layout elements with labels for coordinate intervals, rectangle, quadrilateral, textblock with extra features, and layout list. Includes transformation and operation APIs.\n", + "\n", + "**Extracted Text:** \n", + "Coordinate \n", + "coordinate \n", + "start \n", + "start \n", + "x-interval \n", + "end \n", + "y-interval \n", + "end \n", + "(x1, y1) \n", + "Rectangle \n", + "(x2, y2) \n", + "(x1, y1) \n", + "Quadrilateral \n", + "(x2, y2) \n", + "(x4, y4) \n", + "(x3, y3) \n", + "The same transformation and operation APIs \n", + "textblock \n", + "Coordinate \n", + "Extra features \n", + "Block Text \n", + "Block Type \n", + "Reading Order \n", + "... \n", + "layout \n", + "coordinate1, textblock1, ... \n", + "..., textblock2, layout1 \n", + "A list of the layout elements ](#)\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders.parsers import LLMImageBlobParser\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "loader = PyPDFLoader(\n", + " \"./example_data/layout-parser-paper.pdf\",\n", + " mode=\"page\",\n", + " images_inner_format=\"markdown-img\",\n", + " images_parser=LLMImageBlobParser(model=ChatOpenAI(model=\"gpt-4o\", max_tokens=1024)),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[5].page_content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Working with Files\n", + "\n", + "Many document loaders involve parsing files. The difference between such loaders usually stems from how the file is parsed, rather than how the file is loaded. For example, you can use `open` to read the binary content of either a PDF or a markdown file, but you need different parsing logic to convert that binary data into text.\n", + "\n", + "As a result, it can be helpful to decouple the parsing logic from the loading logic, which makes it easier to re-use a given parser regardless of how the data was loaded.\n", + "You can use this strategy to analyze different files, with the same parsing parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2025-01-21T08:05:08.330141Z", + "start_time": "2025-01-21T08:05:07.997956Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LayoutParser: A Unified Toolkit for Deep\n", + "Learning Based Document Image Analysis\n", + "Zejiang Shen1 (\u0000 ), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\n", + "Lee4, Jacob Carlson3, and Weining Li5\n", + "1 Allen Institute for AI\n", + "shannons@allenai.org\n", + "2 Brown University\n", + "ruochen zhang@brown.edu\n", + "3 Harvard University\n", + "{melissadell,jacob carlson}@fas.harvard.edu\n", + "4 University of Washington\n", + "bcgl@cs.washington.edu\n", + "5 University of Waterloo\n", + "w422li@uwaterloo.ca\n", + "Abstract. Recent advances in document image analysis (DIA) have been\n", + "primarily driven by the application of neural networks. Ideally, research\n", + "outcomes could be easily deployed in production and extended for further\n", + "investigation. However, various factors like loosely organized codebases\n", + "and sophisticated model configurations complicate the easy reuse of im-\n", + "portant innovations by a wide audience. Though there have been on-going\n", + "efforts to improve reusability and simplify deep learning (DL) model\n", + "development in disciplines like natural language processing and computer\n", + "vision, none of them are optimized for challenges in the domain of DIA.\n", + "This represents a major gap in the existing toolkit, as DIA is central to\n", + "academic research across a wide range of disciplines in the social sciences\n", + "and humanities. This paper introduces LayoutParser, an open-source\n", + "library for streamlining the usage of DL in DIA research and applica-\n", + "tions. The core LayoutParser library comes with a set of simple and\n", + "intuitive interfaces for applying and customizing DL models for layout de-\n", + "tection, character recognition, and many other document processing tasks.\n", + "To promote extensibility, LayoutParser also incorporates a community\n", + "platform for sharing both pre-trained models and full document digiti-\n", + "zation pipelines. We demonstrate that LayoutParser is helpful for both\n", + "lightweight and large-scale digitization pipelines in real-word use cases.\n", + "The library is publicly available at https://layout-parser.github.io.\n", + "Keywords: Document Image Analysis · Deep Learning · Layout Analysis\n", + "· Character Recognition · Open Source library · Toolkit.\n", + "1 Introduction\n", + "Deep Learning(DL)-based approaches are the state-of-the-art for a wide range of\n", + "document image analysis (DIA) tasks including document image classification [11,\n", + "arXiv:2103.15348v2 [cs.CV] 21 Jun 2021\n", + "{'producer': 'pdfTeX-1.40.21',\n", + " 'creator': 'LaTeX with hyperref',\n", + " 'creationdate': '2021-06-22T01:27:10+00:00',\n", + " 'author': '',\n", + " 'keywords': '',\n", + " 'moddate': '2021-06-22T01:27:10+00:00',\n", + " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n", + " '2020) kpathsea version 6.3.2',\n", + " 'subject': '',\n", + " 'title': '',\n", + " 'trapped': '/False',\n", + " 'source': 'example_data/layout-parser-paper.pdf',\n", + " 'total_pages': 16,\n", + " 'page': 0,\n", + " 'page_label': '1'}\n" + ] + } + ], + "source": [ + "from langchain_community.document_loaders import FileSystemBlobLoader\n", + "from langchain_community.document_loaders.generic import GenericLoader\n", + "from langchain_community.document_loaders.parsers import PyPDFParser\n", + "\n", + "loader = GenericLoader(\n", + " blob_loader=FileSystemBlobLoader(\n", + " path=\"./example_data/\",\n", + " glob=\"*.pdf\",\n", + " ),\n", + " blob_parser=PyPDFParser(),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[0].page_content)\n", + "pprint.pp(docs[0].metadata)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "It is possible to work with files from cloud storage." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import CloudBlobLoader\n", + "from langchain_community.document_loaders.generic import GenericLoader\n", + "\n", + "loader = GenericLoader(\n", + " blob_loader=CloudBlobLoader(\n", + " url=\"s3://mybucket\", # Supports s3://, az://, gs://, file:// schemes.\n", + " glob=\"*.pdf\",\n", + " ),\n", + " blob_parser=PyPDFParser(),\n", + ")\n", + "docs = loader.load()\n", + "print(docs[0].page_content)\n", + "pprint.pp(docs[0].metadata)" ] }, { diff --git a/libs/community/langchain_community/document_loaders/parsers/pdf.py b/libs/community/langchain_community/document_loaders/parsers/pdf.py index eb65b73f1dc..e6a6338f83d 100644 --- a/libs/community/langchain_community/document_loaders/parsers/pdf.py +++ b/libs/community/langchain_community/document_loaders/parsers/pdf.py @@ -18,6 +18,7 @@ from typing import ( Optional, Sequence, Union, + cast, ) from urllib.parse import urlparse @@ -240,86 +241,226 @@ def _merge_text_and_extras(extras: list[str], text_from_page: str) -> str: class PyPDFParser(BaseBlobParser): - """Load `PDF` using `pypdf`""" + """Parse a blob from a PDF using `pypdf` library. + + This class provides methods to parse a blob from a PDF document, supporting various + configurations such as handling password-protected PDFs, extracting images. + It integrates the 'pypdf' library for PDF processing and offers synchronous blob + parsing. + + Examples: + Setup: + + .. code-block:: bash + + pip install -U langchain-community pypdf + + Load a blob from a PDF file: + + .. code-block:: python + + from langchain_core.documents.base import Blob + + blob = Blob.from_path("./example_data/layout-parser-paper.pdf") + + Instantiate the parser: + + .. code-block:: python + + from langchain_community.document_loaders.parsers import PyPDFParser + + parser = PyPDFParser( + # password = None, + mode = "single", + pages_delimiter = "\n\f", + # extract_images = True, + # images_parser = TesseractBlobParser(), + ) + + Lazily parse the blob: + + .. code-block:: python + + docs = [] + docs_lazy = parser.lazy_parse(blob) + + for doc in docs_lazy: + docs.append(doc) + print(docs[0].page_content[:100]) + print(docs[0].metadata) + """ def __init__( self, password: Optional[Union[str, bytes]] = None, extract_images: bool = False, *, - extraction_mode: str = "plain", + mode: Literal["single", "page"] = "page", + pages_delimiter: str = _DEFAULT_PAGES_DELIMITER, + images_parser: Optional[BaseImageBlobParser] = None, + images_inner_format: Literal["text", "markdown-img", "html-img"] = "text", + extraction_mode: Literal["plain", "layout"] = "plain", extraction_kwargs: Optional[dict[str, Any]] = None, ): - self.password = password + """Initialize a parser based on PyPDF. + + Args: + password: Optional password for opening encrypted PDFs. + extract_images: Whether to extract images from the PDF. + mode: The extraction mode, either "single" for the entire document or "page" + for page-wise extraction. + pages_delimiter: A string delimiter to separate pages in single-mode + extraction. + images_parser: Optional image blob parser. + images_inner_format: The format for the parsed output. + - "text" = return the content as is + - "markdown-img" = wrap the content into an image markdown link, w/ link + pointing to (`![body)(#)`] + - "html-img" = wrap the content as the `alt` text of an tag and link to + (`{body}`) + extraction_mode: “plain” for legacy functionality, “layout” extract text + in a fixed width format that closely adheres to the rendered layout in + the source pdf. + extraction_kwargs: Optional additional parameters for the extraction + process. + + Raises: + ValueError: If the `mode` is not "single" or "page". + """ + super().__init__() + if mode not in ["single", "page"]: + raise ValueError("mode must be single or page") self.extract_images = extract_images + if extract_images and not images_parser: + images_parser = RapidOCRBlobParser() + self.images_parser = images_parser + self.images_inner_format = images_inner_format + self.password = password + self.mode = mode + self.pages_delimiter = pages_delimiter self.extraction_mode = extraction_mode self.extraction_kwargs = extraction_kwargs or {} def lazy_parse(self, blob: Blob) -> Iterator[Document]: # type: ignore[valid-type] - """Lazily parse the blob.""" + """ + Lazily parse the blob. + Insert image, if possible, between two paragraphs. + In this way, a paragraph can be continued on the next page. + + Args: + blob: The blob to parse. + + Raises: + ImportError: If the `pypdf` package is not found. + + Yield: + An iterator over the parsed documents. + """ try: import pypdf except ImportError: raise ImportError( - "`pypdf` package not found, please install it with " - "`pip install pypdf`" + "pypdf package not found, please install it with `pip install pypdf`" ) def _extract_text_from_page(page: pypdf.PageObject) -> str: - """Extract text from image given the version of pypdf.""" + """ + Extract text from image given the version of pypdf. + + Args: + page: The page object to extract text from. + + Returns: + str: The extracted text. + """ if pypdf.__version__.startswith("3"): return page.extract_text() else: return page.extract_text( - extraction_mode=self.extraction_mode, # type: ignore[arg-type] - **self.extraction_kwargs, # type: ignore[arg-type] + extraction_mode=self.extraction_mode, + **self.extraction_kwargs, ) with blob.as_bytes_io() as pdf_file_obj: # type: ignore[attr-defined] pdf_reader = pypdf.PdfReader(pdf_file_obj, password=self.password) - yield from [ - Document( - page_content=_extract_text_from_page(page=page) - + self._extract_images_from_page(page), - metadata={ - "source": blob.source, - "page": page_number, - "page_label": pdf_reader.page_labels[page_number], - }, - # type: ignore[attr-defined] + doc_metadata = _purge_metadata( + {"producer": "PyPDF", "creator": "PyPDF", "creationdate": ""} + | cast(dict, pdf_reader.metadata or {}) + | { + "source": blob.source, + "total_pages": len(pdf_reader.pages), + } + ) + single_texts = [] + for page_number, page in enumerate(pdf_reader.pages): + text_from_page = _extract_text_from_page(page=page) + images_from_page = self.extract_images_from_page(page) + all_text = _merge_text_and_extras( + [images_from_page], text_from_page + ).strip() + if self.mode == "page": + yield Document( + page_content=all_text, + metadata=_validate_metadata( + doc_metadata + | { + "page": page_number, + "page_label": pdf_reader.page_labels[page_number], + } + ), + ) + else: + single_texts.append(all_text) + if self.mode == "single": + yield Document( + page_content=self.pages_delimiter.join(single_texts), + metadata=_validate_metadata(doc_metadata), ) - for page_number, page in enumerate(pdf_reader.pages) - ] - def _extract_images_from_page(self, page: pypdf.PageObject) -> str: - """Extract images from page and get the text with RapidOCR.""" - if not self.extract_images or "/XObject" not in page["/Resources"].keys(): # type: ignore[attr-defined] + def extract_images_from_page(self, page: pypdf._page.PageObject) -> str: + """Extract images from a PDF page and get the text using images_to_text. + + Args: + page: The page object from which to extract images. + + Returns: + str: The extracted text from the images on the page. + """ + if not self.images_parser: + return "" + from PIL import Image + + if "/XObject" not in cast(dict, page["/Resources"]).keys(): return "" - xObject = page["/Resources"]["/XObject"].get_object() # type: ignore + xObject = page["/Resources"]["/XObject"].get_object() # type: ignore[index] images = [] for obj in xObject: + np_image: Any = None if xObject[obj]["/Subtype"] == "/Image": if xObject[obj]["/Filter"][1:] in _PDF_FILTER_WITHOUT_LOSS: height, width = xObject[obj]["/Height"], xObject[obj]["/Width"] - images.append( - np.frombuffer(xObject[obj].get_data(), dtype=np.uint8).reshape( - height, width, -1 - ) - ) + np_image = np.frombuffer( + xObject[obj].get_data(), dtype=np.uint8 + ).reshape(height, width, -1) elif xObject[obj]["/Filter"][1:] in _PDF_FILTER_WITH_LOSS: - images.append(xObject[obj].get_data()) - elif ( - isinstance(xObject[obj]["/Filter"], list) - and xObject[obj]["/Filter"] - and xObject[obj]["/Filter"][0][1:] in _PDF_FILTER_WITH_LOSS - ): - images.append(xObject[obj].get_data()) + np_image = np.array(Image.open(io.BytesIO(xObject[obj].get_data()))) + else: - warnings.warn("Unknown PDF Filter!") - return extract_from_images_with_rapidocr(images) + logger.warning("Unknown PDF Filter!") + if np_image is not None: + image_bytes = io.BytesIO() + Image.fromarray(np_image).save(image_bytes, format="PNG") + blob = Blob.from_data(image_bytes.getvalue(), mime_type="image/png") + image_text = next(self.images_parser.lazy_parse(blob)).page_content + images.append( + _format_inner_image(blob, image_text, self.images_inner_format) + ) + return _FORMAT_IMAGE_STR.format( + image_text=_JOIN_IMAGES.join(filter(None, images)) + ) class PDFMinerParser(BaseBlobParser): diff --git a/libs/community/langchain_community/document_loaders/pdf.py b/libs/community/langchain_community/document_loaders/pdf.py index af0aa86b4b5..5350411e62c 100644 --- a/libs/community/langchain_community/document_loaders/pdf.py +++ b/libs/community/langchain_community/document_loaders/pdf.py @@ -184,64 +184,56 @@ class OnlinePDFLoader(BasePDFLoader): class PyPDFLoader(BasePDFLoader): - """PyPDFLoader document loader integration + """Load and parse a PDF file using 'pypdf' library. - Setup: - Install ``langchain-community``. + This class provides methods to load and parse PDF documents, supporting various + configurations such as handling password-protected files, extracting images, and + defining extraction mode. It integrates the `pypdf` library for PDF processing and + offers both synchronous and asynchronous document loading. + + Examples: + Setup: .. code-block:: bash - pip install -U langchain-community + pip install -U langchain-community pypdf + + Instantiate the loader: - Instantiate: .. code-block:: python from langchain_community.document_loaders import PyPDFLoader loader = PyPDFLoader( file_path = "./example_data/layout-parser-paper.pdf", - password = "my-password", - extract_images = True, # headers = None - # extraction_mode = "plain", - # extraction_kwargs = None, + # password = None, + mode = "single", + pages_delimiter = "\n\f", + # extract_images = True, + # images_parser = RapidOCRBlobParser(), ) - Lazy load: + Lazy load documents: + .. code-block:: python docs = [] docs_lazy = loader.lazy_load() - # async variant: - # docs_lazy = await loader.alazy_load() - for doc in docs_lazy: docs.append(doc) print(docs[0].page_content[:100]) print(docs[0].metadata) - .. code-block:: python + Load documents asynchronously: - LayoutParser : A Unified Toolkit for Deep - Learning Based Document Image Analysis - Zejiang Shen1( ), R - {'source': './example_data/layout-parser-paper.pdf', 'page': 0} - - Async load: .. code-block:: python docs = await loader.aload() print(docs[0].page_content[:100]) print(docs[0].metadata) - - .. code-block:: python - - LayoutParser : A Unified Toolkit for Deep - Learning Based Document Image Analysis - Zejiang Shen1( ), R - {'source': './example_data/layout-parser-paper.pdf', 'page': 0} - """ # noqa: E501 + """ def __init__( self, @@ -250,20 +242,50 @@ class PyPDFLoader(BasePDFLoader): headers: Optional[dict] = None, extract_images: bool = False, *, - extraction_mode: str = "plain", + mode: Literal["single", "page"] = "page", + images_parser: Optional[BaseImageBlobParser] = None, + images_inner_format: Literal["text", "markdown-img", "html-img"] = "text", + pages_delimiter: str = _DEFAULT_PAGES_DELIMITER, + extraction_mode: Literal["plain", "layout"] = "plain", extraction_kwargs: Optional[dict] = None, ) -> None: - """Initialize with a file path.""" - try: - import pypdf # noqa:F401 - except ImportError: - raise ImportError( - "pypdf package not found, please install it with `pip install pypdf`" - ) + """Initialize with a file path. + + Args: + file_path: The path to the PDF file to be loaded. + headers: Optional headers to use for GET request to download a file from a + web path. + password: Optional password for opening encrypted PDFs. + mode: The extraction mode, either "single" for the entire document or "page" + for page-wise extraction. + pages_delimiter: A string delimiter to separate pages in single-mode + extraction. + extract_images: Whether to extract images from the PDF. + images_parser: Optional image blob parser. + images_inner_format: The format for the parsed output. + - "text" = return the content as is + - "markdown-img" = wrap the content into an image markdown link, w/ link + pointing to (`![body)(#)`] + - "html-img" = wrap the content as the `alt` text of an tag and link to + (`{body}`) + extraction_mode: “plain” for legacy functionality, “layout” extract text + in a fixed width format that closely adheres to the rendered layout in + the source pdf + extraction_kwargs: Optional additional parameters for the extraction + process. + + Returns: + This method does not directly return data. Use the `load`, `lazy_load` or + `aload` methods to retrieve parsed documents with content and metadata. + """ super().__init__(file_path, headers=headers) self.parser = PyPDFParser( password=password, + mode=mode, extract_images=extract_images, + images_parser=images_parser, + images_inner_format=images_inner_format, + pages_delimiter=pages_delimiter, extraction_mode=extraction_mode, extraction_kwargs=extraction_kwargs, ) @@ -271,12 +293,18 @@ class PyPDFLoader(BasePDFLoader): def lazy_load( self, ) -> Iterator[Document]: - """Lazy load given path as pages.""" + """ + Lazy load given path as pages. + Insert image, if possible, between two paragraphs. + In this way, a paragraph can be continued on the next page. + """ if self.web_path: - blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path) # type: ignore[attr-defined] + blob = Blob.from_data( # type: ignore[attr-defined] + open(self.file_path, "rb").read(), path=self.web_path + ) else: blob = Blob.from_path(self.file_path) # type: ignore[attr-defined] - yield from self.parser.parse(blob) + yield from self.parser.lazy_parse(blob) class PyPDFium2Loader(BasePDFLoader): @@ -305,9 +333,56 @@ class PyPDFium2Loader(BasePDFLoader): class PyPDFDirectoryLoader(BaseLoader): - """Load a directory with `PDF` files using `pypdf` and chunks at character level. + """Load and parse a directory of PDF files using 'pypdf' library. - Loader also stores page numbers in metadata. + This class provides methods to load and parse multiple PDF documents in a directory, + supporting options for recursive search, handling password-protected files, + extracting images, and defining extraction modes. It integrates the `pypdf` library + for PDF processing and offers synchronous document loading. + + Examples: + Setup: + + .. code-block:: bash + + pip install -U langchain-community pypdf + + Instantiate the loader: + + .. code-block:: python + + from langchain_community.document_loaders import PyPDFDirectoryLoader + + loader = PyPDFDirectoryLoader( + path = "./example_data/", + glob = "**/[!.]*.pdf", + silent_errors = False, + load_hidden = False, + recursive = False, + extract_images = False, + password = None, + mode = "page", + images_to_text = None, + headers = None, + extraction_mode = "plain", + # extraction_kwargs = None, + ) + + Load documents: + + .. code-block:: python + + docs = loader.load() + print(docs[0].page_content[:100]) + print(docs[0].metadata) + + Load documents asynchronously: + + .. code-block:: python + + docs = await loader.aload() + print(docs[0].page_content[:100]) + print(docs[0].metadata) """ def __init__( @@ -318,16 +393,53 @@ class PyPDFDirectoryLoader(BaseLoader): load_hidden: bool = False, recursive: bool = False, extract_images: bool = False, + *, + password: Optional[str] = None, + mode: Literal["single", "page"] = "page", + images_parser: Optional[BaseImageBlobParser] = None, + headers: Optional[dict] = None, + extraction_mode: Literal["plain", "layout"] = "plain", + extraction_kwargs: Optional[dict] = None, ): + """Initialize with a directory path. + + Args: + path: The path to the directory containing PDF files to be loaded. + glob: The glob pattern to match files in the directory. + silent_errors: Whether to log errors instead of raising them. + load_hidden: Whether to include hidden files in the search. + recursive: Whether to search subdirectories recursively. + extract_images: Whether to extract images from PDFs. + password: Optional password for opening encrypted PDFs. + mode: The extraction mode, either "single" for extracting the entire + document or "page" for page-wise extraction. + images_parser: Optional image blob parser.. + headers: Optional headers to use for GET request to download a file from a + web path. + extraction_mode: “plain” for legacy functionality, “layout” for + experimental layout mode functionality + extraction_kwargs: Optional additional parameters for the extraction + process. + + Returns: + This method does not directly return data. Use the `load` method to + retrieve parsed documents with content and metadata. + """ + self.password = password + self.mode = mode self.path = path self.glob = glob self.load_hidden = load_hidden self.recursive = recursive self.silent_errors = silent_errors self.extract_images = extract_images + self.images_parser = images_parser + self.headers = headers + self.extraction_mode = extraction_mode + self.extraction_kwargs = extraction_kwargs @staticmethod - def _is_visible(path: Path) -> bool: + def _is_visible(path: PurePath) -> bool: return not any(part.startswith(".") for part in path.parts) def load(self) -> list[Document]: @@ -338,7 +450,16 @@ class PyPDFDirectoryLoader(BaseLoader): if i.is_file(): if self._is_visible(i.relative_to(p)) or self.load_hidden: try: - loader = PyPDFLoader(str(i), extract_images=self.extract_images) + loader = PyPDFLoader( + str(i), + password=self.password, + mode=self.mode, + extract_images=self.extract_images, + images_parser=self.images_parser, + headers=self.headers, + extraction_mode=self.extraction_mode, + extraction_kwargs=self.extraction_kwargs, + ) sub_docs = loader.load() for doc in sub_docs: doc.metadata["source"] = str(i) diff --git a/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py b/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py index 44cc8294643..5595ec5de97 100644 --- a/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py +++ b/libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py @@ -14,7 +14,6 @@ from langchain_community.document_loaders.parsers import ( PDFMinerParser, PDFPlumberParser, PyPDFium2Parser, - PyPDFParser, ) if TYPE_CHECKING: @@ -98,11 +97,6 @@ def _assert_with_duplicate_parser(parser: BaseBlobParser, dedupe: bool = False) assert "11000000 SSeerriieess" == docs[0].page_content.split("\n")[0] -def test_pypdf_parser() -> None: - """Test PyPDF parser.""" - _assert_with_parser(PyPDFParser()) - - def test_pdfminer_parser() -> None: """Test PDFMiner parser.""" # Does not follow defaults to split by page. @@ -122,11 +116,6 @@ def test_pdfplumber_parser() -> None: _assert_with_duplicate_parser(PDFPlumberParser(dedupe=True), dedupe=True) -def test_extract_images_text_from_pdf_pypdfparser() -> None: - """Test extract image from pdf and recognize text with rapid ocr - PyPDFParser""" - _assert_with_parser(PyPDFParser(extract_images=True)) - - def test_extract_images_text_from_pdf_pdfminerparser() -> None: """Test extract image from pdf and recognize text with rapid ocr - PDFMinerParser""" _assert_with_parser(PDFMinerParser(extract_images=True)) @@ -150,6 +139,8 @@ class EmptyImageBlobParser(BaseImageBlobParser): "parser_factory,params", [ ("PyMuPDFParser", {}), + ("PyPDFParser", {"extraction_mode": "plain"}), + ("PyPDFParser", {"extraction_mode": "layout"}), ], ) @pytest.mark.requires("pillow") @@ -176,6 +167,8 @@ def test_mode_and_extract_images_variations( "parser_factory,params", [ ("PyMuPDFParser", {}), + ("PyPDFParser", {"extraction_mode": "plain"}), + ("PyPDFParser", {"extraction_mode": "layout"}), ], ) @pytest.mark.requires("pillow") diff --git a/libs/community/tests/integration_tests/document_loaders/test_pdf.py b/libs/community/tests/integration_tests/document_loaders/test_pdf.py index 7eae7ef710d..98e25bf8f30 100644 --- a/libs/community/tests/integration_tests/document_loaders/test_pdf.py +++ b/libs/community/tests/integration_tests/document_loaders/test_pdf.py @@ -212,6 +212,7 @@ def test_amazontextract_loader_failures() -> None: "parser_factory,params", [ ("PyMuPDFLoader", {}), + ("PyPDFLoader", {}), ], ) def test_standard_parameters( @@ -229,12 +230,10 @@ def test_standard_parameters( loader = loader_class( file_path, mode="page", - page_delimiter="---", + pages_delimiter="---", images_parser=None, images_inner_format="text", password=None, - extract_tables=None, - extract_tables_settings=None, ) docs = loader.load() assert len(docs) == 16 diff --git a/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py b/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py index 3044305a341..c12d7d55bdf 100644 --- a/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py +++ b/libs/community/tests/unit_tests/document_loaders/parsers/test_pdf_parsers.py @@ -12,7 +12,6 @@ from langchain_community.document_loaders.blob_loaders import Blob from langchain_community.document_loaders.parsers.pdf import ( PDFMinerParser, PyPDFium2Parser, - PyPDFParser, _merge_text_and_extras, ) @@ -76,12 +75,6 @@ def _assert_with_parser(parser: BaseBlobParser, *, splits_by_page: bool = True) assert int(metadata["page"]) == 0 -@pytest.mark.requires("pypdf") -def test_pypdf_parser() -> None: - """Test PyPDF parser.""" - _assert_with_parser(PyPDFParser()) - - @pytest.mark.requires("pdfminer") def test_pdfminer_parser() -> None: """Test PDFMiner parser.""" @@ -100,6 +93,7 @@ def test_pypdfium2_parser() -> None: "parser_factory,require,params", [ ("PyMuPDFParser", "pymupdf", {}), + ("PyPDFParser", "pypdf", {}), ], ) def test_parsers( diff --git a/libs/community/tests/unit_tests/document_loaders/test_pdf.py b/libs/community/tests/unit_tests/document_loaders/test_pdf.py index d3e9e2b586d..fed2c986ed9 100644 --- a/libs/community/tests/unit_tests/document_loaders/test_pdf.py +++ b/libs/community/tests/unit_tests/document_loaders/test_pdf.py @@ -65,7 +65,8 @@ def test_pypdf_loader_with_layout() -> None: expected = path_to_layout_pdf_txt.read_text(encoding="utf-8") cleaned_first_page = re.sub(r"\x00", "", first_page) cleaned_expected = re.sub(r"\x00", "", expected) - assert cleaned_first_page == cleaned_expected + + assert cleaned_first_page == cleaned_expected.strip() @pytest.mark.requires("pypdf")