github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-26 13:01:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix revue

Browse Source
This commit is contained in:
Philippe Prados 2025-03-10 15:04:52 +01:00
parent 951e577214
commit 97996eaff1
5 changed files with 235 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
docs/docs/integrations/document_loaders/zeroxpdfloader.ipynb
View File

@ -43,14 +43,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:24:17.543284Z", "end_time": "2025-03-10T16:08:44.721334Z",
"start_time": "2025-02-10T11:24:17.415355Z" "start_time": "2025-03-10T16:08:44.603495Z"
} }
}, },
"outputs": [],
"source": [ "source": [
"import os\n", "import os\n",
"from getpass import getpass\n", "from getpass import getpass\n",
@ -65,7 +63,9 @@
" os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")\n", " os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")\n",
"\n", "\n",
"nest_asyncio.apply()" "nest_asyncio.apply()"
] ],
"outputs": [],
"execution_count": 1
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@ -77,14 +77,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:24:18.899123Z", "end_time": "2025-03-10T16:08:46.797620Z",
"start_time": "2025-02-10T11:24:18.857755Z" "start_time": "2025-03-10T16:08:46.685305Z"
} }
}, },
"outputs": [],
"source": [ "source": [
"from getpass import getpass\n", "from getpass import getpass\n",
"\n", "\n",
@ -94,7 +92,9 @@
" os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")\n", " os.environ[\"OPENAI_API_KEY\"] = getpass(\"OpenAI API key =\")\n",
"\n", "\n",
"nest_asyncio.apply()" "nest_asyncio.apply()"
] ],
"outputs": [],
"execution_count": 2
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@ -106,18 +106,18 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:24:20.152622Z", "end_time": "2025-03-10T16:08:48.610751Z",
"start_time": "2025-02-10T11:24:20.145640Z" "start_time": "2025-03-10T16:08:48.604079Z"
} }
}, },
"outputs": [],
"source": [ "source": [
"# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n", "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n",
"# os.environ[\"LANGSMITH_TRACING\"] = \"true\"" "# os.environ[\"LANGSMITH_TRACING\"] = \"true\""
] ],
"outputs": [],
"execution_count": 3
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@ -130,13 +130,15 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:24:23.723647Z", "end_time": "2025-03-10T16:30:21.902215Z",
"start_time": "2025-02-10T11:24:21.394870Z" "start_time": "2025-03-10T16:30:19.755014Z"
} }
}, },
"source": [
"%pip install -qU langchain_community py-zerox"
],
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
@ -146,9 +148,7 @@
] ]
} }
], ],
"source": [ "execution_count": 4
"%pip install -qU langchain_community py-zerox"
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@ -160,21 +160,109 @@
] ]
}, },
{ {
"cell_type": "code",
"execution_count": 5,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:24:25.819517Z", "end_time": "2025-03-10T16:30:23.336449Z",
"start_time": "2025-02-10T11:24:25.091340Z" "start_time": "2025-03-10T16:30:23.328747Z"
} }
}, },
"outputs": [], "cell_type": "code",
"source": [ "source": [
"from langchain_community.document_loaders import ZeroxPDFLoader\n", "import asyncio\n",
"import nest_asyncio\n",
"nest_asyncio.apply()"
],
"outputs": [],
"execution_count": 5
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-03-10T16:30:25.533375Z",
"start_time": "2025-03-10T16:30:25.095083Z"
}
},
"cell_type": "code",
"source": [
"def run_in_thread():\n",
" loop = asyncio.get_event_loop() # Créer une nouvelle boucle d'événements pour ce thread\n",
" result = loop.run_until_complete(toto())\n",
" return result\n",
"\n",
"from multiprocessing.pool import ThreadPool\n",
"pool = ThreadPool(processes=1)\n",
"async_result = pool.apply_async(run_in_thread) # tuple of args for foo\n",
"result=async_result.get()\n",
"print(result)\n"
],
"outputs": [
{
"ename": "NameError",
"evalue": "name 'toto' is not defined",
"output_type": "error",
"traceback": [
"\u001B[31m---------------------------------------------------------------------------\u001B[39m",
"\u001B[31mNameError\u001B[39m Traceback (most recent call last)",
"\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[6]\u001B[39m\u001B[32m, line 9\u001B[39m\n\u001B[32m 7\u001B[39m pool = ThreadPool(processes=\u001B[32m1\u001B[39m)\n\u001B[32m 8\u001B[39m async_result = pool.apply_async(run_in_thread) \u001B[38;5;66;03m# tuple of args for foo\u001B[39;00m\n\u001B[32m----> \u001B[39m\u001B[32m9\u001B[39m result=\u001B[43masync_result\u001B[49m\u001B[43m.\u001B[49m\u001B[43mget\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m 10\u001B[39m \u001B[38;5;28mprint\u001B[39m(result)\n",
"\u001B[36mFile \u001B[39m\u001B[32m~/miniconda3/lib/python3.12/multiprocessing/pool.py:774\u001B[39m, in \u001B[36mApplyResult.get\u001B[39m\u001B[34m(self, timeout)\u001B[39m\n\u001B[32m 772\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m._value\n\u001B[32m 773\u001B[39m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[32m--> \u001B[39m\u001B[32m774\u001B[39m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;28mself\u001B[39m._value\n",
"\u001B[36mFile \u001B[39m\u001B[32m~/miniconda3/lib/python3.12/multiprocessing/pool.py:125\u001B[39m, in \u001B[36mworker\u001B[39m\u001B[34m(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception)\u001B[39m\n\u001B[32m 123\u001B[39m job, i, func, args, kwds = task\n\u001B[32m 124\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m--> \u001B[39m\u001B[32m125\u001B[39m result = (\u001B[38;5;28;01mTrue\u001B[39;00m, \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwds\u001B[49m\u001B[43m)\u001B[49m)\n\u001B[32m 126\u001B[39m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[32m 127\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m wrap_exception \u001B[38;5;129;01mand\u001B[39;00m func \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m _helper_reraises_exception:\n",
"\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[6]\u001B[39m\u001B[32m, line 3\u001B[39m, in \u001B[36mrun_in_thread\u001B[39m\u001B[34m()\u001B[39m\n\u001B[32m 1\u001B[39m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34mrun_in_thread\u001B[39m():\n\u001B[32m 2\u001B[39m loop = asyncio.get_event_loop() \u001B[38;5;66;03m# Créer une nouvelle boucle d'événements pour ce thread\u001B[39;00m\n\u001B[32m----> \u001B[39m\u001B[32m3\u001B[39m result = loop.run_until_complete(\u001B[43mtoto\u001B[49m())\n\u001B[32m 4\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m result\n",
"\u001B[31mNameError\u001B[39m: name 'toto' is not defined"
]
}
],
"execution_count": 6
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-03-10T16:05:43.143734Z",
"start_time": "2025-03-10T16:05:43.111814Z"
}
},
"cell_type": "code",
"source": [
"loop = asyncio.get_running_loop()\n",
"\n",
"def _run_in_thread(loop):\n",
" loop = asyncio.get_event_loop() # Créer une nouvelle boucle d'événements pour ce thread\n",
" result = loop.run_until_complete(toto())\n",
" return result\n",
"\n",
"from multiprocessing.pool import ThreadPool\n",
"pool = ThreadPool(processes=1)\n",
"async_result = pool.apply_async(_run_in_thread,(loop,)) # tuple of args for foo\n",
"result = async_result.get()\n",
"print(result)"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"toto\n",
"hello\n"
]
}
],
"execution_count": 26
},
{
"cell_type": "code",
"metadata": {
"ExecuteTime": {
"end_time": "2025-03-10T16:30:30.217668Z",
"start_time": "2025-03-10T16:30:29.274838Z"
}
},
"source": [
"from langchain_community.document_loaders.pdf import ZeroxPDFLoader\n",
"\n", "\n",
"file_path = \"./example_data/layout-parser-paper.pdf\"\n", "file_path = \"./example_data/layout-parser-paper.pdf\"\n",
"loader = ZeroxPDFLoader(file_path)" "loader = ZeroxPDFLoader(file_path)"
] ],
"outputs": [],
"execution_count": 7
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@ -185,39 +273,58 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 6,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:27:17.781571Z", "end_time": "2025-03-10T16:31:07.654904Z",
"start_time": "2025-02-10T11:24:27.817480Z" "start_time": "2025-03-10T16:30:33.071884Z"
} }
}, },
"source": [
"docs = loader.load()\n",
"docs[0]"
],
"outputs": [ "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/pprados/workspace.bda/langchain/libs/community/.venv/lib/python3.12/site-packages/pyzerox/models/modellitellm.py:52: UserWarning: \n",
" Custom system prompt was provided which overrides the default system prompt. We assume that you know what you are doing. \n",
" . Default prompt for zerox is:\n",
" \n",
" Convert the following PDF page to markdown.\n",
" Return only the markdown with no explanation text.\n",
" Do not exclude any content from the page.\n",
" \n",
" warnings.warn(f\"{Messages.CUSTOM_SYSTEM_PROMPT_WARNING}. Default prompt for zerox is:\\n {DEFAULT_SYSTEM_PROMPT}\")\n"
]
},
{ {
"data": { "data": {
"text/plain": [ "text/plain": [
"Document(metadata={'producer': 'pdfTeX-1.40.21', 'creator': 'LaTeX with hyperref', 'creationdate': '2021-06-22T01:27:10+00:00', 'author': '', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'subject': '', 'title': '', 'trapped': 'False', 'total_pages': 16, 'source': './example_data/layout-parser-paper.pdf', 'num_pages': 16, 'page': 0}, page_content='# LayoutParser: A Unified Toolkit for Deep Learning Based Document Image Analysis\\n\\nZejian Shen¹, Ruochen Zhang², Melissa Dell³, Benjamin Charles Germain Lee⁴, Jacob Carlson³, and Weining Li⁵\\n\\n1 Allen Institute for AI \\nshannon@allenai.org \\n2 Brown University \\nruochen_zhang@brown.edu \\n3 Harvard University \\n{melissade11, jacob.carlson}@fas.harvard.edu \\n4 University of Washington \\nbgclgs.washington.edu \\n5 University of Waterloo \\nw422ii@uwaterloo.ca \\n\\n## Abstract\\n\\nRecent advances in document image analysis (DIA) have been primarily driven by the application of neural networks. Ideally, research outcomes could be easily deployed in production and extended for further investigation. However, various factors like loosely organized codebases and sophisticated model configurations complicate the easy reuse of important innovations by a wide audience. Though there have been on-going efforts to improve reusability and simplify deep learning (DL) model development in disciplines like natural language processing and computer vision, none of them are optimized for challenges in the domain of DIA. This represents a major gap in the existing toolkit, as DIA is central to academic research across a wide range of disciplines in the social sciences and humanities. This paper introduces LayoutParser, an open-source library for streamlining the usage of DL in DIA research and applications. The core LayoutParser library comes with a set of simple and intuitive interfaces for applying and customizing DL models for layout detection, character recognition, and many other document processing tasks. To promote extensibility, LayoutParser also incorporates a community platform for sharing both pre-trained models and full document digitization pipelines. We demonstrate that LayoutParser is helpful for both lightweight and large-scale digitization pipelines in real-world use cases. The library is publicly available at https://layout-parser.github.io\\n\\n**Keywords:** Document Image Analysis · Deep Learning · Layout Analysis · Character Recognition · Open Source library · Toolkit.\\n\\n## 1 Introduction\\n\\nDeep Learning (DL)-based approaches are the state-of-the-art for a wide range of document image analysis (DIA) tasks including document image classification [11]')" "Document(metadata={'producer': 'pdfTeX-1.40.21', 'creator': 'LaTeX with hyperref', 'creationdate': '2021-06-22T01:27:10+00:00', 'author': '', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'subject': '', 'title': '', 'trapped': 'False', 'total_pages': 16, 'source': './example_data/layout-parser-paper.pdf', 'num_pages': 16, 'page': 0}, page_content='# LayoutParser: A Unified Toolkit for Deep Learning Based Document Image Analysis\\n\\nZejian Shen¹, Ruochen Zhang², Melissa Dell³, Benjamin Charles Germain Lee⁴, Jacob Carlson³, and Weining Li⁵\\n\\n¹ Allen Institute for AI \\nshannons@allenai.org \\n² Brown University \\nruochen_zhang@brown.edu \\n³ Harvard University \\n{melissadell, jacob.carlson}@fas.harvard.edu \\n⁴ University of Washington \\nbgcl@cs.washington.edu \\n⁵ University of Waterloo \\nw4221@uwaterloo.ca \\n\\n**Abstract.** Recent advances in document image analysis (DIA) have been primarily driven by the application of neural networks. Ideally, research outcomes could be easily deployed in production and extended for further investigation. However, various factors like loosely organized codebases and sophisticated model configurations complicate the easy reuse of important innovations by a wide audience. Though there have been on-going efforts to improve reusability and simplify deep learning (DL) model development in disciplines like natural language processing and computer vision, none of them are optimized for challenges in the domain of DIA. This represents a major gap in the existing toolkit, as DIA is central to academic research across a wide range of disciplines in the social sciences and humanities. This paper introduces LayoutParser, an open-source library for streamlining the usage of DL in DIA research and applications. The core LayoutParser library comes with a set of simple and intuitive interfaces for applying and customizing DL models for layout detection, character recognition, and many other document processing tasks. To promote extensibility, LayoutParser also incorporates a community platform for sharing both pre-trained models and full document digitization pipelines. We demonstrate that LayoutParser is helpful for both lightweight and large-scale digitization pipelines in real-world use cases. The library is publicly available at https://layout-parser.github.io \\n\\n**Keywords:** Document Image Analysis · Deep Learning · Layout Analysis · Character Recognition · Open Source library · Toolkit. \\n\\n# 1 Introduction\\n\\nDeep Learning (DL)-based approaches are the state-of-the-art for a wide range of document image analysis (DIA) tasks including document image classification [11].')"
] ]
}, },
"execution_count": 6, "execution_count": 8,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
], ],
"source": [ "execution_count": 8
"docs = loader.load()\n",
"docs[0]"
]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 7,
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:27:51.893033Z", "end_time": "2025-03-10T16:31:58.192569Z",
"start_time": "2025-02-10T11:27:51.889072Z" "start_time": "2025-03-10T16:31:58.182464Z"
} }
}, },
"source": [
"import pprint\n",
"\n",
"pprint.pp(docs[0].metadata)"
],
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
@ -241,11 +348,7 @@
] ]
} }
], ],
"source": [ "execution_count": 9
"import pprint\n",
"\n",
"pprint.pp(docs[0].metadata)"
]
}, },
{ {
"cell_type": "markdown", "cell_type": "markdown",
@ -256,25 +359,14 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8,
"metadata": { "metadata": {
"jupyter": {
"is_executing": true
},
"ExecuteTime": { "ExecuteTime": {
"end_time": "2025-02-10T11:28:44.103420Z", "start_time": "2025-03-10T16:33:11.308283Z"
"start_time": "2025-02-10T11:28:05.933389Z"
} }
}, },
"outputs": [
{
"data": {
"text/plain": [
"6"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"pages = []\n", "pages = []\n",
"for doc in loader.lazy_load():\n", "for doc in loader.lazy_load():\n",
@ -285,7 +377,25 @@
"\n", "\n",
" pages = []\n", " pages = []\n",
"len(pages)" "len(pages)"
] ],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/pprados/workspace.bda/langchain/libs/community/.venv/lib/python3.12/site-packages/pyzerox/models/modellitellm.py:52: UserWarning: \n",
" Custom system prompt was provided which overrides the default system prompt. We assume that you know what you are doing. \n",
" . Default prompt for zerox is:\n",
" \n",
" Convert the following PDF page to markdown.\n",
" Return only the markdown with no explanation text.\n",
" Do not exclude any content from the page.\n",
" \n",
" warnings.warn(f\"{Messages.CUSTOM_SYSTEM_PROMPT_WARNING}. Default prompt for zerox is:\\n {DEFAULT_SYSTEM_PROMPT}\")\n"
]
}
],
"execution_count": null
}, },
{ {
"cell_type": "code", "cell_type": "code",

3
libs/community/langchain_community/document_loaders/parsers/__init__.py
View File

@ -32,7 +32,6 @@ if TYPE_CHECKING:
PyMuPDFParser, PyMuPDFParser,
PyPDFium2Parser, PyPDFium2Parser,
PyPDFParser, PyPDFParser,
ZeroxPDFParser,
) )
from langchain_community.document_loaders.parsers.vsdx import ( from langchain_community.document_loaders.parsers.vsdx import (
VsdxParser, VsdxParser,
@ -56,7 +55,6 @@ _module_lookup = {
"RapidOCRBlobParser": "langchain_community.document_loaders.parsers.images", "RapidOCRBlobParser": "langchain_community.document_loaders.parsers.images",
"TesseractBlobParser": "langchain_community.document_loaders.parsers.images", "TesseractBlobParser": "langchain_community.document_loaders.parsers.images",
"VsdxParser": "langchain_community.document_loaders.parsers.vsdx", "VsdxParser": "langchain_community.document_loaders.parsers.vsdx",
"ZeroxPDFParser": "langchain_community.document_loaders.parsers.pdf",
} }
@ -84,5 +82,4 @@ __all__ = [
"RapidOCRBlobParser", "RapidOCRBlobParser",
"TesseractBlobParser", "TesseractBlobParser",
"VsdxParser", "VsdxParser",
"ZeroxPDFParser",
] ]

36
libs/community/langchain_community/document_loaders/parsers/pdf.py
View File

@ -8,6 +8,7 @@ import io
import logging import logging
import threading import threading
import warnings import warnings
from asyncio import AbstractEventLoop
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory from tempfile import NamedTemporaryFile, TemporaryDirectory
@ -1542,6 +1543,12 @@ class ZeroxPDFParser(BaseBlobParser):
"Do not exclude any content from the page. " "Do not exclude any content from the page. "
) )
@staticmethod
def _run_async_from_thread(coro, loop):
future = asyncio.run_coroutine_threadsafe(coro,
loop) # Lancer la coroutine dans la boucle existante
return future.result() # Bloque en attendant le résultat
def __init__( def __init__(
self, self,
mode: Literal["single", "page"] = "page", mode: Literal["single", "page"] = "page",
@ -1659,14 +1666,7 @@ class ZeroxPDFParser(BaseBlobParser):
) )
temp_file = None temp_file = None
try: try:
if not ZeroxPDFParser._is_valid_url(str(blob.path)): file_path = str(blob.path)
temp_file = NamedTemporaryFile()
with open(temp_file.name, "wb") as f:
f.write(blob.as_bytes())
file_path = temp_file.name
else:
file_path = str(blob.path)
with blob.as_bytes_io() as pdf_file_obj: with blob.as_bytes_io() as pdf_file_obj:
doc_metadata = _purge_metadata(self._get_metadata(pdf_file_obj)) doc_metadata = _purge_metadata(self._get_metadata(pdf_file_obj))
@ -1686,8 +1686,11 @@ class ZeroxPDFParser(BaseBlobParser):
zerox_prompt = PromptTemplate.from_template( zerox_prompt = PromptTemplate.from_template(
self.custom_system_prompt self.custom_system_prompt
).format(prompt_tables=prompt_tables, prompt_images=prompt_images) ).format(prompt_tables=prompt_tables, prompt_images=prompt_images)
zerox_output = asyncio.run( # async def toto():
zerox( # await asyncio.sleep(0)
# return "hello"
# coro=toto()
coro=zerox(
file_path=str(file_path), file_path=str(file_path),
model=self.model, model=self.model,
cleanup=self.cleanup, cleanup=self.cleanup,
@ -1697,7 +1700,16 @@ class ZeroxPDFParser(BaseBlobParser):
select_pages=self.select_pages, select_pages=self.select_pages,
**self.zerox_kwargs, **self.zerox_kwargs,
) )
) try:
loop = asyncio.get_running_loop()
from multiprocessing.pool import ThreadPool
pool = ThreadPool(processes=1)
zerox_output = pool.apply_async(
lambda : loop.run_until_complete(coro)).get() # tuple of args for foo
except RuntimeError:
zerox_output = asyncio.run(coro)
# Convert zerox output to Document instances and yield them # Convert zerox output to Document instances and yield them
if len(zerox_output.pages) > 0: if len(zerox_output.pages) > 0:
@ -1716,7 +1728,7 @@ class ZeroxPDFParser(BaseBlobParser):
single_texts = [] single_texts = []
for page in zerox_output.pages: for page in zerox_output.pages:
text_from_page = page.content text_from_page = page.content
images_from_page = "" # FIXME images_from_page = ""
all_text = _merge_text_and_extras( all_text = _merge_text_and_extras(
[images_from_page], text_from_page [images_from_page], text_from_page
) )

69
libs/community/tests/integration_tests/document_loaders/parsers/test_pdf_parsers.py
View File

@ -2,37 +2,48 @@
import re import re
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Iterator
import pytest import pytest
from typing import TYPE_CHECKING, Iterator
import langchain_community.document_loaders.parsers as pdf_parsers from langchain_community.document_loaders import PDFMinerLoader, PDFPlumberLoader, \
PyMuPDFLoader, PyPDFium2Loader, PyPDFLoader
from langchain_community.document_loaders.base import BaseBlobParser from langchain_community.document_loaders.base import BaseBlobParser
from langchain_community.document_loaders.blob_loaders import Blob from langchain_community.document_loaders.blob_loaders import Blob
from langchain_community.document_loaders.parsers import ( from langchain_community.document_loaders.parsers import (
BaseImageBlobParser, BaseImageBlobParser,
PDFPlumberParser, PDFPlumberParser,
) )
from langchain_community.document_loaders.parsers.pdf import ZeroxPDFParser, \
PyMuPDFParser, PDFMinerParser, PyPDFium2Parser, PyPDFParser
if TYPE_CHECKING: if TYPE_CHECKING:
from PIL.Image import Image from PIL.Image import Image
_map_parser = {
'PDFMinerParser': PDFMinerParser,
'PDFPlumberParser': PDFPlumberParser,
'PyMuPDFParser': PyMuPDFParser,
'PyPDFium2Parser': PyPDFium2Parser,
'PyPDFParser': PyPDFParser,
'ZeroxPDFParser': ZeroxPDFParser,
}
# PDFs to test parsers on. # PDFs to test parsers on.
HELLO_PDF = Path(__file__).parent.parent.parent / "examples" / "hello.pdf" HELLO_PDF = Path(__file__).parent.parent.parent / "examples" / "hello.pdf"
LAYOUT_PARSER_PAPER_PDF = ( LAYOUT_PARSER_PAPER_PDF = (
Path(__file__).parent.parent.parent / "examples" / "layout-parser-paper.pdf" Path(__file__).parent.parent.parent / "examples" / "layout-parser-paper.pdf"
) )
LAYOUT_PARSER_PAPER_PASSWORD_PDF = ( LAYOUT_PARSER_PAPER_PASSWORD_PDF = (
Path(__file__).parent.parent.parent Path(__file__).parent.parent.parent
/ "examples" / "examples"
/ "layout-parser-paper-password.pdf" / "layout-parser-paper-password.pdf"
) )
DUPLICATE_CHARS = ( DUPLICATE_CHARS = (
Path(__file__).parent.parent.parent / "examples" / "duplicate-chars.pdf" Path(__file__).parent.parent.parent / "examples" / "duplicate-chars.pdf"
) )
@ -124,16 +135,16 @@ class EmptyImageBlobParser(BaseImageBlobParser):
) )
@pytest.mark.requires("pillow") @pytest.mark.requires("pillow")
def test_mode_and_extract_images_variations( def test_mode_and_extract_images_variations(
parser_factory: str, parser_factory: str,
params: dict, params: dict,
mode: str, mode: str,
image_parser: BaseImageBlobParser, image_parser: BaseImageBlobParser,
) -> None: ) -> None:
if parser_factory == "ZeroxPDFParser": if parser_factory == "ZeroxPDFParser":
try: try:
import pyzerox # noqa: F401 import pyzerox # noqa: F401
except ImportError: except ImportError:
pytest.skip("pyzerox is valid only with Python +3.11") pytest.skip("py-zerox is valid only with Python +3.11")
_test_matrix( _test_matrix(
parser_factory, parser_factory,
params, params,
@ -160,15 +171,15 @@ def test_mode_and_extract_images_variations(
) )
@pytest.mark.requires("pillow") @pytest.mark.requires("pillow")
def test_mode_and_image_formats_variations( def test_mode_and_image_formats_variations(
parser_factory: str, parser_factory: str,
params: dict, params: dict,
images_inner_format: str, images_inner_format: str,
) -> None: ) -> None:
if parser_factory == "ZeroxPDFParser": if parser_factory == "ZeroxPDFParser":
try: try:
import pyzerox # noqa: F401 import pyzerox # noqa: F401
except ImportError: except ImportError:
pytest.skip("pyzerox is valid only with Python +3.11") pytest.skip("py-zerox is valid only with Python +3.11")
mode = "single" mode = "single"
image_parser = EmptyImageBlobParser() image_parser = EmptyImageBlobParser()
@ -182,11 +193,11 @@ def test_mode_and_image_formats_variations(
def _test_matrix( def _test_matrix(
parser_factory: str, parser_factory: str,
params: dict, params: dict,
mode: str, mode: str,
image_parser: BaseImageBlobParser, image_parser: BaseImageBlobParser,
images_inner_format: str, images_inner_format: str,
) -> None: ) -> None:
"""Apply the same test for all *standard* PDF parsers. """Apply the same test for all *standard* PDF parsers.
@ -234,7 +245,7 @@ def _test_matrix(
assert len(docs) assert len(docs)
parser.password = old_password parser.password = old_password
parser_class = getattr(pdf_parsers, parser_factory) parser_class = _map_parser[parser_factory]
parser = parser_class( parser = parser_class(
mode=mode, mode=mode,
@ -258,20 +269,20 @@ def _test_matrix(
"parser_factory,params", "parser_factory,params",
[ [
("PyMuPDFParser", {}), ("PyMuPDFParser", {}),
("ZeroxPDFParser", {}), ("ZeroxPDFParser", {"model": "gpt-4o-mini"}),
], ],
) )
def test_parser_with_table( def test_parser_with_table(
parser_factory: str, parser_factory: str,
params: dict, params: dict,
mode: str, mode: str,
extract_tables: str, extract_tables: str,
) -> None: ) -> None:
if parser_factory == "ZeroxPDFParser": if parser_factory == "ZeroxPDFParser":
try: try:
import pyzerox # noqa: F401 import pyzerox # noqa: F401
except ImportError: except ImportError:
pytest.skip("pyzerox is valid only with Python +3.11") pytest.skip("py-zerox is valid only with Python +3.11")
from PIL.Image import Image from PIL.Image import Image
@ -322,7 +333,7 @@ def test_parser_with_table(
def _analyze_image(self, img: Image) -> str: def _analyze_image(self, img: Image) -> str:
return "![image](.)" return "![image](.)"
parser_class = getattr(pdf_parsers, parser_factory) parser_class = _map_parser[parser_factory]
parser = parser_class( parser = parser_class(
mode=mode, mode=mode,

1
libs/community/tests/unit_tests/document_loaders/parsers/test_public_api.py
View File

@ -20,5 +20,4 @@ def test_parsers_public_api_correct() -> None:
"RapidOCRBlobParser", "RapidOCRBlobParser",
"TesseractBlobParser", "TesseractBlobParser",
"VsdxParser", "VsdxParser",
"ZeroxPDFParser",
} }