From 6b506dfe3825ec01e712d387647c7db0822b20ae Mon Sep 17 00:00:00 2001 From: Philippe Prados Date: Fri, 4 Apr 2025 12:11:29 +0200 Subject: [PATCH] Fix notebooks --- .../document_loaders/pdfminer.ipynb | 119 +++++++++--------- .../document_loaders/pdfplumber.ipynb | 2 - 2 files changed, 60 insertions(+), 61 deletions(-) diff --git a/docs/docs/integrations/document_loaders/pdfminer.ipynb b/docs/docs/integrations/document_loaders/pdfminer.ipynb index 307f6feffe9..ea674eb50d7 100644 --- a/docs/docs/integrations/document_loaders/pdfminer.ipynb +++ b/docs/docs/integrations/document_loaders/pdfminer.ipynb @@ -41,18 +41,18 @@ }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-01-21T08:43:51.976447Z", - "start_time": "2025-01-21T08:43:51.974157Z" + "end_time": "2025-04-03T14:49:37.091916Z", + "start_time": "2025-04-03T14:49:37.088609Z" } }, - "outputs": [], "source": [ "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n", "# os.environ[\"LANGSMITH_TRACING\"] = \"true\"" - ] + ], + "outputs": [], + "execution_count": 1 }, { "cell_type": "markdown", @@ -65,25 +65,26 @@ }, { "cell_type": "code", - "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2025-01-21T08:43:54.801251Z", - "start_time": "2025-01-21T08:43:53.303500Z" + "end_time": "2025-04-03T14:49:37.300206Z", + "start_time": "2025-04-03T14:49:37.142212Z" } }, + "source": [ + "%pip install -qU langchain_community pdfminer.six" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "/home/pprados/workspace.bda/langchain/libs/community/.venv/bin/python: No module named pip\r\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], - "source": [ - "%pip install -qU langchain_community pdfminer.six" - ] + "execution_count": 2 }, { "cell_type": "markdown", @@ -96,20 +97,20 @@ }, { "cell_type": "code", - "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2025-01-21T08:43:56.963855Z", - "start_time": "2025-01-21T08:43:56.313777Z" + "end_time": "2025-04-03T14:49:43.057906Z", + "start_time": "2025-04-03T14:49:41.859991Z" } }, - "outputs": [], "source": [ "from langchain_community.document_loaders import PDFMinerLoader\n", "\n", "file_path = \"./example_data/layout-parser-paper.pdf\"\n", "loader = PDFMinerLoader(file_path)" - ] + ], + "outputs": [], + "execution_count": 3 }, { "cell_type": "markdown", @@ -120,65 +121,65 @@ }, { "cell_type": "code", - "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2025-01-21T08:43:59.203805Z", - "start_time": "2025-01-21T08:43:58.598938Z" + "end_time": "2025-04-03T14:49:45.592807Z", + "start_time": "2025-04-03T14:49:45.278864Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "Document(metadata={'author': '', 'creationdate': '2021-06-22T01:27:10+00:00', 'creator': 'LaTeX with hyperref', 'keywords': '', 'moddate': '2021-06-22T01:27:10+00:00', 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020) kpathsea version 6.3.2', 'producer': 'pdfTeX-1.40.21', 'subject': '', 'title': '', 'trapped': 'False', 'total_pages': 16, 'source': './example_data/layout-parser-paper.pdf'}, page_content='1\\n2\\n0\\n2\\n\\nn\\nu\\nJ\\n\\n1\\n2\\n\\n]\\n\\nV\\nC\\n.\\ns\\nc\\n[\\n\\n2\\nv\\n8\\n4\\n3\\n5\\n1\\n.\\n3\\n0\\n1\\n2\\n:\\nv\\ni\\nX\\nr\\na\\n\\nLayoutParser: A Unified Toolkit for Deep\\nLearning Based Document Image Analysis\\n\\nZejiang Shen1 ((cid:0)), Ruochen Zhang2, Melissa Dell3, Benjamin Charles Germain\\nLee4, Jacob Carlson3, and Weining Li5\\n\\n1 Allen Institute for AI\\nshannons@allenai.org\\n2 Brown University\\nruochen zhang@brown.edu\\n3 Harvard University\\n{melissadell,jacob carlson}@fas.harvard.edu\\n4 University of Washington\\nbcgl@cs.washington.edu\\n5 University of Waterloo\\nw422li@uwaterloo.ca\\n\\nAbstract. Recent advances in document image analysis (DIA) have been\\nprimarily driven by the application of neural networks. Ideally, research\\noutcomes could be easily deployed in production and extended for further\\ninvestigation. However, various factors like loosely organized codebases\\nand sophisticated model configurations complicate the easy reuse of im-\\nportant innovations by a wide audience. Though there have been on-going\\nefforts to improve reusability and simplify deep learning (DL) model\\ndevelopment in disciplines like natural language processing and computer\\nvision, none of them are optimized for challenges in the domain of DIA.\\nThis represents a major gap in the existing toolkit, as DIA is central to\\nacademic research across a wide range of disciplines in the social sciences\\nand humanities. This paper introduces LayoutParser, an open-source\\nlibrary for streamlining the usage of DL in DIA research and applica-\\ntions. The core LayoutParser library comes with a set of simple and\\nintuitive interfaces for applying and customizing DL models for layout de-\\ntection, character recognition, and many other document processing tasks.\\nTo promote extensibility, LayoutParser also incorporates a community\\nplatform for sharing both pre-trained models and full document digiti-\\nzation pipelines. We demonstrate that LayoutParser is helpful for both\\nlightweight and large-scale digitization pipelines in real-word use cases.\\nThe library is publicly available at https://layout-parser.github.io.\\n\\nKeywords: Document Image Analysis · Deep Learning · Layout Analysis\\n· Character Recognition · Open Source library · Toolkit.\\n\\n1\\n\\nIntroduction\\n\\nDeep Learning(DL)-based approaches are the state-of-the-art for a wide range of\\ndocument image analysis (DIA) tasks including document image classification [11,\\n\\x0c2\\n\\nZ. Shen et al.\\n\\n37], layout detection [38, 22], table detection [26], and scene text detection [4].\\nA generalized learning-based framework dramatically reduces the need for the\\nmanual specification of complicated rules, which is the status quo with traditional\\nmethods. DL has the potential to transform DIA pipelines and benefit a broad\\nspectrum of large-scale document digitization projects.\\n\\nHowever, there are several practical difficulties for taking advantages of re-\\ncent advances in DL-based methods: 1) DL models are notoriously convoluted\\nfor reuse and extension. Existing models are developed using distinct frame-\\nworks like TensorFlow [1] or PyTorch [24], and the high-level parameters can\\nbe obfuscated by implementation details [8]. It can be a time-consuming and\\nfrustrating experience to debug, reproduce, and adapt existing models for DIA,\\nand many researchers who would benefit the most from using these methods lack\\nthe technical background to implement them from scratch. 2) Document images\\ncontain diverse and disparate patterns across domains, and customized training\\nis often required to achieve a desirable detection accuracy. Currently there is no\\nfull-fledged infrastructure for easily curating the target document image datasets\\nand fine-tuning or re-training the models. 3) DIA usually requires a sequence of\\nmodels and other processing to obtain the final outputs. Often research teams use\\nDL models and then perform further document analyses in separate processes,\\nand these pipelines are not documented in any central location (and often not\\ndocumented at all). This makes it difficult for research teams to learn about how\\nfull pipelines are implemented and leads them to invest significant resources in\\nreinventing the DIA wheel.\\n\\nLayoutParser provides a unified toolkit to support DL-based document image\\nanalysis and processing. To address the aforementioned challenges, LayoutParser\\nis built with the following components:\\n\\n1. An off-the-shelf toolkit for applying DL models for layout detection, character\\n\\nrecognition, and other DIA tasks (Section 3)\\n\\n2. A rich repository of pre-trained neural network models (Model Zoo) that\\n\\nunderlies the off-the-shelf usage\\n\\n3. Comprehensive tools for efficient document image data annotation and model\\n\\ntuning to support different levels of customization\\n\\n4. A DL model hub and community platform for the easy sharing, distribu-\\ntion, and discussion of DIA models and pipelines, to promote reusability,\\nreproducibility, and extensibility (Section 4)\\n\\nThe library implements simple and intuitive Python APIs without sacrificing\\ngeneralizability and versatility, and can be easily installed via pip. Its convenient\\nfunctions for handling document image data can be seamlessly integrated with\\nexisting DIA pipelines. With detailed documentations and carefully curated\\ntutorials, we hope this tool will benefit a variety of end-users, and will lead to\\nadvances in applications in both industry and academic research.\\n\\nLayoutParser is well aligned with recent efforts for improving DL model\\nreusability in other disciplines like natural language processing [8, 34] and com-\\nputer vision [35], but with a focus on unique challenges in DIA. We show\\nLayoutParser can be applied in sophisticated and large-scale digitization projects\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n3\\n\\nthat require precision, efficiency, and robustness, as well as simple and light-\\nweight document processing tasks focusing on efficacy and flexibility (Section 5).\\nLayoutParser is being actively maintained, and support for more deep learning\\nmodels and novel methods in text-based layout analysis methods [37, 34] is\\nplanned.\\n\\nThe rest of the paper is organized as follows. Section 2 provides an overview\\nof related work. The core LayoutParser library, DL Model Zoo, and customized\\nmodel training are described in Section 3, and the DL model hub and commu-\\nnity platform are detailed in Section 4. Section 5 shows two examples of how\\nLayoutParser can be used in practical DIA projects, and Section 6 concludes.\\n\\n2 Related Work\\n\\nRecently, various DL models and datasets have been developed for layout analysis\\ntasks. The dhSegment [22] utilizes fully convolutional networks [20] for segmen-\\ntation tasks on historical documents. Object detection-based methods like Faster\\nR-CNN [28] and Mask R-CNN [12] are used for identifying document elements [38]\\nand detecting tables [30, 26]. Most recently, Graph Neural Networks [29] have also\\nbeen used in table detection [27]. However, these models are usually implemented\\nindividually and there is no unified framework to load and use such models.\\n\\nThere has been a surge of interest in creating open-source tools for document\\nimage processing: a search of document image analysis in Github leads to 5M\\nrelevant code pieces 6; yet most of them rely on traditional rule-based methods\\nor provide limited functionalities. The closest prior research to our work is the\\nOCR-D project7, which also tries to build a complete toolkit for DIA. However,\\nsimilar to the platform developed by Neudecker et al. [21], it is designed for\\nanalyzing historical documents, and provides no supports for recent DL models.\\nThe DocumentLayoutAnalysis project8 focuses on processing born-digital PDF\\ndocuments via analyzing the stored PDF data. Repositories like DeepLayout9\\nand Detectron2-PubLayNet10 are individual deep learning models trained on\\nlayout analysis datasets without support for the full DIA pipeline. The Document\\nAnalysis and Exploitation (DAE) platform [15] and the DeepDIVA project [2]\\naim to improve the reproducibility of DIA methods (or DL models), yet they\\nare not actively maintained. OCR engines like Tesseract [14], easyOCR11 and\\npaddleOCR12 usually do not come with comprehensive functionalities for other\\nDIA tasks like layout analysis.\\n\\nRecent years have also seen numerous efforts to create libraries for promoting\\nreproducibility and reusability in the field of DL. Libraries like Dectectron2 [35],\\n\\n6 The number shown is obtained by specifying the search type as ‘code’.\\n7 https://ocr-d.de/en/about\\n8 https://github.com/BobLd/DocumentLayoutAnalysis\\n9 https://github.com/leonlulu/DeepLayout\\n10 https://github.com/hpanwar08/detectron2\\n11 https://github.com/JaidedAI/EasyOCR\\n12 https://github.com/PaddlePaddle/PaddleOCR\\n\\x0c4\\n\\nZ. Shen et al.\\n\\nFig. 1: The overall architecture of LayoutParser. For an input document image,\\nthe core LayoutParser library provides a set of off-the-shelf tools for layout\\ndetection, OCR, visualization, and storage, backed by a carefully designed layout\\ndata structure. LayoutParser also supports high level customization via efficient\\nlayout annotation and model training functions. These improve model accuracy\\non the target samples. The community platform enables the easy sharing of DIA\\nmodels and whole digitization pipelines to promote reusability and reproducibility.\\nA collection of detailed documentation, tutorials and exemplar projects make\\nLayoutParser easy to learn and use.\\n\\nAllenNLP [8] and transformers [34] have provided the community with complete\\nDL-based support for developing and deploying models for general computer\\nvision and natural language processing problems. LayoutParser, on the other\\nhand, specializes specifically in DIA tasks. LayoutParser is also equipped with a\\ncommunity platform inspired by established model hubs such as Torch Hub [23]\\nand TensorFlow Hub [1]. It enables the sharing of pretrained models as well as\\nfull document processing pipelines that are unique to DIA tasks.\\n\\nThere have been a variety of document data collections to facilitate the\\ndevelopment of DL models. Some examples include PRImA [3](magazine layouts),\\nPubLayNet [38](academic paper layouts), Table Bank [18](tables in academic\\npapers), Newspaper Navigator Dataset [16, 17](newspaper figure layouts) and\\nHJDataset [31](historical Japanese document layouts). A spectrum of models\\ntrained on these datasets are currently available in the LayoutParser model zoo\\nto support different use cases.\\n\\n3 The Core LayoutParser Library\\n\\nAt the core of LayoutParser is an off-the-shelf toolkit that streamlines DL-\\nbased document image analysis. Five components support a simple interface\\nwith comprehensive functionalities: 1) The layout detection models enable using\\npre-trained or self-trained DL models for layout detection with just four lines\\nof code. 2) The detected layout information is stored in carefully engineered\\n\\nEfficient Data AnnotationCustomized Model TrainingModel CustomizationDIA Model HubDIA Pipeline SharingCommunity PlatformLayout Detection ModelsDocument Images The Core LayoutParser LibraryOCR ModuleStorage & VisualizationLayout Data Structure\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n5\\n\\nTable 1: Current layout detection models in the LayoutParser model zoo\\n\\nDataset\\n\\nBase Model1 Large Model Notes\\n\\nPubLayNet [38]\\nPRImA [3]\\nNewspaper [17]\\nTableBank [18]\\nHJDataset [31]\\n\\nF / M\\nM\\nF\\nF\\nF / M\\n\\nM\\n-\\n-\\nF\\n-\\n\\nLayouts of modern scientific documents\\nLayouts of scanned modern magazines and scientific reports\\nLayouts of scanned US newspapers from the 20th century\\nTable region on modern scientific and business document\\nLayouts of history Japanese documents\\n\\n1 For each dataset, we train several models of different sizes for different needs (the trade-off between accuracy\\nvs. computational cost). For “base model” and “large model”, we refer to using the ResNet 50 or ResNet 101\\nbackbones [13], respectively. One can train models of different architectures, like Faster R-CNN [28] (F) and Mask\\nR-CNN [12] (M). For example, an F in the Large Model column indicates it has a Faster R-CNN model trained\\nusing the ResNet 101 backbone. The platform is maintained and a number of additions will be made to the model\\nzoo in coming months.\\n\\nlayout data structures, which are optimized for efficiency and versatility. 3) When\\nnecessary, users can employ existing or customized OCR models via the unified\\nAPI provided in the OCR module. 4) LayoutParser comes with a set of utility\\nfunctions for the visualization and storage of the layout data. 5) LayoutParser\\nis also highly customizable, via its integration with functions for layout data\\nannotation and model training. We now provide detailed descriptions for each\\ncomponent.\\n\\n3.1 Layout Detection Models\\n\\nIn LayoutParser, a layout model takes a document image as an input and\\ngenerates a list of rectangular boxes for the target content regions. Different\\nfrom traditional methods, it relies on deep convolutional neural networks rather\\nthan manually curated rules to identify content regions. It is formulated as an\\nobject detection problem and state-of-the-art models like Faster R-CNN [28] and\\nMask R-CNN [12] are used. This yields prediction results of high accuracy and\\nmakes it possible to build a concise, generalized interface for layout detection.\\nLayoutParser, built upon Detectron2 [35], provides a minimal API that can\\nperform layout detection with only four lines of code in Python:\\n\\n1 import layoutparser as lp\\n2 image = cv2 . imread ( \" image_file \" ) # load images\\n3 model = lp . De t e c tro n2 Lay outM odel (\\n\\n\" lp :// PubLayNet / f as t er _ r c nn _ R _ 50 _ F P N_ 3 x / config \" )\\n\\n4\\n5 layout = model . detect ( image )\\n\\nLayoutParser provides a wealth of pre-trained model weights using various\\ndatasets covering different languages, time periods, and document types. Due to\\ndomain shift [7], the prediction performance can notably drop when models are ap-\\nplied to target samples that are significantly different from the training dataset. As\\ndocument structures and layouts vary greatly in different domains, it is important\\nto select models trained on a dataset similar to the test samples. A semantic syntax\\nis used for initializing the model weights in LayoutParser, using both the dataset\\nname and model name lp:///.\\n\\x0c6\\n\\nZ. Shen et al.\\n\\nFig. 2: The relationship between the three types of layout data structures.\\nCoordinate supports three kinds of variation; TextBlock consists of the co-\\nordinate information and extra features like block text, types, and reading orders;\\na Layout object is a list of all possible layout elements, including other Layout\\nobjects. They all support the same set of transformation and operation APIs for\\nmaximum flexibility.\\n\\nShown in Table 1, LayoutParser currently hosts 9 pre-trained models trained\\non 5 different datasets. Description of the training dataset is provided alongside\\nwith the trained models such that users can quickly identify the most suitable\\nmodels for their tasks. Additionally, when such a model is not readily available,\\nLayoutParser also supports training customized layout models and community\\nsharing of the models (detailed in Section 3.5).\\n\\n3.2 Layout Data Structures\\n\\nA critical feature of LayoutParser is the implementation of a series of data\\nstructures and operations that can be used to efficiently process and manipulate\\nthe layout elements. In document image analysis pipelines, various post-processing\\non the layout analysis model outputs is usually required to obtain the final\\noutputs. Traditionally, this requires exporting DL model outputs and then loading\\nthe results into other pipelines. All model outputs from LayoutParser will be\\nstored in carefully engineered data types optimized for further processing, which\\nmakes it possible to build an end-to-end document digitization pipeline within\\nLayoutParser. There are three key components in the data structure, namely\\nthe Coordinate system, the TextBlock, and the Layout. They provide different\\nlevels of abstraction for the layout data, and a set of APIs are supported for\\ntransformations or operations on these classes.\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n7\\n\\nCoordinates are the cornerstones for storing layout information. Currently,\\nthree types of Coordinate data structures are provided in LayoutParser, shown\\nin Figure 2. Interval and Rectangle are the most common data types and\\nsupport specifying 1D or 2D regions within a document. They are parameterized\\nwith 2 and 4 parameters. A Quadrilateral class is also implemented to support\\na more generalized representation of rectangular regions when the document\\nis skewed or distorted, where the 4 corner points can be specified and a total\\nof 8 degrees of freedom are supported. A wide collection of transformations\\nlike shift, pad, and scale, and operations like intersect, union, and is_in,\\nare supported for these classes. Notably, it is common to separate a segment\\nof the image and analyze it individually. LayoutParser provides full support\\nfor this scenario via image cropping operations crop_image and coordinate\\ntransformations like relative_to and condition_on that transform coordinates\\nto and from their relative representations. We refer readers to Table 2 for a more\\ndetailed description of these operations13.\\n\\nBased on Coordinates, we implement the TextBlock class that stores both\\nthe positional and extra features of individual layout elements. It also supports\\nspecifying the reading orders via setting the parent field to the index of the parent\\nobject. A Layout class is built that takes in a list of TextBlocks and supports\\nprocessing the elements in batch. Layout can also be nested to support hierarchical\\nlayout structures. They support the same operations and transformations as the\\nCoordinate classes, minimizing both learning and deployment effort.\\n\\n3.3 OCR\\n\\nLayoutParser provides a unified interface for existing OCR tools. Though there\\nare many OCR tools available, they are usually configured differently with distinct\\nAPIs or protocols for using them. It can be inefficient to add new OCR tools into\\nan existing pipeline, and difficult to make direct comparisons among the available\\ntools to find the best option for a particular project. To this end, LayoutParser\\nbuilds a series of wrappers among existing OCR engines, and provides nearly\\nthe same syntax for using them. It supports a plug-and-play style of using OCR\\nengines, making it effortless to switch, evaluate, and compare different OCR\\nmodules:\\n\\n1 ocr_agent = lp . TesseractAgent ()\\n2 # Can be easily switched to other OCR software\\n3 tokens = ocr_agent . detect ( image )\\n\\nThe OCR outputs will also be stored in the aforementioned layout data\\nstructures and can be seamlessly incorporated into the digitization pipeline.\\nCurrently LayoutParser supports the Tesseract and Google Cloud Vision OCR\\nengines.\\n\\nLayoutParser also comes with a DL-based CNN-RNN OCR model [6] trained\\nwith the Connectionist Temporal Classification (CTC) loss [10]. It can be used\\nlike the other OCR modules, and can be easily trained on customized datasets.\\n\\n13 This is also available in the LayoutParser documentation pages.\\n\\x0c8\\n\\nZ. Shen et al.\\n\\nTable 2: All operations supported by the layout elements. The same APIs are\\nsupported across different layout element classes including Coordinate types,\\nTextBlock and Layout.\\n\\nOperation Name\\n\\nDescription\\n\\nblock.pad(top, bottom, right, left) Enlarge the current block according to the input\\n\\nblock.scale(fx, fy)\\n\\nblock.shift(dx, dy)\\n\\nScale the current block given the ratio\\nin x and y direction\\n\\nMove the current block with the shift\\ndistances in x and y direction\\n\\nblock1.is in(block2)\\n\\nWhether block1 is inside of block2\\n\\nblock1.intersect(block2)\\n\\nblock1.union(block2)\\n\\nblock1.relative to(block2)\\n\\nblock1.condition on(block2)\\n\\nReturn the intersection region of block1 and block2.\\nCoordinate type to be determined based on the inputs.\\n\\nReturn the union region of block1 and block2.\\nCoordinate type to be determined based on the inputs.\\n\\nConvert the absolute coordinates of block1 to\\nrelative coordinates to block2\\n\\nCalculate the absolute coordinates of block1 given\\nthe canvas block2’s absolute coordinates\\n\\nblock.crop image(image)\\n\\nObtain the image segments in the block region\\n\\n3.4 Storage and visualization\\n\\nThe end goal of DIA is to transform the image-based document data into a\\nstructured database. LayoutParser supports exporting layout data into different\\nformats like JSON, csv, and will add the support for the METS/ALTO XML\\nformat 14 . It can also load datasets from layout analysis-specific formats like\\nCOCO [38] and the Page Format [25] for training layout models (Section 3.5).\\nVisualization of the layout detection results is critical for both presentation\\nand debugging. LayoutParser is built with an integrated API for displaying the\\nlayout information along with the original document image. Shown in Figure 3, it\\nenables presenting layout data with rich meta information and features in different\\nmodes. More detailed information can be found in the online LayoutParser\\ndocumentation page.\\n\\n3.5 Customized Model Training\\n\\nBesides the off-the-shelf library, LayoutParser is also highly customizable with\\nsupports for highly unique and challenging document analysis tasks. Target\\ndocument images can be vastly different from the existing datasets for train-\\ning layout models, which leads to low layout detection accuracy. Training data\\n\\n14 https://altoxml.github.io\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n9\\n\\nFig. 3: Layout detection and OCR results visualization generated by the\\nLayoutParser APIs. Mode I directly overlays the layout region bounding boxes\\nand categories over the original image. Mode II recreates the original document\\nvia drawing the OCR’d texts at their corresponding positions on the image\\ncanvas. In this figure, tokens in textual regions are filtered using the API and\\nthen displayed.\\n\\ncan also be highly sensitive and not sharable publicly. To overcome these chal-\\nlenges, LayoutParser is built with rich features for efficient data annotation and\\ncustomized model training.\\n\\nLayoutParser incorporates a toolkit optimized for annotating document lay-\\nouts using object-level active learning [32]. With the help from a layout detection\\nmodel trained along with labeling, only the most important layout objects within\\neach image, rather than the whole image, are required for labeling. The rest of\\nthe regions are automatically annotated with high confidence predictions from\\nthe layout detection model. This allows a layout dataset to be created more\\nefficiently with only around 60% of the labeling budget.\\n\\nAfter the training dataset is curated, LayoutParser supports different modes\\nfor training the layout models. Fine-tuning can be used for training models on a\\nsmall newly-labeled dataset by initializing the model with existing pre-trained\\nweights. Training from scratch can be helpful when the source dataset and\\ntarget are significantly different and a large training set is available. However, as\\nsuggested in Studer et al.’s work[33], loading pre-trained weights on large-scale\\ndatasets like ImageNet [5], even from totally different domains, can still boost\\nmodel performance. Through the integrated API provided by LayoutParser,\\nusers can easily compare model performances on the benchmark datasets.\\n\\x0c10\\n\\nZ. Shen et al.\\n\\nFig. 4: Illustration of (a) the original historical Japanese document with layout\\ndetection results and (b) a recreated version of the document image that achieves\\nmuch better character recognition recall. The reorganization algorithm rearranges\\nthe tokens based on the their detected bounding boxes given a maximum allowed\\nheight.\\n\\n4 LayoutParser Community Platform\\n\\nAnother focus of LayoutParser is promoting the reusability of layout detection\\nmodels and full digitization pipelines. Similar to many existing deep learning\\nlibraries, LayoutParser comes with a community model hub for distributing\\nlayout models. End-users can upload their self-trained models to the model hub,\\nand these models can be loaded into a similar interface as the currently available\\nLayoutParser pre-trained models. For example, the model trained on the News\\nNavigator dataset [17] has been incorporated in the model hub.\\n\\nBeyond DL models, LayoutParser also promotes the sharing of entire doc-\\nument digitization pipelines. For example, sometimes the pipeline requires the\\ncombination of multiple DL models to achieve better accuracy. Currently, pipelines\\nare mainly described in academic papers and implementations are often not pub-\\nlicly available. To this end, the LayoutParser community platform also enables\\nthe sharing of layout pipelines to promote the discussion and reuse of techniques.\\nFor each shared pipeline, it has a dedicated project page, with links to the source\\ncode, documentation, and an outline of the approaches. A discussion panel is\\nprovided for exchanging ideas. Combined with the core LayoutParser library,\\nusers can easily build reusable components based on the shared pipelines and\\napply them to solve their unique problems.\\n\\n5 Use Cases\\n\\nThe core objective of LayoutParser is to make it easier to create both large-scale\\nand light-weight document digitization pipelines. Large-scale document processing\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n11\\n\\nfocuses on precision, efficiency, and robustness. The target documents may have\\ncomplicated structures, and may require training multiple layout detection models\\nto achieve the optimal accuracy. Light-weight pipelines are built for relatively\\nsimple documents, with an emphasis on development ease, speed and flexibility.\\nIdeally one only needs to use existing resources, and model training should be\\navoided. Through two exemplar projects, we show how practitioners in both\\nacademia and industry can easily build such pipelines using LayoutParser and\\nextract high-quality structured document data for their downstream tasks. The\\nsource code for these projects will be publicly available in the LayoutParser\\ncommunity hub.\\n\\n5.1 A Comprehensive Historical Document Digitization Pipeline\\n\\nThe digitization of historical documents can unlock valuable data that can shed\\nlight on many important social, economic, and historical questions. Yet due to\\nscan noises, page wearing, and the prevalence of complicated layout structures, ob-\\ntaining a structured representation of historical document scans is often extremely\\ncomplicated.\\nIn this example, LayoutParser was\\nused to develop a comprehensive\\npipeline, shown in Figure 5, to gener-\\nate high-quality structured data from\\nhistorical Japanese firm financial ta-\\nbles with complicated layouts. The\\npipeline applies two layout models to\\nidentify different levels of document\\nstructures and two customized OCR\\nengines for optimized character recog-\\nnition accuracy.\\n\\nAs shown in Figure 4 (a), the\\ndocument contains columns of text\\nwritten vertically 15, a common style\\nin Japanese. Due to scanning noise\\nand archaic printing technology, the\\ncolumns can be skewed or have vari-\\nable widths, and hence cannot be eas-\\nily identified via rule-based methods.\\nWithin each column, words are sepa-\\nrated by white spaces of variable size,\\nand the vertical positions of objects\\ncan be an indicator of their layout\\ntype.\\n\\nFig. 5: Illustration of how LayoutParser\\nhelps with the historical document digi-\\ntization pipeline.\\n\\n15 A document page consists of eight rows like this. For simplicity we skip the row\\n\\nsegmentation discussion and refer readers to the source code when available.\\n\\x0c12\\n\\nZ. Shen et al.\\n\\nTo decipher the complicated layout\\n\\nstructure, two object detection models have been trained to recognize individual\\ncolumns and tokens, respectively. A small training set (400 images with approxi-\\nmately 100 annotations each) is curated via the active learning based annotation\\ntool [32] in LayoutParser. The models learn to identify both the categories and\\nregions for each token or column via their distinct visual features. The layout\\ndata structure enables easy grouping of the tokens within each column, and\\nrearranging columns to achieve the correct reading orders based on the horizontal\\nposition. Errors are identified and rectified via checking the consistency of the\\nmodel predictions. Therefore, though trained on a small dataset, the pipeline\\nachieves a high level of layout detection accuracy: it achieves a 96.97 AP [19]\\nscore across 5 categories for the column detection model, and a 89.23 AP across\\n4 categories for the token detection model.\\n\\nA combination of character recognition methods is developed to tackle the\\nunique challenges in this document. In our experiments, we found that irregular\\nspacing between the tokens led to a low character recognition recall rate, whereas\\nexisting OCR models tend to perform better on densely-arranged texts. To\\novercome this challenge, we create a document reorganization algorithm that\\nrearranges the text based on the token bounding boxes detected in the layout\\nanalysis step. Figure 4 (b) illustrates the generated image of dense text, which is\\nsent to the OCR APIs as a whole to reduce the transaction costs. The flexible\\ncoordinate system in LayoutParser is used to transform the OCR results relative\\nto their original positions on the page.\\n\\nAdditionally, it is common for historical documents to use unique fonts\\nwith different glyphs, which significantly degrades the accuracy of OCR models\\ntrained on modern texts. In this document, a special flat font is used for printing\\nnumbers and could not be detected by off-the-shelf OCR engines. Using the highly\\nflexible functionalities from LayoutParser, a pipeline approach is constructed\\nthat achieves a high recognition accuracy with minimal effort. As the characters\\nhave unique visual structures and are usually clustered together, we train the\\nlayout model to identify number regions with a dedicated category. Subsequently,\\nLayoutParser crops images within these regions, and identifies characters within\\nthem using a self-trained OCR model based on a CNN-RNN [6]. The model\\ndetects a total of 15 possible categories, and achieves a 0.98 Jaccard score16 and\\na 0.17 average Levinstein distances17 for token prediction on the test set.\\n\\nOverall, it is possible to create an intricate and highly accurate digitization\\npipeline for large-scale digitization using LayoutParser. The pipeline avoids\\nspecifying the complicated rules used in traditional methods, is straightforward\\nto develop, and is robust to outliers. The DL models also generate fine-grained\\nresults that enable creative approaches like page reorganization for OCR.\\n\\n16 This measures the overlap between the detected and ground-truth characters, and\\n\\nthe maximum is 1.\\n\\n17 This measures the number of edits from the ground-truth text to the predicted text,\\n\\nand lower is better.\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n13\\n\\nFig. 6: This lightweight table detector can identify tables (outlined in red) and\\ncells (shaded in blue) in different locations on a page. In very few cases (d), it\\nmight generate minor error predictions, e.g, failing to capture the top text line of\\na table.\\n\\n5.2 A light-weight Visual Table Extractor\\n\\nDetecting tables and parsing their structures (table extraction) are of central im-\\nportance for many document digitization tasks. Many previous works [26, 30, 27]\\nand tools 18 have been developed to identify and parse table structures. Yet they\\nmight require training complicated models from scratch, or are only applicable\\nfor born-digital PDF documents. In this section, we show how LayoutParser can\\nhelp build a light-weight accurate visual table extractor for legal docket tables\\nusing the existing resources with minimal effort.\\n\\nThe extractor uses a pre-trained layout detection model for identifying the\\ntable regions and some simple rules for pairing the rows and the columns in the\\nPDF image. Mask R-CNN [12] trained on the PubLayNet dataset [38] from the\\nLayoutParser Model Zoo can be used for detecting table regions. By filtering\\nout model predictions of low confidence and removing overlapping predictions,\\nLayoutParser can identify the tabular regions on each page, which significantly\\nsimplifies the subsequent steps. By applying the line detection functions within\\nthe tabular segments, provided in the utility module from LayoutParser, the\\npipeline can identify the three distinct columns in the tables. A row clustering\\nmethod is then applied via analyzing the y coordinates of token bounding boxes in\\nthe left-most column, which are obtained from the OCR engines. A non-maximal\\nsuppression algorithm is used to remove duplicated rows with extremely small\\ngaps. Shown in Figure 6, the built pipeline can detect tables at different positions\\non a page accurately. Continued tables from different pages are concatenated,\\nand a structured table representation has been easily created.\\n\\n18 https://github.com/atlanhq/camelot, https://github.com/tabulapdf/tabula\\n\\x0c14\\n\\nZ. Shen et al.\\n\\n6 Conclusion\\n\\nLayoutParser provides a comprehensive toolkit for deep learning-based document\\nimage analysis. The off-the-shelf library is easy to install, and can be used to\\nbuild flexible and accurate pipelines for processing documents with complicated\\nstructures. It also supports high-level customization and enables easy labeling and\\ntraining of DL models on unique document image datasets. The LayoutParser\\ncommunity platform facilitates sharing DL models and DIA pipelines, inviting\\ndiscussion and promoting code reproducibility and reusability. The LayoutParser\\nteam is committed to keeping the library updated continuously and bringing\\nthe most recent advances in DL-based DIA, such as multi-modal document\\nmodeling [37, 36, 9] (an upcoming priority), to a diverse audience of end-users.\\n\\nAcknowledgements We thank the anonymous reviewers for their comments\\nand suggestions. This project is supported in part by NSF Grant OIA-2033558\\nand funding from the Harvard Data Science Initiative and Harvard Catalyst.\\nZejiang Shen thanks Doug Downey for suggestions.\\n\\nReferences\\n\\n[1] Abadi, M., Agarwal, A., Barham, P., Brevdo, E., Chen, Z., Citro, C., Corrado,\\nG.S., Davis, A., Dean, J., Devin, M., Ghemawat, S., Goodfellow, I., Harp, A.,\\nIrving, G., Isard, M., Jia, Y., Jozefowicz, R., Kaiser, L., Kudlur, M., Levenberg,\\nJ., Man´e, D., Monga, R., Moore, S., Murray, D., Olah, C., Schuster, M., Shlens, J.,\\nSteiner, B., Sutskever, I., Talwar, K., Tucker, P., Vanhoucke, V., Vasudevan, V.,\\nVi´egas, F., Vinyals, O., Warden, P., Wattenberg, M., Wicke, M., Yu, Y., Zheng,\\nX.: TensorFlow: Large-scale machine learning on heterogeneous systems (2015),\\nhttps://www.tensorflow.org/, software available from tensorflow.org\\n\\n[2] Alberti, M., Pondenkandath, V., W¨ursch, M., Ingold, R., Liwicki, M.: Deepdiva: a\\nhighly-functional python framework for reproducible experiments. In: 2018 16th\\nInternational Conference on Frontiers in Handwriting Recognition (ICFHR). pp.\\n423–428. IEEE (2018)\\n\\n[3] Antonacopoulos, A., Bridson, D., Papadopoulos, C., Pletschacher, S.: A realistic\\ndataset for performance evaluation of document layout analysis. In: 2009 10th\\nInternational Conference on Document Analysis and Recognition. pp. 296–300.\\nIEEE (2009)\\n\\n[4] Baek, Y., Lee, B., Han, D., Yun, S., Lee, H.: Character region awareness for text\\ndetection. In: Proceedings of the IEEE/CVF Conference on Computer Vision and\\nPattern Recognition. pp. 9365–9374 (2019)\\n\\n[5] Deng, J., Dong, W., Socher, R., Li, L.J., Li, K., Fei-Fei, L.: ImageNet: A Large-Scale\\n\\nHierarchical Image Database. In: CVPR09 (2009)\\n\\n[6] Deng, Y., Kanervisto, A., Ling, J., Rush, A.M.: Image-to-markup generation with\\ncoarse-to-fine attention. In: International Conference on Machine Learning. pp.\\n980–989. PMLR (2017)\\n\\n[7] Ganin, Y., Lempitsky, V.: Unsupervised domain adaptation by backpropagation.\\nIn: International conference on machine learning. pp. 1180–1189. PMLR (2015)\\n\\x0cLayoutParser: A Unified Toolkit for DL-Based DIA\\n\\n15\\n\\n[8] Gardner, M., Grus, J., Neumann, M., Tafjord, O., Dasigi, P., Liu, N., Peters,\\nM., Schmitz, M., Zettlemoyer, L.: Allennlp: A deep semantic natural language\\nprocessing platform. arXiv preprint arXiv:1803.07640 (2018)\\n(cid:32)Lukasz Garncarek, Powalski, R., Stanis(cid:32)lawek, T., Topolski, B., Halama, P.,\\nGrali´nski, F.: Lambert: Layout-aware (language) modeling using bert for in-\\nformation extraction (2020)\\n\\n[9]\\n\\n[10] Graves, A., Fern´andez, S., Gomez, F., Schmidhuber, J.: Connectionist temporal\\nclassification: labelling unsegmented sequence data with recurrent neural networks.\\nIn: Proceedings of the 23rd international conference on Machine learning. pp.\\n369–376 (2006)\\n\\n[11] Harley, A.W., Ufkes, A., Derpanis, K.G.: Evaluation of deep convolutional nets for\\ndocument image classification and retrieval. In: 2015 13th International Conference\\non Document Analysis and Recognition (ICDAR). pp. 991–995. IEEE (2015)\\n[12] He, K., Gkioxari, G., Doll´ar, P., Girshick, R.: Mask r-cnn. In: Proceedings of the\\n\\nIEEE international conference on computer vision. pp. 2961–2969 (2017)\\n\\n[13] He, K., Zhang, X., Ren, S., Sun, J.: Deep residual learning for image recognition.\\nIn: Proceedings of the IEEE conference on computer vision and pattern recognition.\\npp. 770–778 (2016)\\n\\n[14] Kay, A.: Tesseract: An open-source optical character recognition engine. Linux J.\\n\\n2007(159), 2 (Jul 2007)\\n\\n[15] Lamiroy, B., Lopresti, D.: An open architecture for end-to-end document analysis\\nbenchmarking. In: 2011 International Conference on Document Analysis and\\nRecognition. pp. 42–47. IEEE (2011)\\n\\n[16] Lee, B.C., Weld, D.S.: Newspaper navigator: Open faceted search for 1.5\\nmillion images. In: Adjunct Publication of the 33rd Annual ACM Sym-\\nposium on User\\nInterface Software and Technology. p. 120–122. UIST\\n’20 Adjunct, Association for Computing Machinery, New York, NY, USA\\n(2020). https://doi.org/10.1145/3379350.3416143, https://doi-org.offcampus.\\nlib.washington.edu/10.1145/3379350.3416143\\n\\n[17] Lee, B.C.G., Mears, J., Jakeway, E., Ferriter, M., Adams, C., Yarasavage, N.,\\nThomas, D., Zwaard, K., Weld, D.S.: The Newspaper Navigator Dataset: Extracting\\nHeadlines and Visual Content from 16 Million Historic Newspaper Pages in\\nChronicling America, p. 3055–3062. Association for Computing Machinery, New\\nYork, NY, USA (2020), https://doi.org/10.1145/3340531.3412767\\n\\n[18] Li, M., Cui, L., Huang, S., Wei, F., Zhou, M., Li, Z.: Tablebank: Table benchmark\\nfor image-based table detection and recognition. arXiv preprint arXiv:1903.01949\\n(2019)\\n\\n[19] Lin, T.Y., Maire, M., Belongie, S., Hays, J., Perona, P., Ramanan, D., Doll´ar, P.,\\nZitnick, C.L.: Microsoft coco: Common objects in context. In: European conference\\non computer vision. pp. 740–755. Springer (2014)\\n\\n[20] Long, J., Shelhamer, E., Darrell, T.: Fully convolutional networks for semantic\\nsegmentation. In: Proceedings of the IEEE conference on computer vision and\\npattern recognition. pp. 3431–3440 (2015)\\n\\n[21] Neudecker, C., Schlarb, S., Dogan, Z.M., Missier, P., Sufi, S., Williams, A., Wolsten-\\ncroft, K.: An experimental workflow development platform for historical document\\ndigitisation and analysis. In: Proceedings of the 2011 workshop on historical\\ndocument imaging and processing. pp. 161–168 (2011)\\n\\n[22] Oliveira, S.A., Seguin, B., Kaplan, F.: dhsegment: A generic deep-learning approach\\nfor document segmentation. In: 2018 16th International Conference on Frontiers\\nin Handwriting Recognition (ICFHR). pp. 7–12. IEEE (2018)\\n\\x0c16\\n\\nZ. Shen et al.\\n\\n[23] Paszke, A., Gross, S., Chintala, S., Chanan, G., Yang, E., DeVito, Z., Lin, Z.,\\nDesmaison, A., Antiga, L., Lerer, A.: Automatic differentiation in pytorch (2017)\\n[24] Paszke, A., Gross, S., Massa, F., Lerer, A., Bradbury, J., Chanan, G., Killeen,\\nT., Lin, Z., Gimelshein, N., Antiga, L., et al.: Pytorch: An imperative style,\\nhigh-performance deep learning library. arXiv preprint arXiv:1912.01703 (2019)\\n[25] Pletschacher, S., Antonacopoulos, A.: The page (page analysis and ground-truth\\nelements) format framework. In: 2010 20th International Conference on Pattern\\nRecognition. pp. 257–260. IEEE (2010)\\n\\n[26] Prasad, D., Gadpal, A., Kapadni, K., Visave, M., Sultanpure, K.: Cascadetabnet:\\nAn approach for end to end table detection and structure recognition from image-\\nbased documents. In: Proceedings of the IEEE/CVF Conference on Computer\\nVision and Pattern Recognition Workshops. pp. 572–573 (2020)\\n\\n[27] Qasim, S.R., Mahmood, H., Shafait, F.: Rethinking table recognition using graph\\nneural networks. In: 2019 International Conference on Document Analysis and\\nRecognition (ICDAR). pp. 142–147. IEEE (2019)\\n\\n[28] Ren, S., He, K., Girshick, R., Sun, J.: Faster r-cnn: Towards real-time object\\ndetection with region proposal networks. In: Advances in neural information\\nprocessing systems. pp. 91–99 (2015)\\n\\n[29] Scarselli, F., Gori, M., Tsoi, A.C., Hagenbuchner, M., Monfardini, G.: The graph\\nneural network model. IEEE transactions on neural networks 20(1), 61–80 (2008)\\n[30] Schreiber, S., Agne, S., Wolf, I., Dengel, A., Ahmed, S.: Deepdesrt: Deep learning\\nfor detection and structure recognition of tables in document images. In: 2017 14th\\nIAPR international conference on document analysis and recognition (ICDAR).\\nvol. 1, pp. 1162–1167. IEEE (2017)\\n\\n[31] Shen, Z., Zhang, K., Dell, M.: A large dataset of historical japanese documents\\nwith complex layouts. In: Proceedings of the IEEE/CVF Conference on Computer\\nVision and Pattern Recognition Workshops. pp. 548–549 (2020)\\n\\n[32] Shen, Z., Zhao, J., Dell, M., Yu, Y., Li, W.: Olala: Object-level active learning\\n\\nbased layout annotation. arXiv preprint arXiv:2010.01762 (2020)\\n\\n[33] Studer, L., Alberti, M., Pondenkandath, V., Goktepe, P., Kolonko, T., Fischer,\\nA., Liwicki, M., Ingold, R.: A comprehensive study of imagenet pre-training for\\nhistorical document image analysis. In: 2019 International Conference on Document\\nAnalysis and Recognition (ICDAR). pp. 720–725. IEEE (2019)\\n\\n[34] Wolf, T., Debut, L., Sanh, V., Chaumond, J., Delangue, C., Moi, A., Cistac, P.,\\nRault, T., Louf, R., Funtowicz, M., et al.: Huggingface’s transformers: State-of-\\nthe-art natural language processing. arXiv preprint arXiv:1910.03771 (2019)\\n[35] Wu, Y., Kirillov, A., Massa, F., Lo, W.Y., Girshick, R.: Detectron2. https://\\n\\ngithub.com/facebookresearch/detectron2 (2019)\\n\\n[36] Xu, Y., Xu, Y., Lv, T., Cui, L., Wei, F., Wang, G., Lu, Y., Florencio, D., Zhang, C.,\\nChe, W., et al.: Layoutlmv2: Multi-modal pre-training for visually-rich document\\nunderstanding. arXiv preprint arXiv:2012.14740 (2020)\\n\\n[37] Xu, Y., Li, M., Cui, L., Huang, S., Wei, F., Zhou, M.: Layoutlm: Pre-training of\\n\\ntext and layout for document image understanding (2019)\\n\\n[38] Zhong, X., Tang, J., Yepes, A.J.: Publaynet:\\n\\nlayout analysis.\\n\\nument\\nAnalysis and Recognition (ICDAR). pp. 1015–1022.\\nhttps://doi.org/10.1109/ICDAR.2019.00166\\n\\nlargest dataset ever for doc-\\nIn: 2019 International Conference on Document\\nIEEE (Sep 2019).')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ "docs = loader.load()\n", "docs[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "ExecuteTime": { - "end_time": "2025-01-21T08:43:59.809736Z", - "start_time": "2025-01-21T08:43:59.806182Z" - } - }, + ], "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'author': '',\n", - " 'creationdate': '2021-06-22T01:27:10+00:00',\n", - " 'creator': 'LaTeX with hyperref',\n", - " 'keywords': '',\n", - " 'moddate': '2021-06-22T01:27:10+00:00',\n", - " 'ptex.fullbanner': 'This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live '\n", - " '2020) kpathsea version 6.3.2',\n", - " 'producer': 'pdfTeX-1.40.21',\n", - " 'subject': '',\n", - " 'title': '',\n", - " 'trapped': 'False',\n", - " 'total_pages': 16,\n", - " 'source': './example_data/layout-parser-paper.pdf'}\n" + "ename": "ImportError", + "evalue": "pdfminer package not found, please install it with `pip install pdfminer.six`", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mModuleNotFoundError\u001B[0m Traceback (most recent call last)", + "File \u001B[0;32m~/workspace.bda/langchain/libs/community/langchain_community/document_loaders/parsers/pdf.py:697\u001B[0m, in \u001B[0;36mPDFMinerParser.lazy_parse\u001B[0;34m(self, blob)\u001B[0m\n\u001B[1;32m 696\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 697\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mpdfminer\u001B[39;00m\n\u001B[1;32m 698\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mpdfminer\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mconverter\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m PDFLayoutAnalyzer\n", + "\u001B[0;31mModuleNotFoundError\u001B[0m: No module named 'pdfminer'", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001B[0;31mImportError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[4], line 1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m docs \u001B[38;5;241m=\u001B[39m \u001B[43mloader\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 2\u001B[0m docs[\u001B[38;5;241m0\u001B[39m]\n", + "File \u001B[0;32m~/workspace.bda/langchain/libs/core/langchain_core/document_loaders/base.py:32\u001B[0m, in \u001B[0;36mBaseLoader.load\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 30\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21mload\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28mlist\u001B[39m[Document]:\n\u001B[1;32m 31\u001B[0m \u001B[38;5;250m \u001B[39m\u001B[38;5;124;03m\"\"\"Load data into Document objects.\"\"\"\u001B[39;00m\n\u001B[0;32m---> 32\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mlist\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlazy_load\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/workspace.bda/langchain/libs/community/langchain_community/document_loaders/pdf.py:682\u001B[0m, in \u001B[0;36mPDFMinerLoader.lazy_load\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 680\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 681\u001B[0m blob \u001B[38;5;241m=\u001B[39m Blob\u001B[38;5;241m.\u001B[39mfrom_path(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfile_path) \u001B[38;5;66;03m# type: ignore[attr-defined]\u001B[39;00m\n\u001B[0;32m--> 682\u001B[0m \u001B[38;5;28;01myield from\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mparser\u001B[38;5;241m.\u001B[39mlazy_parse(blob)\n", + "File \u001B[0;32m~/workspace.bda/langchain/libs/community/langchain_community/document_loaders/parsers/pdf.py:718\u001B[0m, in \u001B[0;36mPDFMinerParser.lazy_parse\u001B[0;34m(self, blob)\u001B[0m\n\u001B[1;32m 712\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m(\n\u001B[1;32m 713\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mThis parser is tested with pdfminer.six version 20201018 or \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 714\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mlater. Remove pdfminer, and install pdfminer.six with \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 715\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m`pip uninstall pdfminer && pip install pdfminer.six`.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 716\u001B[0m )\n\u001B[1;32m 717\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m:\n\u001B[0;32m--> 718\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m(\n\u001B[1;32m 719\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpdfminer package not found, please install it \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 720\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwith `pip install pdfminer.six`\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 721\u001B[0m )\n\u001B[1;32m 723\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m blob\u001B[38;5;241m.\u001B[39mas_bytes_io() \u001B[38;5;28;01mas\u001B[39;00m pdf_file_obj, TemporaryDirectory() \u001B[38;5;28;01mas\u001B[39;00m tempdir:\n\u001B[1;32m 724\u001B[0m pages \u001B[38;5;241m=\u001B[39m PDFPage\u001B[38;5;241m.\u001B[39mget_pages(pdf_file_obj, password\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mpassword \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", + "\u001B[0;31mImportError\u001B[0m: pdfminer package not found, please install it with `pip install pdfminer.six`" ] } ], + "execution_count": 4 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2025-04-03T14:49:47.923187Z", + "start_time": "2025-04-03T14:49:47.868996Z" + } + }, "source": [ "import pprint\n", "\n", "pprint.pp(docs[0].metadata)" - ] + ], + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'docs' is not defined", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mNameError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[5], line 3\u001B[0m\n\u001B[1;32m 1\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mpprint\u001B[39;00m\n\u001B[0;32m----> 3\u001B[0m pprint\u001B[38;5;241m.\u001B[39mpp(\u001B[43mdocs\u001B[49m[\u001B[38;5;241m0\u001B[39m]\u001B[38;5;241m.\u001B[39mmetadata)\n", + "\u001B[0;31mNameError\u001B[0m: name 'docs' is not defined" + ] + } + ], + "execution_count": 5 }, { "cell_type": "markdown", diff --git a/docs/docs/integrations/document_loaders/pdfplumber.ipynb b/docs/docs/integrations/document_loaders/pdfplumber.ipynb index 17f85383d6d..b850d1e1300 100644 --- a/docs/docs/integrations/document_loaders/pdfplumber.ipynb +++ b/docs/docs/integrations/document_loaders/pdfplumber.ipynb @@ -1,5 +1,4 @@ { -<<<<<<< HEAD "cells": [ { "cell_type": "markdown", @@ -1339,5 +1338,4 @@ }, "nbformat": 4, "nbformat_minor": 2 ->>>>>>> upstream/master }