mirror of
https://github.com/hwchase17/langchain.git
synced 2025-07-15 17:33:53 +00:00
community(pypdfloader): added page_label in metadata for pypdf loader (#29225)
# Description ## Summary This PR adds support for handling multi-labeled page numbers in the **PyPDFLoader**. Some PDFs use complex page numbering systems where the actual content may begin after multiple introductory pages. The page_label field helps accurately reflect the document’s page structure, making it easier to handle such cases during document parsing. ## Motivation This feature improves document parsing accuracy by allowing users to access the actual page labels instead of relying only on the physical page numbers. This is particularly useful for documents where the first few pages have roman numerals or other non-standard page labels. ## Use Case This feature is especially useful for **Retrieval-Augmented Generation** (RAG) systems where users may reference page numbers when asking questions. Some PDFs have both labeled page numbers (like roman numerals for introductory sections) and index-based page numbers. For example, a user might ask: "What is mentioned on page 5?" The system can now check both: • **Index-based page number** (page) • **Labeled page number** (page_label) This dual-check helps improve retrieval accuracy. Additionally, the results can be validated with an **agent or tool** to ensure the retrieved pages match the user’s query contextually. ## Code Changes - Added a page_label field to the metadata of the Document class in **PyPDFLoader**. - Implemented support for retrieving page_label from the pdf_reader.page_labels. - Created a test case (test_pypdf_loader_with_multi_label_page_numbers) with a sample PDF containing multi-labeled pages (geotopo-komprimiert.pdf) [[Source of pdf](https://github.com/py-pdf/sample-files/blob/main/009-pdflatex-geotopo/GeoTopo-komprimiert.pdf)]. - Updated existing tests to ensure compatibility and verify page_label extraction. ## Tests Added - Added a new test case for a PDF with multi-labeled pages. - Verified both page and page_label metadata fields are correctly extracted. ## Screenshots <img width="549" alt="image" src="https://github.com/user-attachments/assets/65db9f5c-032e-4592-926f-824777c28f33" />
This commit is contained in:
parent
1a38948ee3
commit
eaf2fb287f
@ -123,7 +123,11 @@ class PyPDFParser(BaseBlobParser):
|
|||||||
Document(
|
Document(
|
||||||
page_content=_extract_text_from_page(page=page)
|
page_content=_extract_text_from_page(page=page)
|
||||||
+ self._extract_images_from_page(page),
|
+ self._extract_images_from_page(page),
|
||||||
metadata={"source": blob.source, "page": page_number},
|
metadata={
|
||||||
|
"source": blob.source,
|
||||||
|
"page": page_number,
|
||||||
|
"page_label": pdf_reader.page_labels[page_number],
|
||||||
|
},
|
||||||
# type: ignore[attr-defined]
|
# type: ignore[attr-defined]
|
||||||
)
|
)
|
||||||
for page_number, page in enumerate(pdf_reader.pages)
|
for page_number, page in enumerate(pdf_reader.pages)
|
||||||
|
Binary file not shown.
@ -12,6 +12,10 @@ path_to_layout_pdf = (
|
|||||||
Path(__file__).parent.parent
|
Path(__file__).parent.parent
|
||||||
/ "document_loaders/sample_documents/layout-parser-paper.pdf"
|
/ "document_loaders/sample_documents/layout-parser-paper.pdf"
|
||||||
)
|
)
|
||||||
|
path_to_multi_label_page_numbers_pdf = (
|
||||||
|
Path(__file__).parent.parent
|
||||||
|
/ "document_loaders/sample_documents/geotopo-komprimiert.pdf"
|
||||||
|
)
|
||||||
path_to_layout_pdf_txt = (
|
path_to_layout_pdf_txt = (
|
||||||
Path(__file__).parent.parent.parent
|
Path(__file__).parent.parent.parent
|
||||||
/ "integration_tests/examples/layout-parser-paper-page-1.txt"
|
/ "integration_tests/examples/layout-parser-paper-page-1.txt"
|
||||||
@ -32,6 +36,7 @@ def test_pypdf_loader() -> None:
|
|||||||
assert len(docs) == 16
|
assert len(docs) == 16
|
||||||
for page, doc in enumerate(docs):
|
for page, doc in enumerate(docs):
|
||||||
assert doc.metadata["page"] == page
|
assert doc.metadata["page"] == page
|
||||||
|
assert doc.metadata["page_label"] == str(page + 1)
|
||||||
assert doc.metadata["source"].endswith("layout-parser-paper.pdf")
|
assert doc.metadata["source"].endswith("layout-parser-paper.pdf")
|
||||||
assert len(doc.page_content) > 10
|
assert len(doc.page_content) > 10
|
||||||
|
|
||||||
@ -49,6 +54,7 @@ def test_pypdf_loader_with_layout() -> None:
|
|||||||
assert len(docs) == 16
|
assert len(docs) == 16
|
||||||
for page, doc in enumerate(docs):
|
for page, doc in enumerate(docs):
|
||||||
assert doc.metadata["page"] == page
|
assert doc.metadata["page"] == page
|
||||||
|
assert doc.metadata["page_label"] == str(page + 1)
|
||||||
assert doc.metadata["source"].endswith("layout-parser-paper.pdf")
|
assert doc.metadata["source"].endswith("layout-parser-paper.pdf")
|
||||||
assert len(doc.page_content) > 10
|
assert len(doc.page_content) > 10
|
||||||
|
|
||||||
@ -60,3 +66,19 @@ def test_pypdf_loader_with_layout() -> None:
|
|||||||
cleaned_first_page = re.sub(r"\x00", "", first_page)
|
cleaned_first_page = re.sub(r"\x00", "", first_page)
|
||||||
cleaned_expected = re.sub(r"\x00", "", expected)
|
cleaned_expected = re.sub(r"\x00", "", expected)
|
||||||
assert cleaned_first_page == cleaned_expected
|
assert cleaned_first_page == cleaned_expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.requires("pypdf")
|
||||||
|
def test_pypdf_loader_with_multi_labled_page_numbers() -> None:
|
||||||
|
"""Test PyPDFLoader with a pdf that contains multi-labled page numbers."""
|
||||||
|
loader = PyPDFLoader(str(path_to_multi_label_page_numbers_pdf))
|
||||||
|
docs = loader.load()
|
||||||
|
|
||||||
|
assert len(docs) == 7
|
||||||
|
|
||||||
|
assert docs[0].metadata["page"] == 0
|
||||||
|
assert docs[0].metadata["page_label"] == "i"
|
||||||
|
|
||||||
|
# Since the actual page numbers in this pdf starts from 4th page
|
||||||
|
assert docs[3].metadata["page"] == 3
|
||||||
|
assert docs[3].metadata["page_label"] == "1"
|
||||||
|
Loading…
Reference in New Issue
Block a user