update pdf document loaders' metadata source to url for online pdf (#13274)

- **Description:** Update 5 pdf document loaders in
`langchain.document_loaders.pdf`, to store a url in the metadata
(instead of a temporary, local file path) if the user provides a web
path to a pdf: `PyPDFium2Loader`, `PDFMinerLoader`,
`PDFMinerPDFasHTMLLoader`, `PyMuPDFLoader`, and `PDFPlumberLoader` were
updated.
- The updates follow the approach used to update `PyPDFLoader` for the
same behavior in #12092
- The `PyMuPDFLoader` changes required additional work in updating
`langchain.document_loaders.parsers.pdf.PyMuPDFParser` to be able to
process either an `io.BufferedReader` (from local pdf) or `io.BytesIO`
(from online pdf)
- The `PDFMinerPDFasHTMLLoader` change used a simpler approach since the
metadata is assigned by the loader and not the parser
  - **Issue:** Fixes #7034
  - **Dependencies:** None


```python
# PyPDFium2Loader example:
# old behavior
>>> from langchain.document_loaders import PyPDFium2Loader
>>> loader = PyPDFium2Loader('https://arxiv.org/pdf/1706.03762.pdf')
>>> docs = loader.load()
>>> docs[0].metadata
{'source': '/var/folders/7z/d5dt407n673drh1f5cm8spj40000gn/T/tmpm5oqa92f/tmp.pdf', 'page': 0}

# new behavior
>>> from langchain.document_loaders import PyPDFium2Loader
>>> loader = PyPDFium2Loader('https://arxiv.org/pdf/1706.03762.pdf')
>>> docs = loader.load()
>>> docs[0].metadata
{'source': 'https://arxiv.org/pdf/1706.03762.pdf', 'page': 0}
```
This commit is contained in:
123-fake-st 2023-11-29 14:07:46 -06:00 committed by GitHub
parent 6f64cb5078
commit 9bd6e9df36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 6 deletions

View File

@ -235,7 +235,10 @@ class PyMuPDFParser(BaseBlobParser):
import fitz
with blob.as_bytes_io() as file_path:
doc = fitz.open(file_path) # open document
if blob.data is None:
doc = fitz.open(file_path)
else:
doc = fitz.open(stream=file_path, filetype="pdf")
yield from [
Document(

View File

@ -194,7 +194,10 @@ class PyPDFium2Loader(BasePDFLoader):
self,
) -> Iterator[Document]:
"""Lazy load given path as pages."""
blob = Blob.from_path(self.file_path)
if self.web_path:
blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)
else:
blob = Blob.from_path(self.file_path)
yield from self.parser.parse(blob)
@ -284,7 +287,10 @@ class PDFMinerLoader(BasePDFLoader):
self,
) -> Iterator[Document]:
"""Lazily load documents."""
blob = Blob.from_path(self.file_path)
if self.web_path:
blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)
else:
blob = Blob.from_path(self.file_path)
yield from self.parser.parse(blob)
@ -318,7 +324,9 @@ class PDFMinerPDFasHTMLLoader(BasePDFLoader):
laparams=LAParams(),
output_type="html",
)
metadata = {"source": self.file_path}
metadata = {
"source": self.file_path if self.web_path is None else self.web_path
}
return [Document(page_content=output_string.getvalue(), metadata=metadata)]
@ -357,7 +365,10 @@ class PyMuPDFLoader(BasePDFLoader):
parser = PyMuPDFParser(
text_kwargs=text_kwargs, extract_images=self.extract_images
)
blob = Blob.from_path(self.file_path)
if self.web_path:
blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)
else:
blob = Blob.from_path(self.file_path)
return parser.parse(blob)
@ -523,7 +534,10 @@ class PDFPlumberLoader(BasePDFLoader):
dedupe=self.dedupe,
extract_images=self.extract_images,
)
blob = Blob.from_path(self.file_path)
if self.web_path:
blob = Blob.from_data(open(self.file_path, "rb").read(), path=self.web_path)
else:
blob = Blob.from_path(self.file_path)
return parser.parse(blob)