community: better support of pathlib paths in document loaders (#18396)

So this arose from the
https://github.com/langchain-ai/langchain/pull/18397 problem of document
loaders not supporting `pathlib.Path`.

This pull request provides more uniform support for Path as an argument.
The core ideas for this upgrade: 
- if there is a local file path used as an argument, it should be
supported as `pathlib.Path`
- if there are some external calls that may or may not support Pathlib,
the argument is immidiately converted to `str`
- if there `self.file_path` is used in a way that it allows for it to
stay pathlib without conversion, is is only converted for the metadata.

Twitter handle: https://twitter.com/mwmajewsk
This commit is contained in:
mwmajewsk
2024-03-26 16:51:52 +01:00
committed by GitHub
parent 94b869a974
commit f7a1fd91b8
32 changed files with 147 additions and 80 deletions

View File

@@ -1,5 +1,6 @@
import os
from typing import Any, Iterator, List
from pathlib import Path
from typing import Any, Iterator, List, Union
from langchain_core.documents import Document
@@ -41,7 +42,10 @@ class UnstructuredEmailLoader(UnstructuredFileLoader):
"""
def __init__(
self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
self,
file_path: Union[str, Path],
mode: str = "single",
**unstructured_kwargs: Any,
):
process_attachments = unstructured_kwargs.get("process_attachments")
attachment_partitioner = unstructured_kwargs.get("attachment_partitioner")
@@ -79,17 +83,17 @@ class OutlookMessageLoader(BaseLoader):
https://github.com/TeamMsgExtractor/msg-extractor
"""
def __init__(self, file_path: str):
def __init__(self, file_path: Union[str, Path]):
"""Initialize with a file path.
Args:
file_path: The path to the Outlook Message file.
"""
self.file_path = file_path
self.file_path = str(file_path)
if not os.path.isfile(self.file_path):
raise ValueError("File path %s is not a valid file" % self.file_path)
raise ValueError(f"File path {self.file_path} is not a valid file")
try:
import extract_msg # noqa:F401