github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-01 02:43:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add recursive parameter to DirectoryLoader (#1389)

Browse Source 
This PR allows loading a directory recursively.
This commit is contained in:
Kacper Łukawski 2023-03-02 16:06:26 +01:00 committed by GitHub
parent 9ac442624c
commit f032609f8d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
langchain/document_loaders/directory.py
View File

@ -30,6 +30,7 @@ class DirectoryLoader(BaseLoader):
silent_errors: bool = False,
load_hidden: bool = False,
loader_cls: FILE_LOADER_TYPE = UnstructuredFileLoader,
recursive: bool = False,
):
"""Initialize with path to directory and how to glob over it."""
self.path = path
@ -37,12 +38,14 @@ class DirectoryLoader(BaseLoader):
self.load_hidden = load_hidden
self.loader_cls = loader_cls
self.silent_errors = silent_errors
self.recursive = recursive
def load(self) -> List[Document]:
"""Load documents."""
p = Path(self.path)
docs = []
for i in p.glob(self.glob):
items = p.rglob(self.glob) if self.recursive else p.glob(self.glob)
for i in items:
if i.is_file():
if _is_visible(i.relative_to(p)) or self.load_hidden:
try: