github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-09 21:08:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

community: Implement lazy_load() for GithubFileLoader (#18584)

Browse Source
This commit is contained in:
Christophe Bornet 2024-03-05 18:35:50 +01:00 committed by GitHub
parent 04d134df17
commit c8a171a154
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libs/community/langchain_community/document_loaders/github.py
View File

@ -217,9 +217,7 @@ class GithubFileLoader(BaseGitHubLoader, ABC):
return "" return ""
def load(self) -> List[Document]: def lazy_load(self) -> Iterator[Document]:
documents = []
files = self.get_file_paths() files = self.get_file_paths()
for file in files: for file in files:
content = self.get_file_content_by_path(file["path"]) content = self.get_file_content_by_path(file["path"])
@ -232,6 +230,4 @@ class GithubFileLoader(BaseGitHubLoader, ABC):
"source": f"{self.github_api_url}/{self.repo}/{file['type']}/" "source": f"{self.github_api_url}/{self.repo}/{file['type']}/"
f"{self.branch}/{file['path']}", f"{self.branch}/{file['path']}",
} }
documents.append(Document(page_content=content, metadata=metadata)) yield Document(page_content=content, metadata=metadata)
return documents