github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-21 14:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

core[minor]: Add aload to document loader (#19936)

Browse Source 
Add aload to document loader
This commit is contained in:
Eugene Yurtsev 2024-04-03 10:46:47 -04:00 committed by GitHub
parent 31a641a155
commit d293431e10
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libs/community/langchain_community/document_loaders/web_base.py
View File

@ -251,7 +251,7 @@ class WebBaseLoader(BaseLoader):
metadata = _build_metadata(soup, path) metadata = _build_metadata(soup, path)
yield Document(page_content=text, metadata=metadata) yield Document(page_content=text, metadata=metadata)
def aload(self) -> List[Document]: def aload(self) -> List[Document]: # type: ignore
"""Load text from the urls in web_path async into Documents.""" """Load text from the urls in web_path async into Documents."""
results = self.scrape_all(self.web_paths) results = self.scrape_all(self.web_paths)

4
libs/core/langchain_core/document_loaders/base.py
View File

@ -28,6 +28,10 @@ class BaseLoader(ABC):
"""Load data into Document objects.""" """Load data into Document objects."""
return list(self.lazy_load()) return list(self.lazy_load())
async def aload(self) -> List[Document]:
"""Load data into Document objects."""
return [document async for document in self.alazy_load()]
def load_and_split( def load_and_split(
self, text_splitter: Optional[TextSplitter] = None self, text_splitter: Optional[TextSplitter] = None
) -> List[Document]: ) -> List[Document]:

1
libs/core/tests/unit_tests/document_loaders/test_base.py
View File

@ -64,3 +64,4 @@ async def test_default_aload() -> None:
docs = loader.load() docs = loader.load()
assert docs == [Document(page_content="foo"), Document(page_content="bar")] assert docs == [Document(page_content="foo"), Document(page_content="bar")]
assert docs == [doc async for doc in loader.alazy_load()] assert docs == [doc async for doc in loader.alazy_load()]
assert docs == await loader.aload()