github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-09-14 22:17:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

DOCS: update rag use case (#13319)

Browse Source
This commit is contained in:
Bagatur
2023-11-15 10:54:15 -08:00
committed by GitHub
parent a0b39a4325
commit 76c317ed78
15 changed files with 968 additions and 1919 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
libs/langchain/langchain/document_loaders/web_base.py
View File

@@ -245,7 +245,7 @@ class WebBaseLoader(BaseLoader):
def lazy_load(self) -> Iterator[Document]:
"""Lazy load text from the url(s) in web_path."""
for path in self.web_paths:
soup = self._scrape(path)
soup = self._scrape(path, bs_kwargs=self.bs_kwargs)
text = soup.get_text(**self.bs_get_text_kwargs)
metadata = _build_metadata(soup, path)
yield Document(page_content=text, metadata=metadata)