github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-06 23:24:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow configuring content selector

Browse Source
This commit is contained in:
Alex Iribarren 2023-04-01 21:40:21 +02:00
parent e77b63bdad
commit 06017ffd21
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
langchain/document_loaders/gitbook.py
View File

@ -18,6 +18,7 @@ class GitbookLoader(WebBaseLoader):
web_page: str, web_page: str,
load_all_paths: bool = False, load_all_paths: bool = False,
base_url: Optional[str] = None, base_url: Optional[str] = None,
content_selector: str = "main",
): ):
"""Initialize with web page and whether to load all paths. """Initialize with web page and whether to load all paths.
@ -39,6 +40,7 @@ class GitbookLoader(WebBaseLoader):
web_paths = web_page web_paths = web_page
super().__init__(web_paths) super().__init__(web_paths)
self.load_all_paths = load_all_paths self.load_all_paths = load_all_paths
self.content_selector = content_selector
def load(self) -> List[Document]: def load(self) -> List[Document]:
"""Fetch text from one single GitBook page.""" """Fetch text from one single GitBook page."""
@ -61,7 +63,7 @@ class GitbookLoader(WebBaseLoader):
self, soup: Any, custom_url: Optional[str] = None self, soup: Any, custom_url: Optional[str] = None
) -> Optional[Document]: ) -> Optional[Document]:
"""Fetch content from page and return Document.""" """Fetch content from page and return Document."""
page_content_raw = soup.find("main") page_content_raw = soup.find(self.content_selector)
if not page_content_raw: if not page_content_raw:
return None return None
content = page_content_raw.get_text(separator="\n").strip() content = page_content_raw.get_text(separator="\n").strip()