github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-05-30 03:28:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support gitbooks in a subdirectory

Browse Source
This commit is contained in:
Alex Iribarren 2023-04-01 21:38:45 +02:00
parent a5b05dcc66
commit e77b63bdad
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
langchain/document_loaders/gitbook.py
View File

@ -1,6 +1,6 @@
"""Loader that loads GitBook."""
from typing import Any, List, Optional
from urllib.parse import urlparse
from urllib.parse import urljoin, urlparse
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
@ -47,7 +47,7 @@ class GitbookLoader(WebBaseLoader):
relative_paths = self._get_paths(soup_info)
documents = []
for path in relative_paths:
url = self.base_url + path
url = urljoin(self.base_url, path)
print(f"Fetching text from {url}")
soup_info = self._scrape(url)
documents.append(self._get_document(soup_info, url))