mirror of
https://github.com/hwchase17/langchain.git
synced 2025-04-29 04:16:02 +00:00
64df60e690
## Description This PR adds a new `sitemap_url` parameter to the `GitbookLoader` class that allows users to specify a custom sitemap URL when loading content from a GitBook site. This is particularly useful for GitBook sites that use non-standard sitemap file names like `sitemap-pages.xml` instead of the default `sitemap.xml`. The standard `GitbookLoader` assumes that the sitemap is located at `/sitemap.xml`, but some GitBook instances (including GitBook's own documentation) use different paths for their sitemaps. This parameter makes the loader more flexible and helps users extract content from a wider range of GitBook sites. ## Issue Fixes bug [30473](https://github.com/langchain-ai/langchain/issues/30473) where the `GitbookLoader` would fail to find pages on GitBook sites that use custom sitemap URLs. ## Dependencies No new dependencies required. *I've added*: * Unit tests to verify the parameter works correctly * Integration tests to confirm the parameter is properly used with real GitBook sites * Updated docstrings with parameter documentation The changes are fully backward compatible, as the parameter is optional with a sensible default. --------- Co-authored-by: andrasfe <andrasf94@gmail.com> Co-authored-by: Eugene Yurtsev <eugene@langchain.dev>
78 lines
2.5 KiB
Python
78 lines
2.5 KiB
Python
from typing import Optional
|
|
|
|
import pytest
|
|
|
|
from langchain_community.document_loaders.gitbook import GitbookLoader
|
|
|
|
|
|
class TestGitbookLoader:
|
|
@pytest.mark.parametrize(
|
|
"web_page, load_all_paths, base_url, expected_web_path",
|
|
[
|
|
("https://example.com/page1", False, None, "https://example.com/page1"),
|
|
(
|
|
"https://example.com/",
|
|
True,
|
|
"https://example.com",
|
|
"https://example.com/sitemap.xml",
|
|
),
|
|
],
|
|
)
|
|
def test_init(
|
|
self,
|
|
web_page: str,
|
|
load_all_paths: bool,
|
|
base_url: Optional[str],
|
|
expected_web_path: str,
|
|
) -> None:
|
|
loader = GitbookLoader(
|
|
web_page, load_all_paths=load_all_paths, base_url=base_url
|
|
)
|
|
print(loader.__dict__) # noqa: T201
|
|
assert (
|
|
loader.base_url == (base_url or web_page)[:-1]
|
|
if (base_url or web_page).endswith("/")
|
|
else (base_url or web_page)
|
|
)
|
|
assert loader.web_path == expected_web_path
|
|
assert loader.load_all_paths == load_all_paths
|
|
|
|
@pytest.mark.parametrize(
|
|
"web_page, expected_number_results",
|
|
[("https://platform-docs.opentargets.org/getting-started", 1)],
|
|
)
|
|
def test_load_single_page(
|
|
self, web_page: str, expected_number_results: int
|
|
) -> None:
|
|
loader = GitbookLoader(web_page)
|
|
result = loader.load()
|
|
assert len(result) == expected_number_results
|
|
|
|
@pytest.mark.parametrize("web_page", [("https://platform-docs.opentargets.org/")])
|
|
def test_load_multiple_pages(self, web_page: str) -> None:
|
|
loader = GitbookLoader(web_page, load_all_paths=True)
|
|
result = loader.load()
|
|
print(len(result)) # noqa: T201
|
|
assert len(result) > 10
|
|
|
|
@pytest.mark.parametrize(
|
|
"web_page, sitemap_url, expected_web_path",
|
|
[
|
|
(
|
|
"https://example.com/",
|
|
"https://example.com/custom-sitemap.xml",
|
|
"https://example.com/custom-sitemap.xml",
|
|
),
|
|
],
|
|
)
|
|
def test_init_with_custom_sitemap(
|
|
self,
|
|
web_page: str,
|
|
sitemap_url: str,
|
|
expected_web_path: str,
|
|
) -> None:
|
|
"""Test that the custom sitemap URL is correctly used when provided."""
|
|
loader = GitbookLoader(web_page, load_all_paths=True, sitemap_url=sitemap_url)
|
|
assert loader.web_path == expected_web_path
|
|
assert loader.load_all_paths
|