github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-07-15 01:13:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix extract sublink bug (#10855)

Browse Source
This commit is contained in:
Bagatur 2023-09-20 13:30:42 -07:00 committed by GitHub
parent 7dec2d399b
commit de0a02f507
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libs/langchain/langchain/utils/html.py
View File

@ -59,11 +59,12 @@ def extract_sub_links(
for link in all_links: for link in all_links:
# Some may be absolute links like https://to/path # Some may be absolute links like https://to/path
if link.startswith("http"): if link.startswith("http"):
if not prevent_outside or link.startswith(base_url): absolute_paths.add(link)
absolute_paths.add(link)
# Some may have omitted the protocol like //to/path # Some may have omitted the protocol like //to/path
elif link.startswith("//"): elif link.startswith("//"):
absolute_paths.add(f"{urlparse(base_url).scheme}:{link}") absolute_paths.add(f"{urlparse(base_url).scheme}:{link}")
else: else:
absolute_paths.add(urljoin(base_url, link)) absolute_paths.add(urljoin(base_url, link))
if prevent_outside:
return [p for p in absolute_paths if p.startswith(base_url)]
return list(absolute_paths) return list(absolute_paths)

9
libs/langchain/tests/unit_tests/utils/test_html.py
View File

@ -86,13 +86,8 @@ def test_extract_sub_links() -> None:
actual = sorted(extract_sub_links(html, "https://foobar.com")) actual = sorted(extract_sub_links(html, "https://foobar.com"))
assert actual == expected assert actual == expected
actual = sorted(extract_sub_links(html, "https://foobar.com/hello")) actual = extract_sub_links(html, "https://foobar.com/hello")
expected = sorted( expected = ["https://foobar.com/hello"]
[
"https://foobar.com/hello",
"https://foobar.com/how/are/you/doing",
]
)
assert actual == expected assert actual == expected
actual = sorted( actual = sorted(