deafult depth tests

This commit is contained in:
isaac hershenson
2024-06-04 20:42:51 -07:00
parent 09db121339
commit 6d3568d992

View File

@@ -2,7 +2,6 @@ import requests_mock
from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader
import aiohttp
def mock_requests(loader):
html1 = (
'<div><a class="blah" href="/one">hullo</a></div>'
@@ -59,6 +58,17 @@ def test_async__init__(mocker):
loader = RecursiveUrlLoader("http://test.com",max_depth=1, use_async=True)
docs = loader.load()
assert len(docs) == 1
def test_sync_default_depth():
loader = RecursiveUrlLoader("http://test.com")
docs = mock_requests(loader)
assert len(docs) == 3
def test_async_default_depth(mocker):
mocker.patch.object(aiohttp.ClientSession, 'get', new=MockGet)
loader = RecursiveUrlLoader("http://test.com", use_async=True)
docs = loader.load()
assert len(docs) == 3
def test_sync_deduplication():
loader = RecursiveUrlLoader("http://test.com",max_depth=3)