Added matching async load func to PlaywrightURLLoader (#5938)

Fixes # (issue)

The existing PlaywrightURLLoader load() function uses a synchronous
browser which is not compatible with jupyter.
This PR adds a sister function aload() which can be run insisde a
notebook.

---------

Co-authored-by: Harrison Chase <hw.chase.17@gmail.com>
This commit is contained in:
EllieRoseS
2023-07-13 22:51:38 +01:00
committed by GitHub
parent ae7714f1ba
commit c087ce74f7
2 changed files with 60 additions and 0 deletions

View File

@@ -1,4 +1,5 @@
"""Tests for the Playwright URL loader"""
import pytest
from langchain.document_loaders import PlaywrightURLLoader
@@ -19,3 +20,22 @@ def test_playwright_url_loader() -> None:
)
docs = loader.load()
assert len(docs) > 0
@pytest.mark.asyncio
async def test_playwright_async_url_loader() -> None:
"""Test Playwright async URL loader."""
urls = [
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
"https://goo.gl/maps/NDSHwePEyaHMFGwh8",
"https://techmeme.com",
"https://techcrunch.com",
]
loader = PlaywrightURLLoader(
urls=urls,
remove_selectors=["header", "footer"],
continue_on_failure=False,
headless=True,
)
docs = await loader.aload()
assert len(docs) > 0