From 1fa5d94591969ec160371963e75816d76287ed08 Mon Sep 17 00:00:00 2001 From: Youngwook Kim Date: Thu, 20 Jul 2023 11:07:17 +0900 Subject: [PATCH 1/6] feat(document_loaders): add sync and async page evaluation methods to PlaywrightURLLoader --- .../document_loaders/url_playwright.py | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/url_playwright.py b/libs/langchain/langchain/document_loaders/url_playwright.py index ee4a47f2516..1c02ac30bb2 100644 --- a/libs/langchain/langchain/document_loaders/url_playwright.py +++ b/libs/langchain/langchain/document_loaders/url_playwright.py @@ -48,6 +48,36 @@ class PlaywrightURLLoader(BaseLoader): self.headless = headless self.remove_selectors = remove_selectors + def sync_evaluate_page(self, page): + """Process a page and return the text content. + This method can be overridden to apply custom logic. + """ + for selector in self.remove_selectors or []: + elements = page.locator(selector).all() + for element in elements: + if element.is_visible(): + element.evaluate("element => element.remove()") + + page_source = page.content() + elements = partition_html(text=page_source) + text = "\n\n".join([str(el) for el in elements]) + return text + + async def async_evaluate_page(self, page): + """Process a page asynchronously and return the text content. + This method can be overridden to apply custom logic. + """ + for selector in self.remove_selectors or []: + elements = await page.locator(selector).all() + for element in elements: + if await element.is_visible(): + await element.evaluate("element => element.remove()") + + page_source = await page.content() + elements = partition_html(text=page_source) + text = "\n\n".join([str(el) for el in elements]) + return text + def load(self) -> List[Document]: """Load the specified URLs using Playwright and create Document instances. @@ -65,16 +95,7 @@ class PlaywrightURLLoader(BaseLoader): try: page = browser.new_page() page.goto(url) - - for selector in self.remove_selectors or []: - elements = page.locator(selector).all() - for element in elements: - if element.is_visible(): - element.evaluate("element => element.remove()") - - page_source = page.content() - elements = partition_html(text=page_source) - text = "\n\n".join([str(el) for el in elements]) + text = self.sync_evaluate_page(page) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: @@ -105,16 +126,7 @@ class PlaywrightURLLoader(BaseLoader): try: page = await browser.new_page() await page.goto(url) - - for selector in self.remove_selectors or []: - elements = await page.locator(selector).all() - for element in elements: - if await element.is_visible(): - await element.evaluate("element => element.remove()") - - page_source = await page.content() - elements = partition_html(text=page_source) - text = "\n\n".join([str(el) for el in elements]) + text = await self.async_evaluate_page(page) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: From dc4b037957167e29342538c4d4ad17a2f3140b0a Mon Sep 17 00:00:00 2001 From: Youngwook Kim Date: Tue, 8 Aug 2023 23:31:27 +0900 Subject: [PATCH 2/6] docs(url_playwright): update docstrings for sync_evaluate_page and async_evaluate_page methods --- .../document_loaders/url_playwright.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/url_playwright.py b/libs/langchain/langchain/document_loaders/url_playwright.py index 1c02ac30bb2..188b7f22fe4 100644 --- a/libs/langchain/langchain/document_loaders/url_playwright.py +++ b/libs/langchain/langchain/document_loaders/url_playwright.py @@ -49,8 +49,13 @@ class PlaywrightURLLoader(BaseLoader): self.remove_selectors = remove_selectors def sync_evaluate_page(self, page): - """Process a page and return the text content. - This method can be overridden to apply custom logic. + """Process a page and return the text content synchronously. + + Args: + page: The page to process. + + Returns: + text: The text content of the page. """ for selector in self.remove_selectors or []: elements = page.locator(selector).all() @@ -64,8 +69,13 @@ class PlaywrightURLLoader(BaseLoader): return text async def async_evaluate_page(self, page): - """Process a page asynchronously and return the text content. - This method can be overridden to apply custom logic. + """Process a page and return the text content asynchronously. + + Args: + page: The page to process. + + Returns: + text: The text content of the page. """ for selector in self.remove_selectors or []: elements = await page.locator(selector).all() From 224263aa24401ccdea5bfa4e17b495d7cb75db43 Mon Sep 17 00:00:00 2001 From: Youngwook Kim Date: Wed, 9 Aug 2023 10:51:34 +0900 Subject: [PATCH 3/6] refactor(document_loaders): modify evaluation methods in PlaywrightURLLoader --- .../document_loaders/url_playwright.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/url_playwright.py b/libs/langchain/langchain/document_loaders/url_playwright.py index 188b7f22fe4..6a18d528d4e 100644 --- a/libs/langchain/langchain/document_loaders/url_playwright.py +++ b/libs/langchain/langchain/document_loaders/url_playwright.py @@ -48,15 +48,19 @@ class PlaywrightURLLoader(BaseLoader): self.headless = headless self.remove_selectors = remove_selectors - def sync_evaluate_page(self, page): + def sync_evaluate(self, page, browser, response): """Process a page and return the text content synchronously. Args: page: The page to process. + browser: The browser instance. + response: The response from page.goto(). Returns: text: The text content of the page. """ + from unstructured.partition.html import partition_html + for selector in self.remove_selectors or []: elements = page.locator(selector).all() for element in elements: @@ -68,15 +72,19 @@ class PlaywrightURLLoader(BaseLoader): text = "\n\n".join([str(el) for el in elements]) return text - async def async_evaluate_page(self, page): + async def async_evaluate(self, page, browser, response): """Process a page and return the text content asynchronously. Args: page: The page to process. + browser: The browser instance. + response: The response from page.goto(). Returns: text: The text content of the page. """ + from unstructured.partition.html import partition_html + for selector in self.remove_selectors or []: elements = await page.locator(selector).all() for element in elements: @@ -95,7 +103,6 @@ class PlaywrightURLLoader(BaseLoader): List[Document]: A list of Document instances with loaded content. """ from playwright.sync_api import sync_playwright - from unstructured.partition.html import partition_html docs: List[Document] = list() @@ -104,8 +111,8 @@ class PlaywrightURLLoader(BaseLoader): for url in self.urls: try: page = browser.new_page() - page.goto(url) - text = self.sync_evaluate_page(page) + response = page.goto(url) + text = self.sync_evaluate(page, browser, response) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: @@ -126,7 +133,6 @@ class PlaywrightURLLoader(BaseLoader): List[Document]: A list of Document instances with loaded content. """ from playwright.async_api import async_playwright - from unstructured.partition.html import partition_html docs: List[Document] = list() @@ -135,8 +141,8 @@ class PlaywrightURLLoader(BaseLoader): for url in self.urls: try: page = await browser.new_page() - await page.goto(url) - text = await self.async_evaluate_page(page) + response = await page.goto(url) + text = await self.async_evaluate(page, browser, response) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: From ef7f4aea32fc89796e29da549d75befd0a82d47f Mon Sep 17 00:00:00 2001 From: Youngwook Kim Date: Wed, 9 Aug 2023 10:59:22 +0900 Subject: [PATCH 4/6] refactor: modify method visibility in url_playwright --- .../langchain/document_loaders/url_playwright.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/url_playwright.py b/libs/langchain/langchain/document_loaders/url_playwright.py index 6a18d528d4e..48c8dcf0129 100644 --- a/libs/langchain/langchain/document_loaders/url_playwright.py +++ b/libs/langchain/langchain/document_loaders/url_playwright.py @@ -48,7 +48,7 @@ class PlaywrightURLLoader(BaseLoader): self.headless = headless self.remove_selectors = remove_selectors - def sync_evaluate(self, page, browser, response): + def _sync_evaluate(self, page, browser, response): """Process a page and return the text content synchronously. Args: @@ -72,7 +72,7 @@ class PlaywrightURLLoader(BaseLoader): text = "\n\n".join([str(el) for el in elements]) return text - async def async_evaluate(self, page, browser, response): + async def _async_evaluate(self, page, browser, response): """Process a page and return the text content asynchronously. Args: @@ -112,7 +112,7 @@ class PlaywrightURLLoader(BaseLoader): try: page = browser.new_page() response = page.goto(url) - text = self.sync_evaluate(page, browser, response) + text = self._sync_evaluate(page, browser, response) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: @@ -142,7 +142,7 @@ class PlaywrightURLLoader(BaseLoader): try: page = await browser.new_page() response = await page.goto(url) - text = await self.async_evaluate(page, browser, response) + text = await self._async_evaluate(page, browser, response) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: From 04fcd2d2e0aee574a0a26b416b4caa0610786b01 Mon Sep 17 00:00:00 2001 From: Youngwook Kim Date: Wed, 9 Aug 2023 14:14:59 +0900 Subject: [PATCH 5/6] refactor(document_loaders): introduce PlaywrightEvaluator abstract base class for custom evalutors and add tests --- .../document_loaders/url_playwright.py | 151 +++++++++++------- .../document_loaders/test_url_playwright.py | 39 +++++ 2 files changed, 131 insertions(+), 59 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/url_playwright.py b/libs/langchain/langchain/document_loaders/url_playwright.py index 48c8dcf0129..87db27fb7d8 100644 --- a/libs/langchain/langchain/document_loaders/url_playwright.py +++ b/libs/langchain/langchain/document_loaders/url_playwright.py @@ -1,6 +1,7 @@ """Loader that uses Playwright to load a page, then uses unstructured to load the html. """ import logging +from abc import ABC, abstractmethod from typing import List, Optional from langchain.docstore.document import Document @@ -9,8 +10,88 @@ from langchain.document_loaders.base import BaseLoader logger = logging.getLogger(__name__) +class PlaywrightEvaluator(ABC): + """Abstract base class for all evaluators. + + Each evaluator should take a page, a browser instance, and a response + object, process the page as necessary, and return the resulting text. + """ + + @abstractmethod + def evaluate(self, page, browser, response): + """Synchronously process the page and return the resulting text. + + Args: + page: The page to process. + browser: The browser instance. + response: The response from page.goto(). + + Returns: + text: The text content of the page. + """ + pass + + @abstractmethod + async def evaluate_async(self, page, browser, response): + """Asynchronously process the page and return the resulting text. + + Args: + page: The page to process. + browser: The browser instance. + response: The response from page.goto(). + + Returns: + text: The text content of the page. + """ + pass + + +class UnstructuredHtmlEvaluator(PlaywrightEvaluator): + """Evaluates the page HTML content using the `unstructured` library.""" + + def __init__(self, remove_selectors: Optional[List[str]] = None): + """Initialize UnstructuredHtmlEvaluator and check if `unstructured` package is installed.""" + try: + import unstructured # noqa:F401 + except ImportError: + raise ImportError( + "unstructured package not found, please install it with " + "`pip install unstructured`" + ) + + self.remove_selectors = remove_selectors + + def evaluate(self, page, browser, response): + """Synchronously process the HTML content of the page and return a text string.""" + from unstructured.partition.html import partition_html + + for selector in self.remove_selectors or []: + elements = page.locator(selector).all() + for element in elements: + if element.is_visible(): + element.evaluate("element => element.remove()") + + page_source = page.content() + elements = partition_html(text=page_source) + return "\n\n".join([str(el) for el in elements]) + + async def evaluate_async(self, page, browser, response): + """Asynchronously process the HTML content of the page and return a text string.""" + from unstructured.partition.html import partition_html + + for selector in self.remove_selectors or []: + elements = await page.locator(selector).all() + for element in elements: + if await element.is_visible(): + await element.evaluate("element => element.remove()") + + page_source = await page.content() + elements = partition_html(text=page_source) + return "\n\n".join([str(el) for el in elements]) + + class PlaywrightURLLoader(BaseLoader): - """Loader that uses Playwright and to load a page and unstructured to load the html. + """Loader that uses Playwright and PlaywrightEvaluator to load the html. This is useful for loading pages that require javascript to render. Attributes: @@ -25,8 +106,9 @@ class PlaywrightURLLoader(BaseLoader): continue_on_failure: bool = True, headless: bool = True, remove_selectors: Optional[List[str]] = None, + evaluator: Optional[PlaywrightEvaluator] = None, ): - """Load a list of URLs using Playwright and unstructured.""" + """Load a list of URLs using Playwright.""" try: import playwright # noqa:F401 except ImportError: @@ -35,66 +117,17 @@ class PlaywrightURLLoader(BaseLoader): "`pip install playwright`" ) - try: - import unstructured # noqa:F401 - except ImportError: - raise ImportError( - "unstructured package not found, please install it with " - "`pip install unstructured`" - ) - self.urls = urls self.continue_on_failure = continue_on_failure self.headless = headless - self.remove_selectors = remove_selectors - def _sync_evaluate(self, page, browser, response): - """Process a page and return the text content synchronously. + if remove_selectors and evaluator: + raise ValueError( + "`remove_selectors` and `evaluator` cannot be both not None" + ) - Args: - page: The page to process. - browser: The browser instance. - response: The response from page.goto(). - - Returns: - text: The text content of the page. - """ - from unstructured.partition.html import partition_html - - for selector in self.remove_selectors or []: - elements = page.locator(selector).all() - for element in elements: - if element.is_visible(): - element.evaluate("element => element.remove()") - - page_source = page.content() - elements = partition_html(text=page_source) - text = "\n\n".join([str(el) for el in elements]) - return text - - async def _async_evaluate(self, page, browser, response): - """Process a page and return the text content asynchronously. - - Args: - page: The page to process. - browser: The browser instance. - response: The response from page.goto(). - - Returns: - text: The text content of the page. - """ - from unstructured.partition.html import partition_html - - for selector in self.remove_selectors or []: - elements = await page.locator(selector).all() - for element in elements: - if await element.is_visible(): - await element.evaluate("element => element.remove()") - - page_source = await page.content() - elements = partition_html(text=page_source) - text = "\n\n".join([str(el) for el in elements]) - return text + # Use the provided evaluator, if any, otherwise, use the default UnstructuredHtmlEvaluator. + self.evaluator = evaluator or UnstructuredHtmlEvaluator(remove_selectors) def load(self) -> List[Document]: """Load the specified URLs using Playwright and create Document instances. @@ -112,7 +145,7 @@ class PlaywrightURLLoader(BaseLoader): try: page = browser.new_page() response = page.goto(url) - text = self._sync_evaluate(page, browser, response) + text = self.evaluator.evaluate(page, browser, response) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: @@ -142,7 +175,7 @@ class PlaywrightURLLoader(BaseLoader): try: page = await browser.new_page() response = await page.goto(url) - text = await self._async_evaluate(page, browser, response) + text = await self.evaluator.evaluate_async(page, browser, response) metadata = {"source": url} docs.append(Document(page_content=text, metadata=metadata)) except Exception as e: diff --git a/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py b/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py index 565646428b2..451d0d71384 100644 --- a/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py +++ b/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py @@ -4,6 +4,16 @@ import pytest from langchain.document_loaders import PlaywrightURLLoader +class TestEvaluator(PageEvaluator): + """A simple evaluator for testing purposes.""" + + def evaluate(self, page, browser, response): + return "test" + + async def evaluate_async(self, page, browser, response): + return "test" + + def test_playwright_url_loader() -> None: """Test Playwright URL loader.""" urls = [ @@ -39,3 +49,32 @@ async def test_playwright_async_url_loader() -> None: ) docs = await loader.aload() assert len(docs) > 0 + + +def test_playwright_url_loader_with_custom_evaluator() -> None: + """Test Playwright URL loader with a custom evaluator.""" + urls = ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"] + loader = PlaywrightURLLoader( + urls=urls, + page_evaluator=TestEvaluator(), + continue_on_failure=False, + headless=True, + ) + docs = loader.load() + assert len(docs) == 1 + assert docs[0].page_content == "test-" + + +@pytest.mark.asyncio +async def test_playwright_async_url_loader_with_custom_evaluator() -> None: + """Test Playwright async URL loader with a custom evaluator.""" + urls = ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"] + loader = PlaywrightURLLoader( + urls=urls, + page_evaluator=TestEvaluator(), + continue_on_failure=False, + headless=True, + ) + docs = await loader.aload() + assert len(docs) == 2 + assert docs[0].page_content == "test" From 429de77b3b644232ed64b0c9beaa7486bc2d4b79 Mon Sep 17 00:00:00 2001 From: Youngwook Kim Date: Wed, 9 Aug 2023 15:56:24 +0900 Subject: [PATCH 6/6] refactor(langchain): improve type annotations in url_playwright and its test --- .../document_loaders/url_playwright.py | 27 ++++++++++++------- .../document_loaders/test_url_playwright.py | 23 +++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/url_playwright.py b/libs/langchain/langchain/document_loaders/url_playwright.py index 87db27fb7d8..bce53f91559 100644 --- a/libs/langchain/langchain/document_loaders/url_playwright.py +++ b/libs/langchain/langchain/document_loaders/url_playwright.py @@ -2,11 +2,16 @@ """ import logging from abc import ABC, abstractmethod -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from langchain.docstore.document import Document from langchain.document_loaders.base import BaseLoader +if TYPE_CHECKING: + from playwright.async_api import AsyncBrowser, AsyncPage, AsyncResponse + from playwright.sync_api import Browser, Page, Response + + logger = logging.getLogger(__name__) @@ -18,7 +23,7 @@ class PlaywrightEvaluator(ABC): """ @abstractmethod - def evaluate(self, page, browser, response): + def evaluate(self, page: "Page", browser: "Browser", response: "Response") -> str: """Synchronously process the page and return the resulting text. Args: @@ -32,7 +37,9 @@ class PlaywrightEvaluator(ABC): pass @abstractmethod - async def evaluate_async(self, page, browser, response): + async def evaluate_async( + self, page: "AsyncPage", browser: "AsyncBrowser", response: "AsyncResponse" + ) -> str: """Asynchronously process the page and return the resulting text. Args: @@ -50,7 +57,7 @@ class UnstructuredHtmlEvaluator(PlaywrightEvaluator): """Evaluates the page HTML content using the `unstructured` library.""" def __init__(self, remove_selectors: Optional[List[str]] = None): - """Initialize UnstructuredHtmlEvaluator and check if `unstructured` package is installed.""" + """Initialize UnstructuredHtmlEvaluator.""" try: import unstructured # noqa:F401 except ImportError: @@ -61,8 +68,8 @@ class UnstructuredHtmlEvaluator(PlaywrightEvaluator): self.remove_selectors = remove_selectors - def evaluate(self, page, browser, response): - """Synchronously process the HTML content of the page and return a text string.""" + def evaluate(self, page: "Page", browser: "Browser", response: "Response") -> str: + """Synchronously process the HTML content of the page.""" from unstructured.partition.html import partition_html for selector in self.remove_selectors or []: @@ -75,8 +82,10 @@ class UnstructuredHtmlEvaluator(PlaywrightEvaluator): elements = partition_html(text=page_source) return "\n\n".join([str(el) for el in elements]) - async def evaluate_async(self, page, browser, response): - """Asynchronously process the HTML content of the page and return a text string.""" + async def evaluate_async( + self, page: "AsyncPage", browser: "AsyncBrowser", response: "AsyncResponse" + ) -> str: + """Asynchronously process the HTML content of the page.""" from unstructured.partition.html import partition_html for selector in self.remove_selectors or []: @@ -126,7 +135,7 @@ class PlaywrightURLLoader(BaseLoader): "`remove_selectors` and `evaluator` cannot be both not None" ) - # Use the provided evaluator, if any, otherwise, use the default UnstructuredHtmlEvaluator. + # Use the provided evaluator, if any, otherwise, use the default. self.evaluator = evaluator or UnstructuredHtmlEvaluator(remove_selectors) def load(self) -> List[Document]: diff --git a/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py b/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py index 451d0d71384..7bea1c6dee7 100644 --- a/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py +++ b/libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py @@ -1,16 +1,25 @@ """Tests for the Playwright URL loader""" +from typing import TYPE_CHECKING + import pytest from langchain.document_loaders import PlaywrightURLLoader +from langchain.document_loaders.url_playwright import PlaywrightEvaluator + +if TYPE_CHECKING: + from playwright.async_api import AsyncBrowser, AsyncPage, AsyncResponse + from playwright.sync_api import Browser, Page, Response -class TestEvaluator(PageEvaluator): +class TestEvaluator(PlaywrightEvaluator): """A simple evaluator for testing purposes.""" - def evaluate(self, page, browser, response): + def evaluate(self, page: "Page", browser: "Browser", response: "Response") -> str: return "test" - async def evaluate_async(self, page, browser, response): + async def evaluate_async( + self, page: "AsyncPage", browser: "AsyncBrowser", response: "AsyncResponse" + ) -> str: return "test" @@ -56,13 +65,13 @@ def test_playwright_url_loader_with_custom_evaluator() -> None: urls = ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"] loader = PlaywrightURLLoader( urls=urls, - page_evaluator=TestEvaluator(), + evaluator=TestEvaluator(), continue_on_failure=False, headless=True, ) docs = loader.load() assert len(docs) == 1 - assert docs[0].page_content == "test-" + assert docs[0].page_content == "test" @pytest.mark.asyncio @@ -71,10 +80,10 @@ async def test_playwright_async_url_loader_with_custom_evaluator() -> None: urls = ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"] loader = PlaywrightURLLoader( urls=urls, - page_evaluator=TestEvaluator(), + evaluator=TestEvaluator(), continue_on_failure=False, headless=True, ) docs = await loader.aload() - assert len(docs) == 2 + assert len(docs) == 1 assert docs[0].page_content == "test"