github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-09-10 07:21:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Install dev, lint, test, typing extra deps for linting steps. (#10249)

Browse Source 
`mypy` cannot type-check code that relies on dependencies that aren't
installed.

Eventually we'll probably want to install as many optional dependencies
as possible. However, the full "extended deps" setup for langchain
creates a 3GB cache file and takes a while to unpack and install. We'll
probably want something a bit more targeted.

This is a first step toward something better.
This commit is contained in:
Predrag Gruevski
2023-09-06 11:15:28 -04:00
committed by GitHub
parent 82d5d4d0ae
commit ccb9e3ee2d
3 changed files with 22 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
libs/langchain/langchain/document_loaders/url_playwright.py
View File

@@ -8,7 +8,9 @@ from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
from playwright.async_api import AsyncBrowser, AsyncPage, AsyncResponse
from playwright.async_api import Browser as AsyncBrowser
from playwright.async_api import Page as AsyncPage
from playwright.async_api import Response as AsyncResponse
from playwright.sync_api import Browser, Page, Response
@@ -155,6 +157,9 @@ class PlaywrightURLLoader(BaseLoader):
try:
page = browser.new_page()
response = page.goto(url)
if response is None:
raise ValueError(f"page.goto() returned None for url {url}")
text = self.evaluator.evaluate(page, browser, response)
metadata = {"source": url}
docs.append(Document(page_content=text, metadata=metadata))
@@ -185,6 +190,9 @@ class PlaywrightURLLoader(BaseLoader):
try:
page = await browser.new_page()
response = await page.goto(url)
if response is None:
raise ValueError(f"page.goto() returned None for url {url}")
text = await self.evaluator.evaluate_async(page, browser, response)
metadata = {"source": url}
docs.append(Document(page_content=text, metadata=metadata))

4
libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py
View File

@@ -7,7 +7,9 @@ from langchain.document_loaders import PlaywrightURLLoader
from langchain.document_loaders.url_playwright import PlaywrightEvaluator
if TYPE_CHECKING:
from playwright.async_api import AsyncBrowser, AsyncPage, AsyncResponse
from playwright.async_api import Browser as AsyncBrowser
from playwright.async_api import Page as AsyncPage
from playwright.async_api import Response as AsyncResponse
from playwright.sync_api import Browser, Page, Response