Install dev, lint, test, typing extra deps for linting steps. (#10249)

`mypy` cannot type-check code that relies on dependencies that aren't
installed.

Eventually we'll probably want to install as many optional dependencies
as possible. However, the full "extended deps" setup for langchain
creates a 3GB cache file and takes a while to unpack and install. We'll
probably want something a bit more targeted.

This is a first step toward something better.

.github/workflows/_lint.yml

@@ -87,7 +87,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           poetry-version: ${{ env.POETRY_VERSION }}
           working-directory: ${{ inputs.working-directory }}
-          cache-key: lint
+          cache-key: lint-with-extras
 
       - name: Check Poetry File
         shell: bash
@@ -102,9 +102,17 @@ jobs:
           poetry lock --check
 
       - name: Install dependencies
+        # Also installs dev/lint/test/typing dependencies, to ensure we have
+        # type hints for as many of our libraries as possible.
+        # This helps catch errors that require dependencies to be spotted, for example:
+        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
+        #
+        # If you change this configuration, make sure to change the `cache-key`
+        # in the `poetry_setup` action above to stop using the old cache.
+        # It doesn't matter how you change it, any change will cause a cache-bust.
         working-directory: ${{ inputs.working-directory }}
         run: |
-          poetry install
+          poetry install --with dev,lint,test,typing
 
       - name: Install langchain editable
         working-directory: ${{ inputs.working-directory }}

libs/langchain/langchain/document_loaders/url_playwright.py

@@ -8,7 +8,9 @@ from langchain.docstore.document import Document
 from langchain.document_loaders.base import BaseLoader
 
 if TYPE_CHECKING:
-    from playwright.async_api import AsyncBrowser, AsyncPage, AsyncResponse
+    from playwright.async_api import Browser as AsyncBrowser
+    from playwright.async_api import Page as AsyncPage
+    from playwright.async_api import Response as AsyncResponse
     from playwright.sync_api import Browser, Page, Response
 
 
@@ -155,6 +157,9 @@ class PlaywrightURLLoader(BaseLoader):
                 try:
                     page = browser.new_page()
                     response = page.goto(url)
+                    if response is None:
+                        raise ValueError(f"page.goto() returned None for url {url}")
+
                     text = self.evaluator.evaluate(page, browser, response)
                     metadata = {"source": url}
                     docs.append(Document(page_content=text, metadata=metadata))
@@ -185,6 +190,9 @@ class PlaywrightURLLoader(BaseLoader):
                 try:
                     page = await browser.new_page()
                     response = await page.goto(url)
+                    if response is None:
+                        raise ValueError(f"page.goto() returned None for url {url}")
+
                     text = await self.evaluator.evaluate_async(page, browser, response)
                     metadata = {"source": url}
                     docs.append(Document(page_content=text, metadata=metadata))

libs/langchain/tests/integration_tests/document_loaders/test_url_playwright.py

@@ -7,7 +7,9 @@ from langchain.document_loaders import PlaywrightURLLoader
 from langchain.document_loaders.url_playwright import PlaywrightEvaluator
 
 if TYPE_CHECKING:
-    from playwright.async_api import AsyncBrowser, AsyncPage, AsyncResponse
+    from playwright.async_api import Browser as AsyncBrowser
+    from playwright.async_api import Page as AsyncPage
+    from playwright.async_api import Response as AsyncResponse
     from playwright.sync_api import Browser, Page, Response