fix sched ci (more) (#9056)

.github/workflows/scheduled_test.yml

@@ -30,9 +30,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           poetry-version: "1.4.2"
+          working-directory: libs/langchain
           install-command: |
             echo "Running scheduled tests, installing dependencies with poetry..."
-            poetry install -E scheduled_testing
+            poetry install --with=test_integration
       - name: Run tests
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

libs/langchain/pyproject.toml

@@ -177,32 +177,8 @@ optional = true
 pytest-vcr = "^1.0.2"
 wrapt = "^1.15.0"
 openai = "^0.27.4"
-elasticsearch = {extras = ["async"], version = "^8.6.2"}
-redis = "^4.5.4"
-pinecone-client = "^2.2.1"
-pinecone-text = "^0.4.2"
-pymongo = "^4.3.3"
-clickhouse-connect = "^0.5.14"
-transformers = "^4.27.4"
-deeplake = "^3.6.8"
-libdeeplake = "^0.0.60"
-weaviate-client = "^3.15.5"
-torch = "^1.0.0"
-chromadb = "^0.4.0"
-tiktoken = "^0.3.3"
 python-dotenv = "^1.0.0"
-sentence-transformers = "^2"
-gptcache = "^0.1.9"
-promptlayer = "^0.1.80"
-tair = "^1.3.3"
-wikipedia = "^1"
 cassio = "^0.0.7"
-arxiv = "^1.4"
-mastodon-py = "^1.8.1"
-momento = "^1.5.0"
-# Please do not add any dependencies in the test_integration group
-# See instructions above ^^
-pygithub = "^1.59.0"
 
 [tool.poetry.group.lint.dependencies]
 ruff = "^0.0.249"
@@ -376,10 +352,6 @@ extended_testing = [
  "anthropic",
 ]
 
-scheduled_testing = [
- "openai",
-]
-
 [tool.ruff]
 select = [
   "E",  # pycodestyle

libs/langchain/tests/integration_tests/document_loaders/test_geodataframe.py

@@ -1,6 +1,7 @@
+from __future__ import annotations
+
 from typing import TYPE_CHECKING
 
-import geopandas
 import pytest
 
 from langchain.document_loaders import GeoDataFrameLoader
@@ -14,6 +15,8 @@ else:
 
 @pytest.mark.requires("geopandas")
 def sample_gdf() -> GeoDataFrame:
+    import geopandas
+
     path_to_data = geopandas.datasets.get_path("nybb")
     gdf = geopandas.read_file(path_to_data)
     gdf["area"] = gdf.area

libs/langchain/tests/integration_tests/document_loaders/test_news.py

@@ -2,12 +2,13 @@ import random
 
 import pytest
 import requests
-from bs4 import BeautifulSoup
 
 from langchain.document_loaders import NewsURLLoader
 
 
 def get_random_news_url() -> str:
+    from bs4 import BeautifulSoup
+
     response = requests.get("https://news.google.com")
     soup = BeautifulSoup(response.text, "html.parser")
 

libs/langchain/tests/integration_tests/document_loaders/test_tensorflow_datasets.py

@@ -1,4 +1,7 @@
 """Integration tests for the TensorFlow Dataset Loader."""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 import pytest
 from pydantic.error_wrappers import ValidationError
@@ -6,13 +9,8 @@ from pydantic.error_wrappers import ValidationError
 from langchain.document_loaders.tensorflow_datasets import TensorflowDatasetLoader
 from langchain.schema.document import Document
 
-# adding tensorflow and tensorflow_datasets to pyproject.toml is not working
+if TYPE_CHECKING:
-# these tests can be run in isolation only
+    import tensorflow as tf  # noqa: E402
-tensorflow = pytest.importorskip("tensorflow")
-tensorflow_datasets = pytest.importorskip("tensorflow_datasets")
-
-# placed here after checking for tensorflow package installation
-import tensorflow as tf  # noqa: E402
 
 
 def decode_to_str(item: tf.Tensor) -> str:

libs/langchain/tests/integration_tests/embeddings/test_self_hosted.py

@@ -1,8 +1,6 @@
 """Test self-hosted embeddings."""
 from typing import Any
 
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-
 from langchain.embeddings import (
     SelfHostedEmbeddings,
     SelfHostedHuggingFaceEmbeddings,
@@ -59,6 +57,8 @@ def test_self_hosted_huggingface_instructor_embedding_query() -> None:
 
 def get_pipeline() -> Any:
     """Get pipeline for testing."""
+    from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
     model_id = "facebook/bart-base"
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     model = AutoModelForCausalLM.from_pretrained(model_id)

libs/langchain/tests/integration_tests/llms/test_huggingface_pipeline.py

@@ -2,8 +2,6 @@
 
 from pathlib import Path
 
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-
 from langchain.llms.huggingface_pipeline import HuggingFacePipeline
 from langchain.llms.loading import load_llm
 from tests.integration_tests.llms.utils import assert_llm_equality
@@ -48,6 +46,8 @@ def test_saving_loading_llm(tmp_path: Path) -> None:
 
 def test_init_with_pipeline() -> None:
     """Test initialization with a HF pipeline."""
+    from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
     model_id = "gpt2"
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     model = AutoModelForCausalLM.from_pretrained(model_id)

libs/langchain/tests/integration_tests/llms/test_self_hosted_llm.py

@@ -2,8 +2,6 @@
 import pickle
 from typing import Any, List, Optional
 
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-
 from langchain.llms import SelfHostedHuggingFaceLLM, SelfHostedPipeline
 
 model_reqs = ["pip:./", "transformers", "torch"]
@@ -58,6 +56,8 @@ def test_self_hosted_huggingface_pipeline_summarization() -> None:
 
 def load_pipeline() -> Any:
     """Load pipeline for testing."""
+    from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
     model_id = "gpt2"
     tokenizer = AutoTokenizer.from_pretrained(model_id)
     model = AutoModelForCausalLM.from_pretrained(model_id)

libs/langchain/tests/integration_tests/memory/test_rockset.py

@@ -8,8 +8,6 @@ and ROCKSET_REGION environment variables set.
 import json
 import os
 
-from rockset import DevRegions, Regions, RocksetClient
-
 from langchain.memory import ConversationBufferMemory
 from langchain.memory.chat_message_histories import RocksetChatMessageHistory
 from langchain.schema.messages import _message_to_dict
@@ -23,6 +21,8 @@ class TestRockset:
 
     @classmethod
     def setup_class(cls) -> None:
+        from rockset import DevRegions, Regions, RocksetClient
+
         assert os.environ.get("ROCKSET_API_KEY") is not None
         assert os.environ.get("ROCKSET_REGION") is not None
 

libs/langchain/tests/integration_tests/storage/test_redis.py

@@ -2,23 +2,13 @@
 import os
 import typing
 import uuid
-from typing import Any
 
 import pytest
-import redis
 
 from langchain.storage.redis import RedisStore
 
 if typing.TYPE_CHECKING:
-    try:
+    from redis import Redis
-        from redis import Redis
-    except ImportError:
-        # Ignoring mypy here to allow assignment of Any to Redis in the event
-        # that redis is not installed.
-        Redis = Any  # type:ignore
-else:
-    Redis = Any  # type:ignore
-
 
 pytest.importorskip("redis")
 
@@ -26,6 +16,8 @@ pytest.importorskip("redis")
 @pytest.fixture
 def redis_client() -> Redis:
     """Yield redis client."""
+    import redis
+
     # Using standard port, but protecting against accidental data loss
     # by requiring a password.
     # This fixture flushes the database!

libs/langchain/tests/integration_tests/utilities/test_tensorflow_datasets.py

@@ -1,16 +1,16 @@
 """Integration tests for the TensorFlow Dataset client."""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 import pytest
-import tensorflow as tf
 from pydantic.error_wrappers import ValidationError
 
 from langchain.schema.document import Document
 from langchain.utilities.tensorflow_datasets import TensorflowDatasets
 
-# adding tensorflow and tensorflow_datasets to pyproject.toml is not working
+if TYPE_CHECKING:
-# these tests can be tested in isolation only
+    import tensorflow as tf  # noqa: E402
-tensorflow = pytest.importorskip("tensorflow")
-tensorflow_datasets = pytest.importorskip("tensorflow_datasets")
 
 
 def decode_to_str(item: tf.Tensor) -> str: