From f7efd89e70d1332f68fbf34260023e2b0cb0cf72 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 28 May 2025 10:56:32 -0400 Subject: [PATCH] document vcr tests --- .../integration_tests/chat_models.py | 111 +++++++++++++++++- .../langchain_tests/unit_tests/chat_models.py | 52 ++++++-- 2 files changed, 149 insertions(+), 14 deletions(-) diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index bc00ef1b895..e423d659960 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -521,7 +521,92 @@ class ChatModelIntegrationTests(ChatModelTests): cached, audio, or reasoning. Only needs to be overridden if these details are supplied. - """ + + .. dropdown:: enable_vcr_tests + + Property controlling whether to enable select tests that rely on + `VCR `_ caching of HTTP calls, such + as benchmarking tests. + + To enable these tests, follow these steps: + + 1. Override the ``enable_vcr_tests`` property to return ``True``: + + .. code-block:: python + + @property + def enable_vcr_tests(self) -> bool: + return True + + 2. Configure VCR to exclude sensitive headers and other information from cassettes. + + .. important:: + VCR will by default record authentication headers and other sensitive + information in cassettes. Read below for how to configure what + information is recorded in cassettes. + + To add configuration to VCR, add a ``conftest.py`` file to the ``tests/`` + directory and implement the ``vcr_config`` fixture there. + + ``langchain-tests`` excludes the headers ``"authorization"``, + ``"x-api-key"``, and ``"api-key"`` from VCR cassettes. To pick up this + configuration, you will need to add ``conftest.py`` as shown below. You can + also exclude additional headers, override the default exclusions, or apply + other customizations to the VCR configuration. See example below: + + .. code-block:: python + :caption: tests/conftest.py + + import pytest + from langchain_tests.conftest import _base_vcr_config as _base_vcr_config + + _EXTRA_HEADERS = [ + # Specify additional headers to redact + ("user-agent", "PLACEHOLDER"), + ] + + + def remove_response_headers(response: dict) -> dict: + # If desired, remove or modify headers in the response. + response["headers"] = {} + return response + + + @pytest.fixture(scope="session") + def vcr_config(_base_vcr_config: dict) -> dict: # noqa: F811 + \"\"\"Extend the default configuration from langchain_tests.\"\"\" + config = _base_vcr_config.copy() + config.setdefault("filter_headers", []).extend(_EXTRA_HEADERS) + config["before_record_response"] = remove_response_headers + + return config + + 3. Run tests to generate VCR cassettes. + + Example: + + .. code-block:: bash + + uv run python -m pytest tests/integration_tests/test_chat_models.py::TestMyModel::test_stream_time + + This will generate a VCR cassette for the test in + ``tests/integration_tests/cassettes/``. + + .. important:: + You should inspect the generated cassette to ensure that it does not + contain sensitive information. If it does, you can modify the + ``vcr_config`` fixture to exclude headers or modify the response + before it is recorded. + + You can then commit the cassette to your repository. Subsequent test runs + will use the cassette instead of making HTTP calls. + + .. tip:: + Adding ``--vcr-record=none`` to the pytest command will ensure that + no new cassettes are recorded, and only existing cassettes are used. + Consider adding this to your CI configuration (e.g., modify relevant + Makefile commands). + """ # noqa: E501 @property def standard_chat_model_params(self) -> dict: @@ -2685,6 +2770,30 @@ class ChatModelIntegrationTests(ChatModelTests): def test_stream_time( self, model: BaseChatModel, benchmark: BenchmarkFixture, vcr: vcr.VCR ) -> None: + """Test that streaming does not introduce undue overhead. + + See ``enable_vcr_tests`` dropdown :class:`above ` + for more information. + + .. dropdown:: Configuration + + This test can be enabled or disabled using the ``enable_vcr_tests`` + property. For example, to disable the test, set this property to ``False``: + + .. code-block:: python + + @property + def enable_vcr_tests(self) -> bool: + return False + + .. important:: + + VCR will by default record authentication headers and other sensitive + information in cassettes. See ``enable_vcr_tests`` dropdown + :class:`above ` for how to configure what + information is recorded in cassettes. + + """ if not self.enable_vcr_tests: pytest.skip("VCR not set up.") diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py index 0bc4e7553e1..52921b6ac43 100644 --- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py @@ -203,7 +203,12 @@ class ChatModelTests(BaseStandardTests): @property def enable_vcr_tests(self) -> bool: - """(bool) whether to enable VCR tests for the chat model.""" + """(bool) whether to enable VCR tests for the chat model. + + .. important:: + See ``enable_vcr_tests`` dropdown :class:`above ` for more + information. + """ return False @property @@ -633,18 +638,17 @@ class ChatModelUnitTests(ChatModelTests): @property def enable_vcr_tests(self) -> bool: return True - - 2. Configure VCR to exclude sensitive headers and other information from VCR - cassettes. - .. warning:: Excluding secrets from VCR cassettes. - VCR will by default record authentication headers and other sensitive - information in cassettes. Read below for how to configure what information - is recorded in cassettes. + 2. Configure VCR to exclude sensitive headers and other information from cassettes. + + .. important:: + VCR will by default record authentication headers and other sensitive + information in cassettes. Read below for how to configure what + information is recorded in cassettes. To add configuration to VCR, add a ``conftest.py`` file to the ``tests/`` directory and implement the ``vcr_config`` fixture there. - + ``langchain-tests`` excludes the headers ``"authorization"``, ``"x-api-key"``, and ``"api-key"`` from VCR cassettes. To pick up this configuration, you will need to add ``conftest.py`` as shown below. You can @@ -658,10 +662,8 @@ class ChatModelUnitTests(ChatModelTests): from langchain_tests.conftest import _base_vcr_config as _base_vcr_config _EXTRA_HEADERS = [ - # Specify additional headers to exclude - ("openai-organization", "PLACEHOLDER"), + # Specify additional headers to redact ("user-agent", "PLACEHOLDER"), - ("x-openai-client-user-agent", "PLACEHOLDER"), ] @@ -680,7 +682,31 @@ class ChatModelUnitTests(ChatModelTests): return config - 3. Run the tests to generate VCR cassettes. + 3. Run tests to generate VCR cassettes. + + Example: + + .. code-block:: bash + + uv run python -m pytest tests/integration_tests/test_chat_models.py::TestMyModel::test_stream_time + + This will generate a VCR cassette for the test in + ``tests/integration_tests/cassettes/``. + + .. important:: + You should inspect the generated cassette to ensure that it does not + contain sensitive information. If it does, you can modify the + ``vcr_config`` fixture to exclude headers or modify the response + before it is recorded. + + You can then commit the cassette to your repository. Subsequent test runs + will use the cassette instead of making HTTP calls. + + .. tip:: + Adding ``--vcr-record=none`` to the pytest command will ensure that + no new cassettes are recorded, and only existing cassettes are used. + Consider adding this to your CI configuration (e.g., modify relevant + Makefile commands). Testing initialization from environment variables