From 422ba4cde57e5f2026d5fa0b185b8d68a00d4bfe Mon Sep 17 00:00:00 2001 From: ccurme Date: Wed, 26 Mar 2025 13:28:56 -0400 Subject: [PATCH] infra: handle flaky tests (#30501) --- libs/partners/anthropic/pyproject.toml | 1 + .../integration_tests/test_chat_models.py | 1 + libs/partners/anthropic/uv.lock | 18 ++++++++++++++++-- libs/partners/openai/pyproject.toml | 1 + .../chat_models/test_responses_api.py | 6 ++++++ libs/partners/openai/uv.lock | 16 +++++++++++++++- 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/libs/partners/anthropic/pyproject.toml b/libs/partners/anthropic/pyproject.toml index e3e96892ff5..8be13a8e10b 100644 --- a/libs/partners/anthropic/pyproject.toml +++ b/libs/partners/anthropic/pyproject.toml @@ -30,6 +30,7 @@ test = [ "pytest-watcher<1.0.0,>=0.3.4", "pytest-asyncio<1.0.0,>=0.21.1", "defusedxml<1.0.0,>=0.7.1", + "pytest-retry<1.8.0,>=1.7.0", "pytest-timeout<3.0.0,>=2.3.1", "pytest-socket<1.0.0,>=0.7.0", "langchain-core", diff --git a/libs/partners/anthropic/tests/integration_tests/test_chat_models.py b/libs/partners/anthropic/tests/integration_tests/test_chat_models.py index e792c1583fd..fc24c7d515c 100644 --- a/libs/partners/anthropic/tests/integration_tests/test_chat_models.py +++ b/libs/partners/anthropic/tests/integration_tests/test_chat_models.py @@ -730,6 +730,7 @@ def test_thinking() -> None: assert block["signature"] and isinstance(block["signature"], str) +@pytest.mark.flaky(retries=3, delay=1) def test_redacted_thinking() -> None: llm = ChatAnthropic( model="claude-3-7-sonnet-latest", diff --git a/libs/partners/anthropic/uv.lock b/libs/partners/anthropic/uv.lock index 401ae87f754..d9d0c5a1e36 100644 --- a/libs/partners/anthropic/uv.lock +++ b/libs/partners/anthropic/uv.lock @@ -432,6 +432,7 @@ test = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-mock" }, + { name = "pytest-retry" }, { name = "pytest-socket" }, { name = "pytest-timeout" }, { name = "pytest-watcher" }, @@ -466,6 +467,7 @@ test = [ { name = "pytest", specifier = ">=7.3.0,<8.0.0" }, { name = "pytest-asyncio", specifier = ">=0.21.1,<1.0.0" }, { name = "pytest-mock", specifier = ">=3.10.0,<4.0.0" }, + { name = "pytest-retry", specifier = ">=1.7.0,<1.8.0" }, { name = "pytest-socket", specifier = ">=0.7.0,<1.0.0" }, { name = "pytest-timeout", specifier = ">=2.3.1,<3.0.0" }, { name = "pytest-watcher", specifier = ">=0.3.4,<1.0.0" }, @@ -483,7 +485,7 @@ typing = [ [[package]] name = "langchain-core" -version = "0.3.45" +version = "0.3.48" source = { editable = "../../core" } dependencies = [ { name = "jsonpatch" }, @@ -541,7 +543,7 @@ typing = [ [[package]] name = "langchain-tests" -version = "0.3.14" +version = "0.3.16" source = { editable = "../../standard-tests" } dependencies = [ { name = "httpx" }, @@ -1009,6 +1011,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/3b/b26f90f74e2986a82df6e7ac7e319b8ea7ccece1caec9f8ab6104dc70603/pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f", size = 9863 }, ] +[[package]] +name = "pytest-retry" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/5b/607b017994cca28de3a1ad22a3eee8418e5d428dcd8ec25b26b18e995a73/pytest_retry-1.7.0.tar.gz", hash = "sha256:f8d52339f01e949df47c11ba9ee8d5b362f5824dff580d3870ec9ae0057df80f", size = 19977 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ff/3266c8a73b9b93c4b14160a7e2b31d1e1088e28ed29f4c2d93ae34093bfd/pytest_retry-1.7.0-py3-none-any.whl", hash = "sha256:a2dac85b79a4e2375943f1429479c65beb6c69553e7dae6b8332be47a60954f4", size = 13775 }, +] + [[package]] name = "pytest-socket" version = "0.7.0" diff --git a/libs/partners/openai/pyproject.toml b/libs/partners/openai/pyproject.toml index ce23eca19f0..42049ff477b 100644 --- a/libs/partners/openai/pyproject.toml +++ b/libs/partners/openai/pyproject.toml @@ -30,6 +30,7 @@ test = [ "pytest-watcher<1.0.0,>=0.3.4", "pytest-asyncio<1.0.0,>=0.21.1", "pytest-cov<5.0.0,>=4.1.0", + "pytest-retry<1.8.0,>=1.7.0", "pytest-socket<1.0.0,>=0.6.0", "pytest-xdist<4.0.0,>=3.6.1", "numpy<2,>=1; python_version < \"3.12\"", diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py index 8824560eae2..9eb6be78613 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_responses_api.py @@ -53,6 +53,7 @@ def _check_response(response: Optional[BaseMessage]) -> None: assert tool_output["type"] +@pytest.mark.flaky(retries=3, delay=1) def test_web_search() -> None: llm = ChatOpenAI(model=MODEL_NAME) first_response = llm.invoke( @@ -108,6 +109,7 @@ def test_web_search() -> None: _check_response(response) +@pytest.mark.flaky(retries=3, delay=1) async def test_web_search_async() -> None: llm = ChatOpenAI(model=MODEL_NAME) response = await llm.ainvoke( @@ -129,6 +131,7 @@ async def test_web_search_async() -> None: _check_response(full) +@pytest.mark.flaky(retries=3, delay=1) def test_function_calling() -> None: def multiply(x: int, y: int) -> int: """return x * y""" @@ -197,6 +200,7 @@ async def test_parsed_pydantic_schema_async() -> None: assert parsed.response +@pytest.mark.flaky(retries=3, delay=1) @pytest.mark.parametrize("schema", [Foo.model_json_schema(), FooDict]) def test_parsed_dict_schema(schema: Any) -> None: llm = ChatOpenAI(model=MODEL_NAME, use_responses_api=True) @@ -241,6 +245,7 @@ def test_parsed_strict() -> None: ) +@pytest.mark.flaky(retries=3, delay=1) @pytest.mark.parametrize("schema", [Foo.model_json_schema(), FooDict]) async def test_parsed_dict_schema_async(schema: Any) -> None: llm = ChatOpenAI(model=MODEL_NAME, use_responses_api=True) @@ -313,6 +318,7 @@ def test_route_from_model_kwargs() -> None: _ = next(llm.stream("Hello")) +@pytest.mark.flaky(retries=3, delay=1) def test_computer_calls() -> None: llm = ChatOpenAI(model="computer-use-preview", model_kwargs={"truncation": "auto"}) tool = { diff --git a/libs/partners/openai/uv.lock b/libs/partners/openai/uv.lock index df15053ab76..ccfa59aea4d 100644 --- a/libs/partners/openai/uv.lock +++ b/libs/partners/openai/uv.lock @@ -547,6 +547,7 @@ test = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-mock" }, + { name = "pytest-retry" }, { name = "pytest-socket" }, { name = "pytest-watcher" }, { name = "pytest-xdist" }, @@ -584,6 +585,7 @@ test = [ { name = "pytest-asyncio", specifier = ">=0.21.1,<1.0.0" }, { name = "pytest-cov", specifier = ">=4.1.0,<5.0.0" }, { name = "pytest-mock", specifier = ">=3.10.0,<4.0.0" }, + { name = "pytest-retry", specifier = ">=1.7.0,<1.8.0" }, { name = "pytest-socket", specifier = ">=0.6.0,<1.0.0" }, { name = "pytest-watcher", specifier = ">=0.3.4,<1.0.0" }, { name = "pytest-xdist", specifier = ">=3.6.1,<4.0.0" }, @@ -603,7 +605,7 @@ typing = [ [[package]] name = "langchain-tests" -version = "0.3.15" +version = "0.3.16" source = { editable = "../../standard-tests" } dependencies = [ { name = "httpx" }, @@ -1110,6 +1112,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/3b/b26f90f74e2986a82df6e7ac7e319b8ea7ccece1caec9f8ab6104dc70603/pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f", size = 9863 }, ] +[[package]] +name = "pytest-retry" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/5b/607b017994cca28de3a1ad22a3eee8418e5d428dcd8ec25b26b18e995a73/pytest_retry-1.7.0.tar.gz", hash = "sha256:f8d52339f01e949df47c11ba9ee8d5b362f5824dff580d3870ec9ae0057df80f", size = 19977 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ff/3266c8a73b9b93c4b14160a7e2b31d1e1088e28ed29f4c2d93ae34093bfd/pytest_retry-1.7.0-py3-none-any.whl", hash = "sha256:a2dac85b79a4e2375943f1429479c65beb6c69553e7dae6b8332be47a60954f4", size = 13775 }, +] + [[package]] name = "pytest-socket" version = "0.7.0"