mirror of
https://github.com/hwchase17/langchain.git
synced 2026-06-09 10:17:00 +00:00
269d628d9c
`test_no_overrides_DO_NOT_OVERRIDE` only treated an override as valid when the method itself carried an `@pytest.mark.xfail(reason=...)`. Overrides that re-parametrize a standard test and xfail only a subset of cases via `pytest.param(..., marks=pytest.mark.xfail(...))` were rejected.
71 lines
2.8 KiB
Python
71 lines
2.8 KiB
Python
"""Standard tests."""
|
|
|
|
|
|
class BaseStandardTests:
|
|
"""Base class for standard tests."""
|
|
|
|
def test_no_overrides_DO_NOT_OVERRIDE(self) -> None: # noqa: N802
|
|
"""Test that no standard tests are overridden."""
|
|
# Find path to standard test implementations
|
|
comparison_class = None
|
|
|
|
def explore_bases(cls: type) -> None:
|
|
nonlocal comparison_class
|
|
for base in cls.__bases__:
|
|
if base.__module__.startswith("langchain_tests."):
|
|
if comparison_class is None:
|
|
comparison_class = base
|
|
else:
|
|
msg = (
|
|
"Multiple standard test base classes found: "
|
|
f"{comparison_class}, {base}"
|
|
)
|
|
raise ValueError(msg)
|
|
else:
|
|
explore_bases(base)
|
|
|
|
explore_bases(self.__class__)
|
|
assert comparison_class is not None, "No standard test base class found."
|
|
|
|
print(f"Comparing {self.__class__} to {comparison_class}") # noqa: T201
|
|
|
|
running_tests = {method for method in dir(self) if method.startswith("test_")}
|
|
base_tests = {
|
|
method for method in dir(comparison_class) if method.startswith("test_")
|
|
}
|
|
deleted_tests = base_tests - running_tests
|
|
assert not deleted_tests, f"Standard tests deleted: {deleted_tests}"
|
|
|
|
overridden_tests = [
|
|
method
|
|
for method in base_tests
|
|
if getattr(self.__class__, method) is not getattr(comparison_class, method)
|
|
]
|
|
|
|
def is_xfail(method: str) -> bool:
|
|
m = getattr(self.__class__, method)
|
|
if not hasattr(m, "pytestmark"):
|
|
return False
|
|
for mark in m.pytestmark:
|
|
if mark.name == "xfail" and mark.kwargs.get("reason"):
|
|
return True
|
|
# Also accept xfail marks on individual `pytest.param` entries
|
|
# within a `parametrize` - supports xfailing only a subset of
|
|
# parametrized cases.
|
|
if mark.name == "parametrize" and len(mark.args) >= 2:
|
|
for param in mark.args[1]:
|
|
for inner in getattr(param, "marks", ()):
|
|
if inner.name == "xfail" and inner.kwargs.get("reason"):
|
|
return True
|
|
return False
|
|
|
|
overridden_not_xfail = [
|
|
method for method in overridden_tests if not is_xfail(method)
|
|
]
|
|
assert not overridden_not_xfail, (
|
|
"Standard tests overridden without "
|
|
f'@pytest.mark.xfail(reason="..."): {overridden_not_xfail}\n'
|
|
"Note: reason is required to explain why the standard test has an expected "
|
|
"failure."
|
|
)
|