diff --git a/libs/standard-tests/langchain_tests/_langsmith_plugin.py b/libs/standard-tests/langchain_tests/_langsmith_plugin.py new file mode 100644 index 00000000000..f1c4147723f --- /dev/null +++ b/libs/standard-tests/langchain_tests/_langsmith_plugin.py @@ -0,0 +1,91 @@ +"""Pytest plugin that forwards CI-set LangSmith env vars into tracing context. + +The LangSmith SDK does not natively read `LANGSMITH_TAGS` / `LANGSMITH_METADATA` +from the environment, so tags/metadata written to `$GITHUB_ENV` by CI workflows +(e.g. `integration_tests.yml`) would otherwise be silently dropped. This plugin +bridges that gap for the duration of the pytest session by entering +`langsmith.run_helpers.tracing_context`. + +To avoid surprising developers who happen to export these vars locally, the +plugin only activates when running under GitHub Actions (`GITHUB_ACTIONS=true`). + +Auto-discovered by pytest in any package that depends on `langchain-tests` +(declared via the `pytest11` entry point in +`libs/standard-tests/pyproject.toml`). +""" + +from __future__ import annotations + +import contextlib +import json +import os +import sys +import warnings +from typing import TYPE_CHECKING, Any + +import pytest +from langsmith.run_helpers import tracing_context + +if TYPE_CHECKING: + from collections.abc import Iterator + + +def _is_github_actions() -> bool: + return os.environ.get("GITHUB_ACTIONS") == "true" + + +def _parse_tags(raw: str) -> list[str]: + return [tag.strip() for tag in raw.split(",") if tag.strip()] + + +def _parse_metadata(raw: str) -> dict[str, Any] | None: + raw = raw.strip() + if not raw: + return None + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + _warn_loud(f"Ignoring LANGSMITH_METADATA: invalid JSON ({exc}).") + return None + if not isinstance(parsed, dict): + _warn_loud("Ignoring LANGSMITH_METADATA: expected a JSON object.") + return None + return parsed + + +def _warn_loud(message: str) -> None: + """Emit a `UserWarning` and mirror it to stderr so CI logs surface it.""" + warnings.warn(message, UserWarning, stacklevel=3) + sys.stderr.write(f"[langchain-tests] {message}\n") + + +@contextlib.contextmanager +def _langsmith_ci_cm() -> Iterator[None]: + """Yield with tracing context applied from current env vars. + + No-op when neither env var is set or when not running on GitHub Actions. + """ + if not _is_github_actions(): + yield + return + + tags = _parse_tags(os.environ.get("LANGSMITH_TAGS", "")) + metadata = _parse_metadata(os.environ.get("LANGSMITH_METADATA", "")) + + if not tags and not metadata: + yield + return + + keys = sorted(metadata or {}) + sys.stderr.write( + f"[langchain-tests] langsmith CI context: tags={tags}, metadata_keys={keys}\n", + ) + with tracing_context(tags=tags or None, metadata=metadata): + yield + + +@pytest.fixture(scope="session", autouse=True) +def _langsmith_ci_context() -> Iterator[None]: + """Apply `LANGSMITH_TAGS`/`LANGSMITH_METADATA` to traces in this session.""" + with _langsmith_ci_cm(): + yield diff --git a/libs/standard-tests/pyproject.toml b/libs/standard-tests/pyproject.toml index d7c0cfa490c..ba594d0c06c 100644 --- a/libs/standard-tests/pyproject.toml +++ b/libs/standard-tests/pyproject.toml @@ -48,6 +48,9 @@ Twitter = "https://x.com/langchain_oss" Slack = "https://www.langchain.com/join-community" Reddit = "https://www.reddit.com/r/LangChain/" +[project.entry-points.pytest11] +langsmith_ci = "langchain_tests._langsmith_plugin" + [dependency-groups] test = ["langchain-core>=1.4.0,<2.0.0"] test_integration = [] diff --git a/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py b/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py new file mode 100644 index 00000000000..b17c508a1fc --- /dev/null +++ b/libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py @@ -0,0 +1,150 @@ +"""Tests for the CI LangSmith env-var pytest plugin.""" + +from __future__ import annotations + +from textwrap import dedent + +import pytest +from langsmith.run_helpers import get_tracing_context + +from langchain_tests._langsmith_plugin import ( + _langsmith_ci_cm, + _parse_metadata, + _parse_tags, +) + +pytest_plugins = ["pytester"] + + +@pytest.fixture(autouse=True) +def _force_ci_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Activate the plugin's CI-gated code paths for every test.""" + monkeypatch.setenv("GITHUB_ACTIONS", "true") + + +class TestParseTags: + def test_splits_and_strips(self) -> None: + assert _parse_tags("a, b ,c") == ["a", "b", "c"] + + def test_drops_empty_segments(self) -> None: + assert _parse_tags(",a,,b,") == ["a", "b"] + + def test_empty_string(self) -> None: + assert _parse_tags("") == [] + + def test_whitespace_only(self) -> None: + assert _parse_tags(" , , ") == [] + + +class TestParseMetadata: + def test_valid_object(self) -> None: + assert _parse_metadata('{"sha": "abc", "n": 1}') == {"sha": "abc", "n": 1} + + def test_empty_string_returns_none(self) -> None: + assert _parse_metadata("") is None + assert _parse_metadata(" ") is None + + def test_invalid_json_warns_and_returns_none(self) -> None: + with pytest.warns(UserWarning, match="invalid JSON"): + result = _parse_metadata("{not json") + assert result is None + + def test_non_object_warns_and_returns_none(self) -> None: + with pytest.warns(UserWarning, match="JSON object"): + result = _parse_metadata('["a", "b"]') + assert result is None + + +class TestContextManager: + def test_applies_tags_and_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_TAGS", "github-actions,sha-deadbeef") + monkeypatch.setenv("LANGSMITH_METADATA", '{"github_run_id": "42"}') + with _langsmith_ci_cm(): + ctx = get_tracing_context() + assert ctx["tags"] == ["github-actions", "sha-deadbeef"] + assert ctx["metadata"] == {"github_run_id": "42"} + + def test_restores_context_on_exit(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_TAGS", "x") + monkeypatch.delenv("LANGSMITH_METADATA", raising=False) + before = get_tracing_context() + with _langsmith_ci_cm(): + pass + after = get_tracing_context() + assert before["tags"] == after["tags"] + assert before["metadata"] == after["metadata"] + + def test_no_env_is_noop(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LANGSMITH_TAGS", raising=False) + monkeypatch.delenv("LANGSMITH_METADATA", raising=False) + before = get_tracing_context() + with _langsmith_ci_cm(): + inside = get_tracing_context() + assert inside["tags"] == before["tags"] + assert inside["metadata"] == before["metadata"] + + def test_whitespace_only_tags_is_noop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LANGSMITH_TAGS", " , , ") + monkeypatch.delenv("LANGSMITH_METADATA", raising=False) + before = get_tracing_context() + with _langsmith_ci_cm(): + inside = get_tracing_context() + assert inside["tags"] == before["tags"] + + def test_only_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LANGSMITH_TAGS", raising=False) + monkeypatch.setenv("LANGSMITH_METADATA", '{"k": "v"}') + with _langsmith_ci_cm(): + ctx = get_tracing_context() + assert ctx["metadata"] == {"k": "v"} + + def test_only_tags(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGSMITH_TAGS", "solo") + monkeypatch.delenv("LANGSMITH_METADATA", raising=False) + with _langsmith_ci_cm(): + ctx = get_tracing_context() + assert ctx["tags"] == ["solo"] + + def test_bad_metadata_does_not_block_tags( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LANGSMITH_TAGS", "kept") + monkeypatch.setenv("LANGSMITH_METADATA", "not-json") + with pytest.warns(UserWarning, match="invalid JSON"), _langsmith_ci_cm(): # noqa: PT031 + ctx = get_tracing_context() + assert ctx["tags"] == ["kept"] + assert ctx["metadata"] is None + + def test_inactive_outside_github_actions( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + monkeypatch.setenv("LANGSMITH_TAGS", "should-not-apply") + before = get_tracing_context() + with _langsmith_ci_cm(): + inside = get_tracing_context() + assert inside["tags"] == before["tags"] + + +class TestPluginDiscovery: + """End-to-end: the `pytest11` entry point wires up the autouse fixture.""" + + def test_autouse_fixture_applies_env_in_subprocess( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("LANGSMITH_TAGS", "discovered,from-entrypoint") + monkeypatch.delenv("LANGSMITH_METADATA", raising=False) + pytester.makepyfile( + dedent(""" + from langsmith.run_helpers import get_tracing_context + + def test_tags_visible_via_autouse_fixture(): + ctx = get_tracing_context() + assert ctx["tags"] == ["discovered", "from-entrypoint"] + """), + ) + result = pytester.runpytest_subprocess("-q") + result.assert_outcomes(passed=1)