feat(standard-tests): forward LangSmith CI env vars to traces (#37645)

Scheduled integration runs set `LANGSMITH_TAGS` and `LANGSMITH_METADATA`
in `$GITHUB_ENV` (per #37615), but the LangSmith SDK does not read those
env vars natively, so the tags/metadata were silently dropped. A new
pytest plugin in `langchain-tests` bridges that gap by entering
`langsmith.run_helpers.tracing_context` for the duration of each
session.

libs/standard-tests/langchain_tests/_langsmith_plugin.py

@@ -0,0 +1,91 @@
+"""Pytest plugin that forwards CI-set LangSmith env vars into tracing context.
+
+The LangSmith SDK does not natively read `LANGSMITH_TAGS` / `LANGSMITH_METADATA`
+from the environment, so tags/metadata written to `$GITHUB_ENV` by CI workflows
+(e.g. `integration_tests.yml`) would otherwise be silently dropped. This plugin
+bridges that gap for the duration of the pytest session by entering
+`langsmith.run_helpers.tracing_context`.
+
+To avoid surprising developers who happen to export these vars locally, the
+plugin only activates when running under GitHub Actions (`GITHUB_ACTIONS=true`).
+
+Auto-discovered by pytest in any package that depends on `langchain-tests`
+(declared via the `pytest11` entry point in
+`libs/standard-tests/pyproject.toml`).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import sys
+import warnings
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from langsmith.run_helpers import tracing_context
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+def _is_github_actions() -> bool:
+    return os.environ.get("GITHUB_ACTIONS") == "true"
+
+
+def _parse_tags(raw: str) -> list[str]:
+    return [tag.strip() for tag in raw.split(",") if tag.strip()]
+
+
+def _parse_metadata(raw: str) -> dict[str, Any] | None:
+    raw = raw.strip()
+    if not raw:
+        return None
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        _warn_loud(f"Ignoring LANGSMITH_METADATA: invalid JSON ({exc}).")
+        return None
+    if not isinstance(parsed, dict):
+        _warn_loud("Ignoring LANGSMITH_METADATA: expected a JSON object.")
+        return None
+    return parsed
+
+
+def _warn_loud(message: str) -> None:
+    """Emit a `UserWarning` and mirror it to stderr so CI logs surface it."""
+    warnings.warn(message, UserWarning, stacklevel=3)
+    sys.stderr.write(f"[langchain-tests] {message}\n")
+
+
+@contextlib.contextmanager
+def _langsmith_ci_cm() -> Iterator[None]:
+    """Yield with tracing context applied from current env vars.
+
+    No-op when neither env var is set or when not running on GitHub Actions.
+    """
+    if not _is_github_actions():
+        yield
+        return
+
+    tags = _parse_tags(os.environ.get("LANGSMITH_TAGS", ""))
+    metadata = _parse_metadata(os.environ.get("LANGSMITH_METADATA", ""))
+
+    if not tags and not metadata:
+        yield
+        return
+
+    keys = sorted(metadata or {})
+    sys.stderr.write(
+        f"[langchain-tests] langsmith CI context: tags={tags}, metadata_keys={keys}\n",
+    )
+    with tracing_context(tags=tags or None, metadata=metadata):
+        yield
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _langsmith_ci_context() -> Iterator[None]:
+    """Apply `LANGSMITH_TAGS`/`LANGSMITH_METADATA` to traces in this session."""
+    with _langsmith_ci_cm():
+        yield

libs/standard-tests/pyproject.toml

@@ -48,6 +48,9 @@ Twitter = "https://x.com/langchain_oss"
 Slack = "https://www.langchain.com/join-community"
 Reddit = "https://www.reddit.com/r/LangChain/"
 
+[project.entry-points.pytest11]
+langsmith_ci = "langchain_tests._langsmith_plugin"
+
 [dependency-groups]
 test = ["langchain-core>=1.4.0,<2.0.0"]
 test_integration = []

libs/standard-tests/tests/unit_tests/test_langsmith_plugin.py

@@ -0,0 +1,150 @@
+"""Tests for the CI LangSmith env-var pytest plugin."""
+
+from __future__ import annotations
+
+from textwrap import dedent
+
+import pytest
+from langsmith.run_helpers import get_tracing_context
+
+from langchain_tests._langsmith_plugin import (
+    _langsmith_ci_cm,
+    _parse_metadata,
+    _parse_tags,
+)
+
+pytest_plugins = ["pytester"]
+
+
+@pytest.fixture(autouse=True)
+def _force_ci_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Activate the plugin's CI-gated code paths for every test."""
+    monkeypatch.setenv("GITHUB_ACTIONS", "true")
+
+
+class TestParseTags:
+    def test_splits_and_strips(self) -> None:
+        assert _parse_tags("a, b ,c") == ["a", "b", "c"]
+
+    def test_drops_empty_segments(self) -> None:
+        assert _parse_tags(",a,,b,") == ["a", "b"]
+
+    def test_empty_string(self) -> None:
+        assert _parse_tags("") == []
+
+    def test_whitespace_only(self) -> None:
+        assert _parse_tags("  ,  ,  ") == []
+
+
+class TestParseMetadata:
+    def test_valid_object(self) -> None:
+        assert _parse_metadata('{"sha": "abc", "n": 1}') == {"sha": "abc", "n": 1}
+
+    def test_empty_string_returns_none(self) -> None:
+        assert _parse_metadata("") is None
+        assert _parse_metadata("   ") is None
+
+    def test_invalid_json_warns_and_returns_none(self) -> None:
+        with pytest.warns(UserWarning, match="invalid JSON"):
+            result = _parse_metadata("{not json")
+        assert result is None
+
+    def test_non_object_warns_and_returns_none(self) -> None:
+        with pytest.warns(UserWarning, match="JSON object"):
+            result = _parse_metadata('["a", "b"]')
+        assert result is None
+
+
+class TestContextManager:
+    def test_applies_tags_and_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("LANGSMITH_TAGS", "github-actions,sha-deadbeef")
+        monkeypatch.setenv("LANGSMITH_METADATA", '{"github_run_id": "42"}')
+        with _langsmith_ci_cm():
+            ctx = get_tracing_context()
+            assert ctx["tags"] == ["github-actions", "sha-deadbeef"]
+            assert ctx["metadata"] == {"github_run_id": "42"}
+
+    def test_restores_context_on_exit(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("LANGSMITH_TAGS", "x")
+        monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+        before = get_tracing_context()
+        with _langsmith_ci_cm():
+            pass
+        after = get_tracing_context()
+        assert before["tags"] == after["tags"]
+        assert before["metadata"] == after["metadata"]
+
+    def test_no_env_is_noop(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.delenv("LANGSMITH_TAGS", raising=False)
+        monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+        before = get_tracing_context()
+        with _langsmith_ci_cm():
+            inside = get_tracing_context()
+        assert inside["tags"] == before["tags"]
+        assert inside["metadata"] == before["metadata"]
+
+    def test_whitespace_only_tags_is_noop(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("LANGSMITH_TAGS", "  ,  ,  ")
+        monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+        before = get_tracing_context()
+        with _langsmith_ci_cm():
+            inside = get_tracing_context()
+        assert inside["tags"] == before["tags"]
+
+    def test_only_metadata(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.delenv("LANGSMITH_TAGS", raising=False)
+        monkeypatch.setenv("LANGSMITH_METADATA", '{"k": "v"}')
+        with _langsmith_ci_cm():
+            ctx = get_tracing_context()
+            assert ctx["metadata"] == {"k": "v"}
+
+    def test_only_tags(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("LANGSMITH_TAGS", "solo")
+        monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+        with _langsmith_ci_cm():
+            ctx = get_tracing_context()
+            assert ctx["tags"] == ["solo"]
+
+    def test_bad_metadata_does_not_block_tags(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("LANGSMITH_TAGS", "kept")
+        monkeypatch.setenv("LANGSMITH_METADATA", "not-json")
+        with pytest.warns(UserWarning, match="invalid JSON"), _langsmith_ci_cm():  # noqa: PT031
+            ctx = get_tracing_context()
+            assert ctx["tags"] == ["kept"]
+            assert ctx["metadata"] is None
+
+    def test_inactive_outside_github_actions(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.delenv("GITHUB_ACTIONS", raising=False)
+        monkeypatch.setenv("LANGSMITH_TAGS", "should-not-apply")
+        before = get_tracing_context()
+        with _langsmith_ci_cm():
+            inside = get_tracing_context()
+        assert inside["tags"] == before["tags"]
+
+
+class TestPluginDiscovery:
+    """End-to-end: the `pytest11` entry point wires up the autouse fixture."""
+
+    def test_autouse_fixture_applies_env_in_subprocess(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("GITHUB_ACTIONS", "true")
+        monkeypatch.setenv("LANGSMITH_TAGS", "discovered,from-entrypoint")
+        monkeypatch.delenv("LANGSMITH_METADATA", raising=False)
+        pytester.makepyfile(
+            dedent("""
+                from langsmith.run_helpers import get_tracing_context
+
+                def test_tags_visible_via_autouse_fixture():
+                    ctx = get_tracing_context()
+                    assert ctx["tags"] == ["discovered", "from-entrypoint"]
+            """),
+        )
+        result = pytester.runpytest_subprocess("-q")
+        result.assert_outcomes(passed=1)