From bdd7f71a1b426675a83915dbd68107ceca069fc8 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Fri, 22 May 2026 22:18:03 -0500 Subject: [PATCH] ci(infra): trace scheduled integration tests (#37615) Wire LangSmith tracing into the scheduled integration test workflow so partner test runs emit traces to a shared project with GitHub Actions metadata attached. Makes failures traceable back to the originating workflow run, sha, and matrix shard without hunting through CI logs. ## Usage - Filter the `oss-python-integration-tests` project by the `github-actions` tag to see only CI runs; add a `sha-` or `working_directory=` filter to narrow to a specific commit or partner shard. - From a failing trace, open the `github_run_url` metadata field to jump directly to the originating workflow run; `github_run_attempt` disambiguates reruns. - Tags are flat strings (good for facets); structured fields like `python_version` and `working_directory` live in metadata for richer querying. --- .github/workflows/_release.yml | 7 ++++- .github/workflows/integration_tests.yml | 36 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 0fbafcf748a..097d60c2716 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -116,7 +116,7 @@ jobs: build: name: ๐Ÿ“ฆ Build distribution if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release - environment: Scheduled testing + environment: Release runs-on: ubuntu-latest permissions: contents: read @@ -274,6 +274,7 @@ jobs: needs: - build - release-notes + environment: Release runs-on: ubuntu-latest permissions: contents: read @@ -433,6 +434,7 @@ jobs: - build - release-notes - pre-release-checks + environment: Release runs-on: ubuntu-latest permissions: # This permission is used for trusted publishing: @@ -475,6 +477,7 @@ jobs: - release-notes - test-pypi-publish - pre-release-checks + environment: Release runs-on: ubuntu-latest permissions: contents: read @@ -638,6 +641,7 @@ jobs: # - test-prior-published-packages-against-new-core # Run if all needed jobs succeeded or were skipped (test-dependents only runs for core/langchain_v1) if: ${{ !cancelled() && !failure() }} + environment: Release runs-on: ubuntu-latest permissions: # This permission is used for trusted publishing: @@ -685,6 +689,7 @@ jobs: - publish # Run if all needed jobs succeeded or were skipped if: ${{ !cancelled() && !failure() }} + environment: Release runs-on: ubuntu-latest permissions: # This permission is needed by `ncipollo/release-action` to diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6ca34bee12f..00b01b79b9a 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -128,6 +128,8 @@ jobs: if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule' name: "๐Ÿ Python ${{ matrix.python-version }}: ${{ matrix.working-directory }}" runs-on: ubuntu-latest + # Scopes LangSmith tracing credentials (and any other env-scoped secrets) + environment: "Scheduled testing" needs: [compute-matrix] timeout-minutes: 30 strategy: @@ -212,6 +214,34 @@ jobs: ;; esac + - name: "๐Ÿงพ Build LangSmith Metadata" + # GHA expression values flow through intermediate env vars (injection + # hardening) and jq -nc builds the JSON, so quotes/newlines in any + # field can't corrupt the payload. + env: + GH_SHA: ${{ github.sha }} + GH_RUN_ID: ${{ github.run_id }} + GH_RUN_ATTEMPT: ${{ github.run_attempt }} + GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_WORKFLOW: ${{ github.workflow }} + GH_EVENT: ${{ github.event_name }} + GH_REF: ${{ github.ref }} + WORKING_DIRECTORY: ${{ matrix.working-directory }} + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + metadata=$(jq -nc \ + --arg github_sha "$GH_SHA" \ + --arg github_run_id "$GH_RUN_ID" \ + --arg github_run_attempt "$GH_RUN_ATTEMPT" \ + --arg github_run_url "$GH_RUN_URL" \ + --arg github_workflow "$GH_WORKFLOW" \ + --arg github_event "$GH_EVENT" \ + --arg github_ref "$GH_REF" \ + --arg working_directory "$WORKING_DIRECTORY" \ + --arg python_version "$PYTHON_VERSION" \ + '{github_sha: $github_sha, github_run_id: $github_run_id, github_run_attempt: $github_run_attempt, github_run_url: $github_run_url, github_workflow: $github_workflow, github_event: $github_event, github_ref: $github_ref, working_directory: $working_directory, python_version: $python_version}') + echo "LANGSMITH_METADATA=$metadata" >> "$GITHUB_ENV" + - name: "๐Ÿš€ Run Integration Tests" # WARNING: All secrets below are available to every matrix job regardless of # which package is being tested. This is intentional for simplicity, but means @@ -219,6 +249,12 @@ jobs: env: LANGCHAIN_TESTS_USER_AGENT: ${{ secrets.LANGCHAIN_TESTS_USER_AGENT }} + # Route traces to one project with GitHub run metadata so failures link back to the originating Actions run. + LANGSMITH_TRACING: "true" + LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} + LANGSMITH_PROJECT: ${{ vars.LANGSMITH_PROJECT || 'scheduled-testing-py' }} + LANGSMITH_TAGS: "github-actions,${{ matrix.working-directory }},python-${{ matrix.python-version }},sha-${{ github.sha }}" + AI21_API_KEY: ${{ secrets.AI21_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}