ci(infra): trace scheduled integration tests (#37615)

Wire LangSmith tracing into the scheduled integration test workflow so
partner test runs emit traces to a shared project with GitHub Actions
metadata attached. Makes failures traceable back to the originating
workflow run, sha, and matrix shard without hunting through CI logs.

## Usage
- Filter the `oss-python-integration-tests` project by the
`github-actions` tag to see only CI runs; add a `sha-<sha>` or
`working_directory=<partner>` filter to narrow to a specific commit or
partner shard.
- From a failing trace, open the `github_run_url` metadata field to jump
directly to the originating workflow run; `github_run_attempt`
disambiguates reruns.
- Tags are flat strings (good for facets); structured fields like
`python_version` and `working_directory` live in metadata for richer
querying.
This commit is contained in:
Mason Daugherty
2026-05-22 22:18:03 -05:00
committed by GitHub
parent 11cdce91dc
commit bdd7f71a1b
2 changed files with 42 additions and 1 deletions

View File

@@ -116,7 +116,7 @@ jobs:
build: build:
name: 📦 Build distribution name: 📦 Build distribution
if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release
environment: Scheduled testing environment: Release
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
contents: read contents: read
@@ -274,6 +274,7 @@ jobs:
needs: needs:
- build - build
- release-notes - release-notes
environment: Release
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
contents: read contents: read
@@ -433,6 +434,7 @@ jobs:
- build - build
- release-notes - release-notes
- pre-release-checks - pre-release-checks
environment: Release
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
# This permission is used for trusted publishing: # This permission is used for trusted publishing:
@@ -475,6 +477,7 @@ jobs:
- release-notes - release-notes
- test-pypi-publish - test-pypi-publish
- pre-release-checks - pre-release-checks
environment: Release
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
contents: read contents: read
@@ -638,6 +641,7 @@ jobs:
# - test-prior-published-packages-against-new-core # - test-prior-published-packages-against-new-core
# Run if all needed jobs succeeded or were skipped (test-dependents only runs for core/langchain_v1) # Run if all needed jobs succeeded or were skipped (test-dependents only runs for core/langchain_v1)
if: ${{ !cancelled() && !failure() }} if: ${{ !cancelled() && !failure() }}
environment: Release
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
# This permission is used for trusted publishing: # This permission is used for trusted publishing:
@@ -685,6 +689,7 @@ jobs:
- publish - publish
# Run if all needed jobs succeeded or were skipped # Run if all needed jobs succeeded or were skipped
if: ${{ !cancelled() && !failure() }} if: ${{ !cancelled() && !failure() }}
environment: Release
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
# This permission is needed by `ncipollo/release-action` to # This permission is needed by `ncipollo/release-action` to

View File

@@ -128,6 +128,8 @@ jobs:
if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule' if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule'
name: "🐍 Python ${{ matrix.python-version }}: ${{ matrix.working-directory }}" name: "🐍 Python ${{ matrix.python-version }}: ${{ matrix.working-directory }}"
runs-on: ubuntu-latest runs-on: ubuntu-latest
# Scopes LangSmith tracing credentials (and any other env-scoped secrets)
environment: "Scheduled testing"
needs: [compute-matrix] needs: [compute-matrix]
timeout-minutes: 30 timeout-minutes: 30
strategy: strategy:
@@ -212,6 +214,34 @@ jobs:
;; ;;
esac esac
- name: "🧾 Build LangSmith Metadata"
# GHA expression values flow through intermediate env vars (injection
# hardening) and jq -nc builds the JSON, so quotes/newlines in any
# field can't corrupt the payload.
env:
GH_SHA: ${{ github.sha }}
GH_RUN_ID: ${{ github.run_id }}
GH_RUN_ATTEMPT: ${{ github.run_attempt }}
GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
GH_WORKFLOW: ${{ github.workflow }}
GH_EVENT: ${{ github.event_name }}
GH_REF: ${{ github.ref }}
WORKING_DIRECTORY: ${{ matrix.working-directory }}
PYTHON_VERSION: ${{ matrix.python-version }}
run: |
metadata=$(jq -nc \
--arg github_sha "$GH_SHA" \
--arg github_run_id "$GH_RUN_ID" \
--arg github_run_attempt "$GH_RUN_ATTEMPT" \
--arg github_run_url "$GH_RUN_URL" \
--arg github_workflow "$GH_WORKFLOW" \
--arg github_event "$GH_EVENT" \
--arg github_ref "$GH_REF" \
--arg working_directory "$WORKING_DIRECTORY" \
--arg python_version "$PYTHON_VERSION" \
'{github_sha: $github_sha, github_run_id: $github_run_id, github_run_attempt: $github_run_attempt, github_run_url: $github_run_url, github_workflow: $github_workflow, github_event: $github_event, github_ref: $github_ref, working_directory: $working_directory, python_version: $python_version}')
echo "LANGSMITH_METADATA=$metadata" >> "$GITHUB_ENV"
- name: "🚀 Run Integration Tests" - name: "🚀 Run Integration Tests"
# WARNING: All secrets below are available to every matrix job regardless of # WARNING: All secrets below are available to every matrix job regardless of
# which package is being tested. This is intentional for simplicity, but means # which package is being tested. This is intentional for simplicity, but means
@@ -219,6 +249,12 @@ jobs:
env: env:
LANGCHAIN_TESTS_USER_AGENT: ${{ secrets.LANGCHAIN_TESTS_USER_AGENT }} LANGCHAIN_TESTS_USER_AGENT: ${{ secrets.LANGCHAIN_TESTS_USER_AGENT }}
# Route traces to one project with GitHub run metadata so failures link back to the originating Actions run.
LANGSMITH_TRACING: "true"
LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
LANGSMITH_PROJECT: ${{ vars.LANGSMITH_PROJECT || 'scheduled-testing-py' }}
LANGSMITH_TAGS: "github-actions,${{ matrix.working-directory }},python-${{ matrix.python-version }},sha-${{ github.sha }}"
AI21_API_KEY: ${{ secrets.AI21_API_KEY }} AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }} ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}