mirror of
https://github.com/hwchase17/langchain.git
synced 2026-03-18 11:07:36 +00:00
a81203bf6a
CodSpeed benchmarks shared the main CI workflow's concurrency group, which has `cancel-in-progress: true`. On push-to-master (post-merge), back-to-back merges would cancel in-flight CodSpeed jobs — failing the `ci_success` gate and marking the merge commit red, even though the PR passed all checks before merging. The cancelled jobs also meant baseline benchmark data was never uploaded, silently breaking CodSpeed's regression detection for subsequent PRs. ## Changes - Extract CodSpeed into a standalone `codspeed.yml` workflow with its own concurrency group — push-to-master runs use `github.sha` as the group key (unique per commit, never cancelled), while PR runs continue to cancel stale runs via `github.ref` - Remove the `codspeed` job, its output, and its `ci_success` gate dependency from `check_diffs.yml` — benchmark results are informational and shouldn't block merges ## Review - The new workflow duplicates the `build` (change detection) job from `check_diffs.yml` since it can't reference jobs across workflows. This means an extra `check_diff.py` run per trigger, but it's lightweight (~2s) and the tradeoff is worth the isolation.
110 lines
4.5 KiB
YAML
110 lines
4.5 KiB
YAML
# CodSpeed performance benchmarks.
|
|
#
|
|
# Runs benchmarks on changed packages and uploads results to CodSpeed.
|
|
# Separated from the main CI workflow so that push-to-master baseline runs
|
|
# are never cancelled by subsequent merges (cancel-in-progress is only
|
|
# enabled for pull_request events).
|
|
|
|
name: "⚡ CodSpeed"
|
|
|
|
on:
|
|
push:
|
|
branches: [master]
|
|
pull_request:
|
|
|
|
# On PRs, cancel stale runs when new commits are pushed.
|
|
# On push-to-master, never cancel — these runs populate CodSpeed baselines.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
UV_FROZEN: "true"
|
|
UV_NO_SYNC: "true"
|
|
|
|
jobs:
|
|
build:
|
|
name: "Detect Changes"
|
|
runs-on: ubuntu-latest
|
|
if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}
|
|
steps:
|
|
- name: "📋 Checkout Code"
|
|
uses: actions/checkout@v6
|
|
- name: "🐍 Setup Python 3.11"
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.11"
|
|
- name: "📂 Get Changed Files"
|
|
id: files
|
|
uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0
|
|
- name: "🔍 Analyze Changed Files"
|
|
id: set-matrix
|
|
run: |
|
|
python -m pip install packaging requests
|
|
python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
|
|
outputs:
|
|
codspeed: ${{ steps.set-matrix.outputs.codspeed }}
|
|
|
|
benchmarks:
|
|
name: "⚡ CodSpeed Benchmarks"
|
|
needs: [build]
|
|
if: ${{ needs.build.outputs.codspeed != '[]' }}
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}
|
|
fail-fast: false
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: "📦 Install UV Package Manager"
|
|
uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7
|
|
with:
|
|
# Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+
|
|
# See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106
|
|
python-version: "3.13.11"
|
|
|
|
- name: "📦 Install Test Dependencies"
|
|
run: uv sync --group test
|
|
working-directory: ${{ matrix.job-configs.working-directory }}
|
|
|
|
- name: "⚡ Run Benchmarks: ${{ matrix.job-configs.working-directory }}"
|
|
uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}
|
|
ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }}
|
|
AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
|
|
AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
|
|
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
|
|
AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
|
|
AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
|
|
AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}
|
|
AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}
|
|
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
|
|
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
|
EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
|
|
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
|
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
|
HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
|
|
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
|
|
NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
|
|
OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}
|
|
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
|
with:
|
|
token: ${{ secrets.CODSPEED_TOKEN }}
|
|
run: |
|
|
cd ${{ matrix.job-configs.working-directory }}
|
|
if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then
|
|
uv run --no-sync pytest ./tests/benchmarks --codspeed
|
|
else
|
|
uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed
|
|
fi
|
|
mode: ${{ matrix.job-configs.codspeed-mode }}
|