From a81203bf6a08ae65b7ed3832448558570e2208f9 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Tue, 17 Mar 2026 21:53:43 -0400 Subject: [PATCH] ci: decouple codspeed from ci gate (#36051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodSpeed benchmarks shared the main CI workflow's concurrency group, which has `cancel-in-progress: true`. On push-to-master (post-merge), back-to-back merges would cancel in-flight CodSpeed jobs — failing the `ci_success` gate and marking the merge commit red, even though the PR passed all checks before merging. The cancelled jobs also meant baseline benchmark data was never uploaded, silently breaking CodSpeed's regression detection for subsequent PRs. ## Changes - Extract CodSpeed into a standalone `codspeed.yml` workflow with its own concurrency group — push-to-master runs use `github.sha` as the group key (unique per commit, never cancelled), while PR runs continue to cancel stale runs via `github.ref` - Remove the `codspeed` job, its output, and its `ci_success` gate dependency from `check_diffs.yml` — benchmark results are informational and shouldn't block merges ## Review - The new workflow duplicates the `build` (change detection) job from `check_diffs.yml` since it can't reference jobs across workflows. This means an extra `check_diff.py` run per trigger, but it's lightweight (~2s) and the tradeoff is worth the isolation. --- .github/workflows/check_diffs.yml | 70 ------------------- .github/workflows/codspeed.yml | 109 ++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/codspeed.yml diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml index a86246f8378..a53aa14f2ec 100644 --- a/.github/workflows/check_diffs.yml +++ b/.github/workflows/check_diffs.yml @@ -8,7 +8,6 @@ # - Pydantic compatibility tests (_test_pydantic.yml) # - Integration test compilation checks (_compile_integration_test.yml) # - Extended test suites that require additional dependencies -# - Codspeed benchmarks (if not labeled 'codspeed-ignore') # # Reports status to GitHub checks and PR status. @@ -67,7 +66,6 @@ jobs: compile-integration-tests: ${{ steps.set-matrix.outputs.compile-integration-tests }} dependencies: ${{ steps.set-matrix.outputs.dependencies }} test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }} - codspeed: ${{ steps.set-matrix.outputs.codspeed }} # Run linting only on packages that have changed files lint: needs: [build] @@ -171,73 +169,6 @@ jobs: # and `set -e` above will cause the step to fail. echo "$STATUS" | grep 'nothing to commit, working tree clean' - # Run codspeed benchmarks only on packages that have changed files - codspeed: - name: "⚡ CodSpeed Benchmarks" - needs: [build] - if: ${{ needs.build.outputs.codspeed != '[]' && !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }} - runs-on: ubuntu-latest - strategy: - matrix: - job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} - fail-fast: false - steps: - - uses: actions/checkout@v6 - - - name: "📦 Install UV Package Manager" - uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7 - with: - # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+ - # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106 - python-version: "3.13.11" - - - uses: actions/setup-python@v6 - with: - # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+ - # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106 - python-version: "3.13.11" - - - name: "📦 Install Test Dependencies" - run: uv sync --group test - working-directory: ${{ matrix.job-configs.working-directory }} - - - name: "⚡ Run Benchmarks: ${{ matrix.job-configs.working-directory }}" - uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4 - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }} - ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }} - AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} - AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} - EXA_API_KEY: ${{ secrets.EXA_API_KEY }} - FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} - NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} - OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - with: - token: ${{ secrets.CODSPEED_TOKEN }} - run: | - cd ${{ matrix.job-configs.working-directory }} - if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then - uv run --no-sync pytest ./tests/benchmarks --codspeed - else - uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed - fi - mode: ${{ matrix.job-configs.codspeed-mode }} - # Final status check - ensures all required jobs passed before allowing merge ci_success: name: "✅ CI Success" @@ -249,7 +180,6 @@ jobs: compile-integration-tests, extended-tests, test-pydantic, - codspeed, ] if: | always() diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..10320299c0c --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,109 @@ +# CodSpeed performance benchmarks. +# +# Runs benchmarks on changed packages and uploads results to CodSpeed. +# Separated from the main CI workflow so that push-to-master baseline runs +# are never cancelled by subsequent merges (cancel-in-progress is only +# enabled for pull_request events). + +name: "⚡ CodSpeed" + +on: + push: + branches: [master] + pull_request: + +# On PRs, cancel stale runs when new commits are pushed. +# On push-to-master, never cancel — these runs populate CodSpeed baselines. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + UV_FROZEN: "true" + UV_NO_SYNC: "true" + +jobs: + build: + name: "Detect Changes" + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }} + steps: + - name: "📋 Checkout Code" + uses: actions/checkout@v6 + - name: "🐍 Setup Python 3.11" + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: "📂 Get Changed Files" + id: files + uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0 + - name: "🔍 Analyze Changed Files" + id: set-matrix + run: | + python -m pip install packaging requests + python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT + outputs: + codspeed: ${{ steps.set-matrix.outputs.codspeed }} + + benchmarks: + name: "⚡ CodSpeed Benchmarks" + needs: [build] + if: ${{ needs.build.outputs.codspeed != '[]' }} + runs-on: ubuntu-latest + strategy: + matrix: + job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} + fail-fast: false + steps: + - uses: actions/checkout@v6 + + - name: "📦 Install UV Package Manager" + uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7 + with: + # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+ + # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106 + python-version: "3.13.11" + + - name: "📦 Install Test Dependencies" + run: uv sync --group test + working-directory: ${{ matrix.job-configs.working-directory }} + + - name: "⚡ Run Benchmarks: ${{ matrix.job-configs.working-directory }}" + uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }} + ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }} + AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} + AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} + AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} + AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + EXA_API_KEY: ${{ secrets.EXA_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} + OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: | + cd ${{ matrix.job-configs.working-directory }} + if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then + uv run --no-sync pytest ./tests/benchmarks --codspeed + else + uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed + fi + mode: ${{ matrix.job-configs.codspeed-mode }}