diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml index a86246f8378..a53aa14f2ec 100644 --- a/.github/workflows/check_diffs.yml +++ b/.github/workflows/check_diffs.yml @@ -8,7 +8,6 @@ # - Pydantic compatibility tests (_test_pydantic.yml) # - Integration test compilation checks (_compile_integration_test.yml) # - Extended test suites that require additional dependencies -# - Codspeed benchmarks (if not labeled 'codspeed-ignore') # # Reports status to GitHub checks and PR status. @@ -67,7 +66,6 @@ jobs: compile-integration-tests: ${{ steps.set-matrix.outputs.compile-integration-tests }} dependencies: ${{ steps.set-matrix.outputs.dependencies }} test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }} - codspeed: ${{ steps.set-matrix.outputs.codspeed }} # Run linting only on packages that have changed files lint: needs: [build] @@ -171,73 +169,6 @@ jobs: # and `set -e` above will cause the step to fail. echo "$STATUS" | grep 'nothing to commit, working tree clean' - # Run codspeed benchmarks only on packages that have changed files - codspeed: - name: "⚡ CodSpeed Benchmarks" - needs: [build] - if: ${{ needs.build.outputs.codspeed != '[]' && !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }} - runs-on: ubuntu-latest - strategy: - matrix: - job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} - fail-fast: false - steps: - - uses: actions/checkout@v6 - - - name: "📦 Install UV Package Manager" - uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7 - with: - # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+ - # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106 - python-version: "3.13.11" - - - uses: actions/setup-python@v6 - with: - # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+ - # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106 - python-version: "3.13.11" - - - name: "📦 Install Test Dependencies" - run: uv sync --group test - working-directory: ${{ matrix.job-configs.working-directory }} - - - name: "⚡ Run Benchmarks: ${{ matrix.job-configs.working-directory }}" - uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4 - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }} - ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }} - AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} - AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} - EXA_API_KEY: ${{ secrets.EXA_API_KEY }} - FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} - MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} - NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} - OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - with: - token: ${{ secrets.CODSPEED_TOKEN }} - run: | - cd ${{ matrix.job-configs.working-directory }} - if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then - uv run --no-sync pytest ./tests/benchmarks --codspeed - else - uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed - fi - mode: ${{ matrix.job-configs.codspeed-mode }} - # Final status check - ensures all required jobs passed before allowing merge ci_success: name: "✅ CI Success" @@ -249,7 +180,6 @@ jobs: compile-integration-tests, extended-tests, test-pydantic, - codspeed, ] if: | always() diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..10320299c0c --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,109 @@ +# CodSpeed performance benchmarks. +# +# Runs benchmarks on changed packages and uploads results to CodSpeed. +# Separated from the main CI workflow so that push-to-master baseline runs +# are never cancelled by subsequent merges (cancel-in-progress is only +# enabled for pull_request events). + +name: "⚡ CodSpeed" + +on: + push: + branches: [master] + pull_request: + +# On PRs, cancel stale runs when new commits are pushed. +# On push-to-master, never cancel — these runs populate CodSpeed baselines. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + UV_FROZEN: "true" + UV_NO_SYNC: "true" + +jobs: + build: + name: "Detect Changes" + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }} + steps: + - name: "📋 Checkout Code" + uses: actions/checkout@v6 + - name: "🐍 Setup Python 3.11" + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: "📂 Get Changed Files" + id: files + uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0 + - name: "🔍 Analyze Changed Files" + id: set-matrix + run: | + python -m pip install packaging requests + python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT + outputs: + codspeed: ${{ steps.set-matrix.outputs.codspeed }} + + benchmarks: + name: "⚡ CodSpeed Benchmarks" + needs: [build] + if: ${{ needs.build.outputs.codspeed != '[]' }} + runs-on: ubuntu-latest + strategy: + matrix: + job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} + fail-fast: false + steps: + - uses: actions/checkout@v6 + + - name: "📦 Install UV Package Manager" + uses: astral-sh/setup-uv@0ca8f610542aa7f4acaf39e65cf4eb3c35091883 # v7 + with: + # Pinned to 3.13.11 to work around CodSpeed walltime segfault on 3.13.12+ + # See: https://github.com/CodSpeedHQ/pytest-codspeed/issues/106 + python-version: "3.13.11" + + - name: "📦 Install Test Dependencies" + run: uv sync --group test + working-directory: ${{ matrix.job-configs.working-directory }} + + - name: "⚡ Run Benchmarks: ${{ matrix.job-configs.working-directory }}" + uses: CodSpeedHQ/action@a50965600eafa04edcd6717761f55b77e52aafbd # v4 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }} + ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }} + AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} + AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} + AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} + AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + EXA_API_KEY: ${{ secrets.EXA_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} + OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: | + cd ${{ matrix.job-configs.working-directory }} + if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then + uv run --no-sync pytest ./tests/benchmarks --codspeed + else + uv run --no-sync pytest ./tests/unit_tests/ -m benchmark --codspeed + fi + mode: ${{ matrix.job-configs.codspeed-mode }}