Merge branch 'cc/benchmarks' of github.com:langchain-ai/langchain into cc/benchmarks

.github/workflows/check_diffs.yml

@@ -42,7 +42,6 @@ jobs:
       dependencies: ${{ steps.set-matrix.outputs.dependencies }}
       test-doc-imports: ${{ steps.set-matrix.outputs.test-doc-imports }}
       test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }}
-      codspeed: ${{ steps.set-matrix.outputs.codspeed }}
   lint:
     name: cd ${{ matrix.job-configs.working-directory }}
     needs: [ build ]
@@ -154,20 +153,6 @@ jobs:
           # and `set -e` above will cause the step to fail.
           echo "$STATUS" | grep 'nothing to commit, working tree clean'
 
-  codspeed:
-    name: cd ${{ matrix.job-configs.working-directory }}
-    needs: [ build ]
-    if: ${{ needs.build.outputs.codspeed != '[]' }}
-    strategy:
-      matrix:
-        job-configs: ${{ fromJson(needs.build.outputs.codspeed) }}
-      fail-fast: false
-    uses: ./.github/workflows/codspeed.yml
-    with:
-      working-directory: ${{ matrix.job-configs.working-directory }}
-      python-version: ${{ matrix.job-configs.python-version }}
-    secrets: inherit
-
   ci_success:
     name: "CI Success"
     needs: [build, lint, test, compile-integration-tests, extended-tests, test-doc-imports, test-pydantic]

.github/workflows/codspeed.yml

@@ -1,56 +1,79 @@
 name: CodSpeed
 
 on:
-  workflow_call:
+  push:
-    inputs:
+    branches:
-      working-directory:
+        - master
-        required: true
+  pull_request:
-        type: string
+    paths:
-        description: "From which folder this pipeline executes"
+      - 'libs/core/**'
-      python-version:
+      - 'libs/partners/**'
-        required: true
-        type: string
-        description: "Python version to use"
 
   workflow_dispatch:
 
+env:
+  AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
+  AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+  AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
+  AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
+  DEEPSEEK_API_KEY: foo
+  FIREWORKS_API_KEY: foo
+
 jobs:
-  codspeed:
+  prepare_matrix:
-    name: Run benchmarks
-    if: contains(github.event.pull_request.labels.*.name, 'run-codspeed-benchmarks') || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: "3.12"
+      - id: files
+        uses: Ana06/get-changed-files@v2.3.0
+      - id: set-matrix
+        run: |
+          uv venv
+          uv pip install packaging requests
+          uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
+    outputs:
+      codspeed: ${{ steps.set-matrix.outputs.codspeed }}
+  codspeed:
+    name: Run benchmarks
+    needs: [ prepare_matrix ]
+    if: ${{ needs.prepare_matrix.outputs.codspeed != '[]' }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        job-configs: ${{ fromJson(needs.prepare_matrix.outputs.codspeed) }}
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # We have to use 3.12 as 3.13 is not yet supported
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.job-configs.python-version }}
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: ${{ matrix.job-configs.python-version }}
 
-      - name: install deps
+      - name: Install dependencies
         run: uv sync --group test
-        working-directory: ${{ inputs.working-directory }}
+        working-directory: ${{ matrix.job-configs.python-version }}
 
       - name: Run benchmarks
         uses: CodSpeedHQ/action@v3
-        env:
-          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
-          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
-          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
-          AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
-          DEEPSEEK_API_KEY: foo
-          FIREWORKS_API_KEY: foo
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: |
-            cd ${{ inputs.working-directory }}
+            cd ${{ matrix.job-configs.working-directory }}
-            if [ "${{ inputs.working-directory }}" = "libs/core" ]; then
+            if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then
               uv run --no-sync pytest ./tests/benchmarks --codspeed
             else
               uv run --no-sync pytest ./tests/ --codspeed
             fi
-          mode: walltime
+          mode: ${{ matrix.job-configs.working-directory == 'libs/core' && 'walltime' || 'instrumentation' }}
+