From 7c5af3a84b1bafbd0819329cdd470fcb7c4ba8e4 Mon Sep 17 00:00:00 2001
From: Sydney Runkle <sydneymarierunkle@gmail.com>
Date: Thu, 29 May 2025 09:39:40 -0400
Subject: [PATCH] try top level codspeed workflow

---
 .github/scripts/check_diff.py  | 11 +------
 .github/workflows/codspeed.yml | 54 +++++++++++++++++++---------------
 2 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py
index 70c377f27cc..110fafca4e4 100644
--- a/.github/scripts/check_diff.py
+++ b/.github/scripts/check_diff.py
@@ -119,9 +119,7 @@ def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
     if job == "test-pydantic":
         return _get_pydantic_test_configs(dir_)
 
-    if job == "codspeed":
-        py_versions = ["3.12"]  # 3.13 is not yet supported
-    elif dir_ == "libs/core":
+    if dir_ == "libs/core":
         py_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"]
     # custom logic for specific directories
     elif dir_ == "libs/partners/milvus":
@@ -212,8 +210,6 @@ def _get_configs_for_multi_dirs(
         )
     elif job == "extended-tests":
         dirs = list(dirs_to_run["extended-test"])
-    elif job == "codspeed":
-        dirs = list(dirs_to_run["codspeed"])
     else:
         raise ValueError(f"Unknown job: {job}")
 
@@ -229,7 +225,6 @@ if __name__ == "__main__":
         "lint": set(),
         "test": set(),
         "extended-test": set(),
-        "codspeed": set(),
     }
     docs_edited = False
 
@@ -253,8 +248,6 @@ if __name__ == "__main__":
             dirs_to_run["extended-test"].update(LANGCHAIN_DIRS)
             dirs_to_run["lint"].add(".")
 
-        if file.startswith("libs/core"):
-            dirs_to_run["codspeed"].add(f"libs/core")
         if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS):
             # add that dir and all dirs after in LANGCHAIN_DIRS
             # for extended testing
@@ -293,7 +286,6 @@ if __name__ == "__main__":
                 if not filename.startswith(".")
             ] != ["README.md"]:
                 dirs_to_run["test"].add(f"libs/partners/{partner_dir}")
-                dirs_to_run["codspeed"].add(f"libs/partners/{partner_dir}")
             # Skip if the directory was deleted or is just a tombstone readme
         elif file == "libs/packages.yml":
             continue
@@ -319,7 +311,6 @@ if __name__ == "__main__":
             "compile-integration-tests",
             "dependencies",
             "test-pydantic",
-            "codspeed",
         ]
     }
     map_job_to_configs["test-doc-imports"] = (
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index dea6c260166..06977ad2d16 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -1,55 +1,61 @@
 name: CodSpeed
 
 on:
-  workflow_call:
-    inputs:
-      working-directory:
-        required: true
-        type: string
-        description: "From which folder this pipeline executes"
-      python-version:
-        required: true
-        type: string
-        description: "Python version to use"
+  push:
+    branches:
+        - master
+  pull_request:
+    paths:
+      - 'libs/core/**'
+      - 'libs/partners/**'
 
   workflow_dispatch:
 
+env:
+  AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
+  AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+  AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
+  AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
+  DEEPSEEK_API_KEY: foo
+  FIREWORKS_API_KEY: foo
+
 jobs:
   codspeed:
     name: Run benchmarks
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        - working-directory: libs/core
+          mode: walltime
+        - working-directory: libs/partners/openai
+        - working-directory: libs/partners/anthropic
+
     steps:
       - uses: actions/checkout@v4
 
+      # We have to use 3.12 as 3.13 is not yet supported
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: "3.12"
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: "3.12"
 
-      - name: install deps
+      - name: Install dependencies
         run: uv sync --group test
-        working-directory: ${{ inputs.working-directory }}
+        working-directory: ${{ matrix.working-directory }}
 
       - name: Run benchmarks
         uses: CodSpeedHQ/action@v3
-        env:
-          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
-          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
-          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
-          AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
-          DEEPSEEK_API_KEY: foo
-          FIREWORKS_API_KEY: foo
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: |
-            cd ${{ inputs.working-directory }}
-            if [ "${{ inputs.working-directory }}" = "libs/core" ]; then
+            cd ${{ matrix.working-directory }}
+            if [ "${{ matrix.working-directory }}" = "libs/core" ]; then
               uv run --no-sync pytest ./tests/benchmarks --codspeed
             else
               uv run --no-sync pytest ./tests/ --codspeed
             fi
-          mode: walltime
+          mode: ${{ matrix.mode || 'instrumentation' }}