From 0c8bb4381ed623e0614b8ac21968489ff7f23283 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:21:35 -0400 Subject: [PATCH 01/10] remove conditional --- .github/workflows/codspeed.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 061b9bdefe4..dea6c260166 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -17,7 +17,6 @@ on: jobs: codspeed: name: Run benchmarks - if: contains(github.event.pull_request.labels.*.name, 'run-codspeed-benchmarks') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 7c5af3a84b1bafbd0819329cdd470fcb7c4ba8e4 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:39:40 -0400 Subject: [PATCH 02/10] try top level codspeed workflow --- .github/scripts/check_diff.py | 11 +------ .github/workflows/codspeed.yml | 54 +++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py index 70c377f27cc..110fafca4e4 100644 --- a/.github/scripts/check_diff.py +++ b/.github/scripts/check_diff.py @@ -119,9 +119,7 @@ def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]: if job == "test-pydantic": return _get_pydantic_test_configs(dir_) - if job == "codspeed": - py_versions = ["3.12"] # 3.13 is not yet supported - elif dir_ == "libs/core": + if dir_ == "libs/core": py_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] # custom logic for specific directories elif dir_ == "libs/partners/milvus": @@ -212,8 +210,6 @@ def _get_configs_for_multi_dirs( ) elif job == "extended-tests": dirs = list(dirs_to_run["extended-test"]) - elif job == "codspeed": - dirs = list(dirs_to_run["codspeed"]) else: raise ValueError(f"Unknown job: {job}") @@ -229,7 +225,6 @@ if __name__ == "__main__": "lint": set(), "test": set(), "extended-test": set(), - "codspeed": set(), } docs_edited = False @@ -253,8 +248,6 @@ if __name__ == "__main__": dirs_to_run["extended-test"].update(LANGCHAIN_DIRS) dirs_to_run["lint"].add(".") - if file.startswith("libs/core"): - dirs_to_run["codspeed"].add(f"libs/core") if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS): # add that dir and all dirs after in LANGCHAIN_DIRS # for extended testing @@ -293,7 +286,6 @@ if __name__ == "__main__": if not filename.startswith(".") ] != ["README.md"]: dirs_to_run["test"].add(f"libs/partners/{partner_dir}") - dirs_to_run["codspeed"].add(f"libs/partners/{partner_dir}") # Skip if the directory was deleted or is just a tombstone readme elif file == "libs/packages.yml": continue @@ -319,7 +311,6 @@ if __name__ == "__main__": "compile-integration-tests", "dependencies", "test-pydantic", - "codspeed", ] } map_job_to_configs["test-doc-imports"] = ( diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index dea6c260166..06977ad2d16 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -1,55 +1,61 @@ name: CodSpeed on: - workflow_call: - inputs: - working-directory: - required: true - type: string - description: "From which folder this pipeline executes" - python-version: - required: true - type: string - description: "Python version to use" + push: + branches: + - master + pull_request: + paths: + - 'libs/core/**' + - 'libs/partners/**' workflow_dispatch: +env: + AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} + AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} + DEEPSEEK_API_KEY: foo + FIREWORKS_API_KEY: foo + jobs: codspeed: name: Run benchmarks runs-on: ubuntu-latest + strategy: + matrix: + - working-directory: libs/core + mode: walltime + - working-directory: libs/partners/openai + - working-directory: libs/partners/anthropic + steps: - uses: actions/checkout@v4 + # We have to use 3.12 as 3.13 is not yet supported - name: Install uv uses: astral-sh/setup-uv@v6 with: - python-version: ${{ inputs.python-version }} + python-version: "3.12" - uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} + python-version: "3.12" - - name: install deps + - name: Install dependencies run: uv sync --group test - working-directory: ${{ inputs.working-directory }} + working-directory: ${{ matrix.working-directory }} - name: Run benchmarks uses: CodSpeedHQ/action@v3 - env: - AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} - AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }} - DEEPSEEK_API_KEY: foo - FIREWORKS_API_KEY: foo with: token: ${{ secrets.CODSPEED_TOKEN }} run: | - cd ${{ inputs.working-directory }} - if [ "${{ inputs.working-directory }}" = "libs/core" ]; then + cd ${{ matrix.working-directory }} + if [ "${{ matrix.working-directory }}" = "libs/core" ]; then uv run --no-sync pytest ./tests/benchmarks --codspeed else uv run --no-sync pytest ./tests/ --codspeed fi - mode: walltime + mode: ${{ matrix.mode || 'instrumentation' }} From a2988c6f0b52915d8cfbe4b9e7e9f169c4b4b0e0 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:44:36 -0400 Subject: [PATCH 03/10] remove other codspeed workflow --- .github/workflows/check_diffs.yml | 15 --------------- .github/workflows/codspeed.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml index dcdf38280b0..0cb3b9be883 100644 --- a/.github/workflows/check_diffs.yml +++ b/.github/workflows/check_diffs.yml @@ -42,7 +42,6 @@ jobs: dependencies: ${{ steps.set-matrix.outputs.dependencies }} test-doc-imports: ${{ steps.set-matrix.outputs.test-doc-imports }} test-pydantic: ${{ steps.set-matrix.outputs.test-pydantic }} - codspeed: ${{ steps.set-matrix.outputs.codspeed }} lint: name: cd ${{ matrix.job-configs.working-directory }} needs: [ build ] @@ -154,20 +153,6 @@ jobs: # and `set -e` above will cause the step to fail. echo "$STATUS" | grep 'nothing to commit, working tree clean' - codspeed: - name: cd ${{ matrix.job-configs.working-directory }} - needs: [ build ] - if: ${{ needs.build.outputs.codspeed != '[]' }} - strategy: - matrix: - job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} - fail-fast: false - uses: ./.github/workflows/codspeed.yml - with: - working-directory: ${{ matrix.job-configs.working-directory }} - python-version: ${{ matrix.job-configs.python-version }} - secrets: inherit - ci_success: name: "CI Success" needs: [build, lint, test, compile-integration-tests, extended-tests, test-doc-imports, test-pydantic] diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 06977ad2d16..2a3859491c8 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -20,6 +20,20 @@ env: FIREWORKS_API_KEY: foo jobs: + # build: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - name: Install uv + # uses: astral-sh/setup-uv@v6 + # with: + # python-version: "3.12" + # - id: files + # uses: Ana06/get-changed-files@v2.3.0 + # - id: set-matrix + # run: | + # uv pip install packaging requests + # uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT codspeed: name: Run benchmarks runs-on: ubuntu-latest @@ -59,3 +73,17 @@ jobs: uv run --no-sync pytest ./tests/ --codspeed fi mode: ${{ matrix.mode || 'instrumentation' }} + +# codspeed: +# name: cd ${{ matrix.job-configs.working-directory }} +# needs: [ build ] +# if: ${{ needs.build.outputs.codspeed != '[]' }} +# strategy: +# matrix: +# job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} +# fail-fast: false +# uses: ./.github/workflows/codspeed.yml +# with: +# working-directory: ${{ matrix.job-configs.working-directory }} +# python-version: ${{ matrix.job-configs.python-version }} +# secrets: inherit From bfeab8e2b59739a50aebc3d3213149bb1214f147 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:52:44 -0400 Subject: [PATCH 04/10] using check diffs output --- .github/scripts/check_diff.py | 11 ++++++- .github/workflows/codspeed.yml | 60 +++++++++++++--------------------- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py index 110fafca4e4..70c377f27cc 100644 --- a/.github/scripts/check_diff.py +++ b/.github/scripts/check_diff.py @@ -119,7 +119,9 @@ def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]: if job == "test-pydantic": return _get_pydantic_test_configs(dir_) - if dir_ == "libs/core": + if job == "codspeed": + py_versions = ["3.12"] # 3.13 is not yet supported + elif dir_ == "libs/core": py_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] # custom logic for specific directories elif dir_ == "libs/partners/milvus": @@ -210,6 +212,8 @@ def _get_configs_for_multi_dirs( ) elif job == "extended-tests": dirs = list(dirs_to_run["extended-test"]) + elif job == "codspeed": + dirs = list(dirs_to_run["codspeed"]) else: raise ValueError(f"Unknown job: {job}") @@ -225,6 +229,7 @@ if __name__ == "__main__": "lint": set(), "test": set(), "extended-test": set(), + "codspeed": set(), } docs_edited = False @@ -248,6 +253,8 @@ if __name__ == "__main__": dirs_to_run["extended-test"].update(LANGCHAIN_DIRS) dirs_to_run["lint"].add(".") + if file.startswith("libs/core"): + dirs_to_run["codspeed"].add(f"libs/core") if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS): # add that dir and all dirs after in LANGCHAIN_DIRS # for extended testing @@ -286,6 +293,7 @@ if __name__ == "__main__": if not filename.startswith(".") ] != ["README.md"]: dirs_to_run["test"].add(f"libs/partners/{partner_dir}") + dirs_to_run["codspeed"].add(f"libs/partners/{partner_dir}") # Skip if the directory was deleted or is just a tombstone readme elif file == "libs/packages.yml": continue @@ -311,6 +319,7 @@ if __name__ == "__main__": "compile-integration-tests", "dependencies", "test-pydantic", + "codspeed", ] } map_job_to_configs["test-doc-imports"] = ( diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 2a3859491c8..a54b6eada0e 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -20,29 +20,28 @@ env: FIREWORKS_API_KEY: foo jobs: - # build: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - name: Install uv - # uses: astral-sh/setup-uv@v6 - # with: - # python-version: "3.12" - # - id: files - # uses: Ana06/get-changed-files@v2.3.0 - # - id: set-matrix - # run: | - # uv pip install packaging requests - # uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT + prepare_matrix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + - id: files + uses: Ana06/get-changed-files@v2.3.0 + - id: set-matrix + run: | + uv pip install packaging requests + uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT codspeed: name: Run benchmarks + needs: [ prepare_matrix ] runs-on: ubuntu-latest strategy: matrix: - - working-directory: libs/core - mode: walltime - - working-directory: libs/partners/openai - - working-directory: libs/partners/anthropic + job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} + fail-fast: false steps: - uses: actions/checkout@v4 @@ -51,39 +50,26 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 with: - python-version: "3.12" + python-version: ${{ matrix.job-configs.python-version }} - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.job-configs.python-version }} - name: Install dependencies run: uv sync --group test - working-directory: ${{ matrix.working-directory }} + working-directory: ${{ matrix.job-configs.python-version }} - name: Run benchmarks uses: CodSpeedHQ/action@v3 with: token: ${{ secrets.CODSPEED_TOKEN }} run: | - cd ${{ matrix.working-directory }} - if [ "${{ matrix.working-directory }}" = "libs/core" ]; then + cd ${{ matrix.job-configs.working-directory }} + if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then uv run --no-sync pytest ./tests/benchmarks --codspeed else uv run --no-sync pytest ./tests/ --codspeed fi - mode: ${{ matrix.mode || 'instrumentation' }} + mode: ${{ matrix.job-configs.working-directory == 'libs/core' && 'walltime' || 'instrumentation' }} -# codspeed: -# name: cd ${{ matrix.job-configs.working-directory }} -# needs: [ build ] -# if: ${{ needs.build.outputs.codspeed != '[]' }} -# strategy: -# matrix: -# job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} -# fail-fast: false -# uses: ./.github/workflows/codspeed.yml -# with: -# working-directory: ${{ matrix.job-configs.working-directory }} -# python-version: ${{ matrix.job-configs.python-version }} -# secrets: inherit From b6da2d293eb144716daa224b0f94ab041cf731d1 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:54:59 -0400 Subject: [PATCH 05/10] uv venv --- .github/workflows/codspeed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index a54b6eada0e..a627a305c2b 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -32,6 +32,7 @@ jobs: uses: Ana06/get-changed-files@v2.3.0 - id: set-matrix run: | + uv venv uv pip install packaging requests uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT codspeed: From 3fddb50df802b5b93952ca80686243c98acbe56e Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:56:18 -0400 Subject: [PATCH 06/10] tweak input --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index a627a305c2b..2e470dea8ce 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - job-configs: ${{ fromJson(needs.build.outputs.codspeed) }} + job-configs: ${{ fromJson(needs.prepare_matrix.outputs.codspeed) }} fail-fast: false steps: From d01d6703080e78145b37c6e22804380dcc4e1f03 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 09:59:51 -0400 Subject: [PATCH 07/10] add conditional --- .github/workflows/codspeed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 2e470dea8ce..d7e9d74e8ef 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -38,6 +38,7 @@ jobs: codspeed: name: Run benchmarks needs: [ prepare_matrix ] + if: ${{ needs.prepare_matrix.outputs.codspeed != '[]' }} runs-on: ubuntu-latest strategy: matrix: From 5c05e6e5e1e851269fb6653fc026769663b971ad Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 10:03:27 -0400 Subject: [PATCH 08/10] debugging --- .github/workflows/codspeed.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d7e9d74e8ef..8a265631c34 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -34,7 +34,9 @@ jobs: run: | uv venv uv pip install packaging requests - uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT + output=$(uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }}) + echo "Script output: $output" + echo "$output" >> $GITHUB_OUTPUT codspeed: name: Run benchmarks needs: [ prepare_matrix ] From f08e4f1274ce06ce34e4de5c6555156c4d6241f4 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 10:06:57 -0400 Subject: [PATCH 09/10] set codspeed output whoops --- .github/workflows/codspeed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 8a265631c34..bbe1e6bc3ff 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -34,9 +34,9 @@ jobs: run: | uv venv uv pip install packaging requests - output=$(uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }}) - echo "Script output: $output" - echo "$output" >> $GITHUB_OUTPUT + uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT + outputs: + codspeed: ${{ steps.set-matrix.outputs.codspeed }} codspeed: name: Run benchmarks needs: [ prepare_matrix ] From 7cfb6a6d6a3617d17b468c9f9a6857139cce792f Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Thu, 29 May 2025 10:07:35 -0400 Subject: [PATCH 10/10] set codspeed output whoops --- .github/workflows/codspeed.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index bbe1e6bc3ff..1dcfacb3176 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -35,8 +35,8 @@ jobs: uv venv uv pip install packaging requests uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT - outputs: - codspeed: ${{ steps.set-matrix.outputs.codspeed }} + outputs: + codspeed: ${{ steps.set-matrix.outputs.codspeed }} codspeed: name: Run benchmarks needs: [ prepare_matrix ]