From 78ec7d886d8ff1337174e775df29f9553bcf859b Mon Sep 17 00:00:00 2001 From: Sydney Runkle <54324534+sydney-runkle@users.noreply.github.com> Date: Wed, 9 Apr 2025 13:00:15 -0400 Subject: [PATCH] [performance]: Adding benchmarks for common `langchain-core` imports (#30747) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first in a sequence of PRs focusing on improving performance in core. We're starting with reducing import times for common structures, hence the benchmarks here. The benchmark looks a little bit complicated - we have to use a process so that we don't suffer from Python's import caching system. I tried doing manual modification of `sys.modules` between runs, but that's pretty tricky / hacky to get right, hence the subprocess approach. Motivated by extremely slow baseline for common imports (we're talking 2-5 seconds): Screenshot 2025-04-09 at 12 48 12 PM Also added a `make benchmark` command to make local runs easy :). Currently using walltimes so that we can track total time despite using a manual proces. --- .github/workflows/codspeed.yml | 37 ++++++++++ .gitignore | 1 + README.md | 1 + libs/core/Makefile | 3 + libs/core/pyproject.toml | 4 +- libs/core/tests/benchmarks/__init__.py | 0 libs/core/tests/benchmarks/test_imports.py | 38 ++++++++++ libs/core/uv.lock | 86 ++++++++++++++++++++++ 8 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 libs/core/tests/benchmarks/__init__.py create mode 100644 libs/core/tests/benchmarks/test_imports.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..16b783fa899 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,37 @@ +name: CodSpeed + +on: + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +jobs: + codspeed: + name: Run benchmarks + runs-on: codspeed-macro + steps: + - uses: actions/checkout@v4 + + # We have to use 3.12, 3.13 is not yet supported + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + # Using this action is still necessary for CodSpeed to work + - uses: actions/setup-python@v3 + with: + python-version: "3.12" + + - name: install deps + run: uv sync --group test + working-directory: ./libs/core + + - name: Run benchmarks + uses: CodSpeedHQ/action@v3 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: | + cd libs/core + uv run --no-sync pytest ./tests/benchmarks --codspeed diff --git a/.gitignore b/.gitignore index 11e074a8837..4f476c95e73 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.codspeed/ # Translations *.mo diff --git a/README.md b/README.md index d99725cc105..a645f23910b 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ [![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode&style=flat-square)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/langchain-ai/langchain) [](https://codespaces.new/langchain-ai/langchain) [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai) +[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/langchain-ai/langchain) > [!NOTE] > Looking for the JS/TS library? Check out [LangChain.js](https://github.com/langchain-ai/langchainjs). diff --git a/libs/core/Makefile b/libs/core/Makefile index d64f9f241f7..bf74eadd9e0 100644 --- a/libs/core/Makefile +++ b/libs/core/Makefile @@ -64,6 +64,9 @@ spell_check: spell_fix: uv run --all-groups codespell --toml pyproject.toml -w +benchmark: + uv run pytest tests/benchmarks --codspeed + ###################### # HELP ###################### diff --git a/libs/core/pyproject.toml b/libs/core/pyproject.toml index ff93fc60e87..fdbdf16c0f1 100644 --- a/libs/core/pyproject.toml +++ b/libs/core/pyproject.toml @@ -57,6 +57,8 @@ test = [ "numpy>=1.26.4; python_version<'3.13'", "numpy>=2.1.0; python_version>='3.13'", "langchain-tests", + "pytest-benchmark", + "pytest-codspeed", ] test_integration = [] @@ -117,7 +119,7 @@ omit = [ "tests/*",] [tool.pytest.ini_options] addopts = "--snapshot-warn-unused --strict-markers --strict-config --durations=5" -markers = [ "requires: mark tests as requiring a specific library", "compile: mark placeholder test used to compile integration tests without running them",] +markers = [ "requires: mark tests as requiring a specific library", "compile: mark placeholder test used to compile integration tests without running them", ] asyncio_mode = "auto" filterwarnings = [ "ignore::langchain_core._api.beta_decorator.LangChainBetaWarning",] asyncio_default_fixture_loop_scope = "function" diff --git a/libs/core/tests/benchmarks/__init__.py b/libs/core/tests/benchmarks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/libs/core/tests/benchmarks/test_imports.py b/libs/core/tests/benchmarks/test_imports.py new file mode 100644 index 00000000000..d3771b0a423 --- /dev/null +++ b/libs/core/tests/benchmarks/test_imports.py @@ -0,0 +1,38 @@ +import subprocess +import sys + +import pytest +from pytest_benchmark.fixture import BenchmarkFixture # type: ignore + + +@pytest.mark.parametrize( + "import_path", + [ + pytest.param( + "from langchain_core.messages import HumanMessage", id="HumanMessage" + ), + pytest.param("from langchain_core.tools import tool", id="tool"), + pytest.param( + "from langchain_core.callbacks import CallbackManager", id="CallbackManager" + ), + pytest.param("from langchain_core.runnables import Runnable", id="Runnable"), + pytest.param( + "from langchain_core.language_models import BaseChatModel", + id="BaseChatModel", + ), + pytest.param( + "from langchain_core.prompts import ChatPromptTemplate", + id="PromChatPromptTemplateptTemplate", + ), + pytest.param("from langchain_core.documents import Document", id="Document"), + pytest.param( + "from langchain_core.vectorstores import InMemoryVectorStore", + id="InMemoryVectorStore", + ), + ], +) +@pytest.mark.benchmark +def test_import_time(benchmark: BenchmarkFixture, import_path: str) -> None: + @benchmark + def import_in_subprocess() -> None: + subprocess.run([sys.executable, "-c", import_path], check=False) diff --git a/libs/core/uv.lock b/libs/core/uv.lock index 22ad21816ab..c15e021fe1d 100644 --- a/libs/core/uv.lock +++ b/libs/core/uv.lock @@ -968,6 +968,8 @@ test = [ { name = "numpy", version = "2.2.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-benchmark" }, + { name = "pytest-codspeed" }, { name = "pytest-mock" }, { name = "pytest-socket" }, { name = "pytest-watcher" }, @@ -1011,6 +1013,8 @@ test = [ { name = "numpy", marker = "python_full_version >= '3.13'", specifier = ">=2.1.0" }, { name = "pytest", specifier = ">=8,<9" }, { name = "pytest-asyncio", specifier = ">=0.21.1,<1.0.0" }, + { name = "pytest-benchmark" }, + { name = "pytest-codspeed" }, { name = "pytest-mock", specifier = ">=3.10.0,<4.0.0" }, { name = "pytest-socket", specifier = ">=0.7.0,<1.0.0" }, { name = "pytest-watcher", specifier = ">=0.3.4,<1.0.0" }, @@ -1125,6 +1129,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/3b/02313e378f6328ada43ee43ecc81a398b4f68e207c94770d1ed6aac6cca2/langsmith-0.3.4-py3-none-any.whl", hash = "sha256:f3b818ce31dc3bdf1f797e75bf32a8a7b062a411f146bd4ffdfc2be0b4b03233", size = 333291 }, ] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, +] + [[package]] name = "markupsafe" version = "3.0.2" @@ -1205,6 +1221,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899 }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, +] + [[package]] name = "mistune" version = "3.1.1" @@ -1669,6 +1694,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335 }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1849,6 +1883,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/17/3493c5624e48fd97156ebaec380dcaafee9506d7e2c46218ceebbb57d7de/pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3", size = 19467 }, ] +[[package]] +name = "pytest-benchmark" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259 }, +] + +[[package]] +name = "pytest-codspeed" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, + { name = "pytest" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/98/16fe3895b1b8a6d537a89eecb120b97358df8f0002c6ecd11555d6304dc8/pytest_codspeed-3.2.0.tar.gz", hash = "sha256:f9d1b1a3b2c69cdc0490a1e8b1ced44bffbd0e8e21d81a7160cfdd923f6e8155", size = 18409 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/31/62b93ee025ca46016d01325f58997d32303752286bf929588c8796a25b13/pytest_codspeed-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5165774424c7ab8db7e7acdb539763a0e5657996effefdf0664d7fd95158d34", size = 26802 }, + { url = "https://files.pythonhosted.org/packages/89/60/2bc46bdf8c8ddb7e59cd9d480dc887d0ac6039f88c856d1ae3d29a4e648d/pytest_codspeed-3.2.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bd55f92d772592c04a55209950c50880413ae46876e66bd349ef157075ca26c", size = 25442 }, + { url = "https://files.pythonhosted.org/packages/31/56/1b65ba0ae1af7fd7ce14a66e7599833efe8bbd0fcecd3614db0017ca224a/pytest_codspeed-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4cf6f56067538f4892baa8d7ab5ef4e45bb59033be1ef18759a2c7fc55b32035", size = 26810 }, + { url = "https://files.pythonhosted.org/packages/23/e6/d1fafb09a1c4983372f562d9e158735229cb0b11603a61d4fad05463f977/pytest_codspeed-3.2.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39a687b05c3d145642061b45ea78e47e12f13ce510104d1a2cda00eee0e36f58", size = 25442 }, + { url = "https://files.pythonhosted.org/packages/0b/8b/9e95472589d17bb68960f2a09cfa8f02c4d43c82de55b73302bbe0fa4350/pytest_codspeed-3.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46a1afaaa1ac4c2ca5b0700d31ac46d80a27612961d031067d73c6ccbd8d3c2b", size = 27182 }, + { url = "https://files.pythonhosted.org/packages/2a/18/82aaed8095e84d829f30dda3ac49fce4e69685d769aae463614a8d864cdd/pytest_codspeed-3.2.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c48ce3af3dfa78413ed3d69d1924043aa1519048dbff46edccf8f35a25dab3c2", size = 25933 }, + { url = "https://files.pythonhosted.org/packages/e2/15/60b18d40da66e7aa2ce4c4c66d5a17de20a2ae4a89ac09a58baa7a5bc535/pytest_codspeed-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:66692506d33453df48b36a84703448cb8b22953eea51f03fbb2eb758dc2bdc4f", size = 27180 }, + { url = "https://files.pythonhosted.org/packages/51/bd/6b164d4ae07d8bea5d02ad664a9762bdb63f83c0805a3c8fe7dc6ec38407/pytest_codspeed-3.2.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:479774f80d0bdfafa16112700df4dbd31bf2a6757fac74795fd79c0a7b3c389b", size = 25923 }, + { url = "https://files.pythonhosted.org/packages/90/bb/5d73c59d750264863c25fc202bcc37c5f8a390df640a4760eba54151753e/pytest_codspeed-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:109f9f4dd1088019c3b3f887d003b7d65f98a7736ca1d457884f5aa293e8e81c", size = 26795 }, + { url = "https://files.pythonhosted.org/packages/65/17/d4bf207b63f1edc5b9c06ad77df565d186e0fd40f13459bb124304b54b1d/pytest_codspeed-3.2.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2f69a03b52c9bb041aec1b8ee54b7b6c37a6d0a948786effa4c71157765b6da", size = 25433 }, + { url = "https://files.pythonhosted.org/packages/f1/9b/952c70bd1fae9baa58077272e7f191f377c86d812263c21b361195e125e6/pytest_codspeed-3.2.0-py3-none-any.whl", hash = "sha256:54b5c2e986d6a28e7b0af11d610ea57bd5531cec8326abe486f1b55b09d91c39", size = 15007 }, +] + [[package]] name = "pytest-mock" version = "3.14.0" @@ -2176,6 +2248,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9", size = 4242 }, ] +[[package]] +name = "rich" +version = "14.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/53/830aa4c3066a8ab0ae9a9955976fb770fe9c6102117c8ec4ab3ea62d89e8/rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725", size = 224078 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229 }, +] + [[package]] name = "rpds-py" version = "0.22.3"