[performance]: Adding benchmarks for common langchain-core imports (#30747)

The first in a sequence of PRs focusing on improving performance in
core. We're starting with reducing import times for common structures,
hence the benchmarks here.

The benchmark looks a little bit complicated - we have to use a process
so that we don't suffer from Python's import caching system. I tried
doing manual modification of `sys.modules` between runs, but that's
pretty tricky / hacky to get right, hence the subprocess approach.

Motivated by extremely slow baseline for common imports (we're talking
2-5 seconds):

<img width="633" alt="Screenshot 2025-04-09 at 12 48 12 PM"
src="https://github.com/user-attachments/assets/994616fe-1798-404d-bcbe-48ad0eb8a9a0"
/>

Also added a `make benchmark` command to make local runs easy :).
Currently using walltimes so that we can track total time despite using
a manual proces.
This commit is contained in:
Sydney Runkle
2025-04-09 13:00:15 -04:00
committed by GitHub
parent 5fb261ce27
commit 78ec7d886d
8 changed files with 169 additions and 1 deletions

View File

View File

@@ -0,0 +1,38 @@
import subprocess
import sys
import pytest
from pytest_benchmark.fixture import BenchmarkFixture # type: ignore
@pytest.mark.parametrize(
"import_path",
[
pytest.param(
"from langchain_core.messages import HumanMessage", id="HumanMessage"
),
pytest.param("from langchain_core.tools import tool", id="tool"),
pytest.param(
"from langchain_core.callbacks import CallbackManager", id="CallbackManager"
),
pytest.param("from langchain_core.runnables import Runnable", id="Runnable"),
pytest.param(
"from langchain_core.language_models import BaseChatModel",
id="BaseChatModel",
),
pytest.param(
"from langchain_core.prompts import ChatPromptTemplate",
id="PromChatPromptTemplateptTemplate",
),
pytest.param("from langchain_core.documents import Document", id="Document"),
pytest.param(
"from langchain_core.vectorstores import InMemoryVectorStore",
id="InMemoryVectorStore",
),
],
)
@pytest.mark.benchmark
def test_import_time(benchmark: BenchmarkFixture, import_path: str) -> None:
@benchmark
def import_in_subprocess() -> None:
subprocess.run([sys.executable, "-c", import_path], check=False)