mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-14 05:56:40 +00:00
[performance]: Adding benchmarks for common langchain-core
imports (#30747)
The first in a sequence of PRs focusing on improving performance in core. We're starting with reducing import times for common structures, hence the benchmarks here. The benchmark looks a little bit complicated - we have to use a process so that we don't suffer from Python's import caching system. I tried doing manual modification of `sys.modules` between runs, but that's pretty tricky / hacky to get right, hence the subprocess approach. Motivated by extremely slow baseline for common imports (we're talking 2-5 seconds): <img width="633" alt="Screenshot 2025-04-09 at 12 48 12 PM" src="https://github.com/user-attachments/assets/994616fe-1798-404d-bcbe-48ad0eb8a9a0" /> Also added a `make benchmark` command to make local runs easy :). Currently using walltimes so that we can track total time despite using a manual proces.
This commit is contained in:
0
libs/core/tests/benchmarks/__init__.py
Normal file
0
libs/core/tests/benchmarks/__init__.py
Normal file
38
libs/core/tests/benchmarks/test_imports.py
Normal file
38
libs/core/tests/benchmarks/test_imports.py
Normal file
@@ -0,0 +1,38 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from pytest_benchmark.fixture import BenchmarkFixture # type: ignore
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"import_path",
|
||||
[
|
||||
pytest.param(
|
||||
"from langchain_core.messages import HumanMessage", id="HumanMessage"
|
||||
),
|
||||
pytest.param("from langchain_core.tools import tool", id="tool"),
|
||||
pytest.param(
|
||||
"from langchain_core.callbacks import CallbackManager", id="CallbackManager"
|
||||
),
|
||||
pytest.param("from langchain_core.runnables import Runnable", id="Runnable"),
|
||||
pytest.param(
|
||||
"from langchain_core.language_models import BaseChatModel",
|
||||
id="BaseChatModel",
|
||||
),
|
||||
pytest.param(
|
||||
"from langchain_core.prompts import ChatPromptTemplate",
|
||||
id="PromChatPromptTemplateptTemplate",
|
||||
),
|
||||
pytest.param("from langchain_core.documents import Document", id="Document"),
|
||||
pytest.param(
|
||||
"from langchain_core.vectorstores import InMemoryVectorStore",
|
||||
id="InMemoryVectorStore",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.benchmark
|
||||
def test_import_time(benchmark: BenchmarkFixture, import_path: str) -> None:
|
||||
@benchmark
|
||||
def import_in_subprocess() -> None:
|
||||
subprocess.run([sys.executable, "-c", import_path], check=False)
|
Reference in New Issue
Block a user