mirror of
https://github.com/hwchase17/langchain.git
synced 2026-07-12 19:31:24 +00:00
fd822b07c0
## Summary - Moves `nltk`, `spacy`, `sentence-transformers`, and `konlpy` imports back inside class constructors/functions so they are only loaded when the respective splitter is actually instantiated - Adds a subprocess-based regression test to verify no heavy packages are imported at `langchain_text_splitters` load time ## Why PR #32325 moved these optional dependency imports to module-level `try/except` blocks (to satisfy ruff's `PLC0415` rule). Since `__init__.py` imports all four splitter modules, this caused `import langchain_text_splitters` to eagerly load all optional heavy packages, resulting in: - A PyTorch NVML warning (`UserWarning: Can't initialize NVML`) on non-GPU machines - A ~650MB memory spike on import (74MB → 736MB), vs ~50MB in 0.3.x The fix restores the lazy import pattern with `# noqa: PLC0415` to suppress the linter rule, which is the correct trade-off when a dependency has high instantiation cost. ## Review notes - The `PLC0415` suppressions are intentional — these are optional heavy dependencies that should never be loaded unless the user explicitly instantiates the splitter class - The regression test uses a subprocess for proper isolation (the test file itself imports `langchain_text_splitters` at the top, so `sys.modules` checks within the same process would not reflect a clean import state) Fixes #35437. > **AI disclaimer:** This PR was developed with assistance from Claude Code (Anthropic AI). --------- Co-authored-by: AshwathB-debug <ashwathbalaji04@gmail.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Mason Daugherty <github@mdrxy.com>
100 lines
3.0 KiB
Python
100 lines
3.0 KiB
Python
"""Text Splitters are classes for splitting text.
|
|
|
|
!!! note
|
|
|
|
`MarkdownHeaderTextSplitter` and `HTMLHeaderTextSplitter` do not derive from
|
|
`TextSplitter`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from importlib import import_module
|
|
from typing import TYPE_CHECKING
|
|
|
|
from langchain_text_splitters.base import (
|
|
Language,
|
|
TextSplitter,
|
|
Tokenizer,
|
|
TokenTextSplitter,
|
|
split_text_on_tokens,
|
|
)
|
|
from langchain_text_splitters.character import (
|
|
CharacterTextSplitter,
|
|
RecursiveCharacterTextSplitter,
|
|
)
|
|
from langchain_text_splitters.html import (
|
|
ElementType,
|
|
HTMLHeaderTextSplitter,
|
|
HTMLSectionSplitter,
|
|
HTMLSemanticPreservingSplitter,
|
|
)
|
|
from langchain_text_splitters.json import RecursiveJsonSplitter
|
|
from langchain_text_splitters.jsx import JSFrameworkTextSplitter
|
|
from langchain_text_splitters.latex import LatexTextSplitter
|
|
from langchain_text_splitters.markdown import (
|
|
ExperimentalMarkdownSyntaxTextSplitter,
|
|
HeaderType,
|
|
LineType,
|
|
MarkdownHeaderTextSplitter,
|
|
MarkdownTextSplitter,
|
|
)
|
|
from langchain_text_splitters.python import PythonCodeTextSplitter
|
|
|
|
if TYPE_CHECKING:
|
|
from langchain_text_splitters.konlpy import KonlpyTextSplitter
|
|
from langchain_text_splitters.nltk import NLTKTextSplitter
|
|
from langchain_text_splitters.sentence_transformers import (
|
|
SentenceTransformersTokenTextSplitter,
|
|
)
|
|
from langchain_text_splitters.spacy import SpacyTextSplitter
|
|
|
|
__all__ = [
|
|
"CharacterTextSplitter",
|
|
"ElementType",
|
|
"ExperimentalMarkdownSyntaxTextSplitter",
|
|
"HTMLHeaderTextSplitter",
|
|
"HTMLSectionSplitter",
|
|
"HTMLSemanticPreservingSplitter",
|
|
"HeaderType",
|
|
"JSFrameworkTextSplitter",
|
|
"KonlpyTextSplitter",
|
|
"Language",
|
|
"LatexTextSplitter",
|
|
"LineType",
|
|
"MarkdownHeaderTextSplitter",
|
|
"MarkdownTextSplitter",
|
|
"NLTKTextSplitter",
|
|
"PythonCodeTextSplitter",
|
|
"RecursiveCharacterTextSplitter",
|
|
"RecursiveJsonSplitter",
|
|
"SentenceTransformersTokenTextSplitter",
|
|
"SpacyTextSplitter",
|
|
"TextSplitter",
|
|
"TokenTextSplitter",
|
|
"Tokenizer",
|
|
"split_text_on_tokens",
|
|
]
|
|
|
|
# Splitters whose modules pull in heavy optional dependencies (konlpy, nltk,
|
|
# spacy, sentence-transformers/torch). Deferring their import behind
|
|
# `__getattr__` keeps `import langchain_text_splitters` lightweight even
|
|
# though the classes remain in `__all__` and are fully accessible on first
|
|
# access.
|
|
_LAZY_SPLITTERS: dict[str, str] = {
|
|
"KonlpyTextSplitter": "konlpy",
|
|
"NLTKTextSplitter": "nltk",
|
|
"SentenceTransformersTokenTextSplitter": "sentence_transformers",
|
|
"SpacyTextSplitter": "spacy",
|
|
}
|
|
|
|
|
|
def __getattr__(attr_name: str) -> object:
|
|
module_name = _LAZY_SPLITTERS.get(attr_name)
|
|
if module_name is not None:
|
|
module = import_module(f".{module_name}", __name__)
|
|
result = getattr(module, attr_name)
|
|
globals()[attr_name] = result
|
|
return result
|
|
msg = f"module {__name__!r} has no attribute {attr_name!r}"
|
|
raise AttributeError(msg)
|