mirror of
https://github.com/hwchase17/langchain.git
synced 2026-07-13 03:42:57 +00:00
fd822b07c0
## Summary - Moves `nltk`, `spacy`, `sentence-transformers`, and `konlpy` imports back inside class constructors/functions so they are only loaded when the respective splitter is actually instantiated - Adds a subprocess-based regression test to verify no heavy packages are imported at `langchain_text_splitters` load time ## Why PR #32325 moved these optional dependency imports to module-level `try/except` blocks (to satisfy ruff's `PLC0415` rule). Since `__init__.py` imports all four splitter modules, this caused `import langchain_text_splitters` to eagerly load all optional heavy packages, resulting in: - A PyTorch NVML warning (`UserWarning: Can't initialize NVML`) on non-GPU machines - A ~650MB memory spike on import (74MB → 736MB), vs ~50MB in 0.3.x The fix restores the lazy import pattern with `# noqa: PLC0415` to suppress the linter rule, which is the correct trade-off when a dependency has high instantiation cost. ## Review notes - The `PLC0415` suppressions are intentional — these are optional heavy dependencies that should never be loaded unless the user explicitly instantiates the splitter class - The regression test uses a subprocess for proper isolation (the test file itself imports `langchain_text_splitters` at the top, so `sys.modules` checks within the same process would not reflect a clean import state) Fixes #35437. > **AI disclaimer:** This PR was developed with assistance from Claude Code (Anthropic AI). --------- Co-authored-by: AshwathB-debug <ashwathbalaji04@gmail.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Mason Daugherty <github@mdrxy.com>
70 lines
2.2 KiB
Python
70 lines
2.2 KiB
Python
"""Spacy text splitter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from importlib import import_module
|
|
from typing import TYPE_CHECKING, Any, cast
|
|
|
|
from typing_extensions import override
|
|
|
|
from langchain_text_splitters.base import TextSplitter
|
|
|
|
if TYPE_CHECKING:
|
|
# Type ignores needed as long as spacy doesn't support Python 3.14.
|
|
from spacy.language import ( # type: ignore[import-not-found, unused-ignore]
|
|
Language,
|
|
)
|
|
|
|
|
|
class SpacyTextSplitter(TextSplitter):
|
|
"""Splitting text using Spacy package.
|
|
|
|
Per default, Spacy's `en_core_web_sm` model is used and
|
|
its default max_length is 1000000 (it is the length of maximum character
|
|
this model takes which can be increased for large files). For a faster, but
|
|
potentially less accurate splitting, you can use `pipeline='sentencizer'`.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
separator: str = "\n\n",
|
|
pipeline: str = "en_core_web_sm",
|
|
max_length: int = 1_000_000,
|
|
*,
|
|
strip_whitespace: bool = True,
|
|
**kwargs: Any,
|
|
) -> None:
|
|
"""Initialize the spacy text splitter."""
|
|
super().__init__(**kwargs)
|
|
self._tokenizer = _make_spacy_pipeline_for_splitting(
|
|
pipeline, max_length=max_length
|
|
)
|
|
self._separator = separator
|
|
self._strip_whitespace = strip_whitespace
|
|
|
|
@override
|
|
def split_text(self, text: str) -> list[str]:
|
|
splits = (
|
|
s.text if self._strip_whitespace else s.text_with_ws
|
|
for s in self._tokenizer(text).sents
|
|
)
|
|
return self._merge_splits(splits, self._separator)
|
|
|
|
|
|
def _make_spacy_pipeline_for_splitting(
|
|
pipeline: str, *, max_length: int = 1_000_000
|
|
) -> Language:
|
|
try:
|
|
spacy = cast("Any", import_module("spacy"))
|
|
english_cls = cast("Any", import_module("spacy.lang.en")).English
|
|
except ImportError as err:
|
|
msg = "Spacy is not installed, please install it with `pip install spacy`."
|
|
raise ImportError(msg) from err
|
|
if pipeline == "sentencizer":
|
|
sentencizer: Language = english_cls()
|
|
sentencizer.add_pipe("sentencizer")
|
|
else:
|
|
sentencizer = spacy.load(pipeline, exclude=["ner", "tagger"])
|
|
sentencizer.max_length = max_length
|
|
return sentencizer
|