From fd822b07c00eefbe395ba3c285a4911f72a463de Mon Sep 17 00:00:00 2001 From: Balaji Seshadri Date: Sun, 5 Jul 2026 23:19:16 -0400 Subject: [PATCH] fix(text-splitters): restore lazy imports for heavy optional dependencies (#35469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Moves `nltk`, `spacy`, `sentence-transformers`, and `konlpy` imports back inside class constructors/functions so they are only loaded when the respective splitter is actually instantiated - Adds a subprocess-based regression test to verify no heavy packages are imported at `langchain_text_splitters` load time ## Why PR #32325 moved these optional dependency imports to module-level `try/except` blocks (to satisfy ruff's `PLC0415` rule). Since `__init__.py` imports all four splitter modules, this caused `import langchain_text_splitters` to eagerly load all optional heavy packages, resulting in: - A PyTorch NVML warning (`UserWarning: Can't initialize NVML`) on non-GPU machines - A ~650MB memory spike on import (74MB → 736MB), vs ~50MB in 0.3.x The fix restores the lazy import pattern with `# noqa: PLC0415` to suppress the linter rule, which is the correct trade-off when a dependency has high instantiation cost. ## Review notes - The `PLC0415` suppressions are intentional — these are optional heavy dependencies that should never be loaded unless the user explicitly instantiates the splitter class - The regression test uses a subprocess for proper isolation (the test file itself imports `langchain_text_splitters` at the top, so `sys.modules` checks within the same process would not reflect a clean import state) Fixes #35437. > **AI disclaimer:** This PR was developed with assistance from Claude Code (Anthropic AI). --------- Co-authored-by: AshwathB-debug Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Mason Daugherty --- .../langchain_text_splitters/__init__.py | 42 +++++- .../langchain_text_splitters/base.py | 76 ++++++----- .../langchain_text_splitters/html.py | 118 +++++++++------- .../langchain_text_splitters/konlpy.py | 20 +-- .../langchain_text_splitters/nltk.py | 17 ++- .../sentence_transformers.py | 20 ++- .../langchain_text_splitters/spacy.py | 29 ++-- .../tests/unit_tests/test_text_splitters.py | 127 ++++++++++++++++++ 8 files changed, 311 insertions(+), 138 deletions(-) diff --git a/libs/text-splitters/langchain_text_splitters/__init__.py b/libs/text-splitters/langchain_text_splitters/__init__.py index 7dc07f50b27..fe364918456 100644 --- a/libs/text-splitters/langchain_text_splitters/__init__.py +++ b/libs/text-splitters/langchain_text_splitters/__init__.py @@ -6,6 +6,11 @@ `TextSplitter`. """ +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING + from langchain_text_splitters.base import ( Language, TextSplitter, @@ -25,7 +30,6 @@ from langchain_text_splitters.html import ( ) from langchain_text_splitters.json import RecursiveJsonSplitter from langchain_text_splitters.jsx import JSFrameworkTextSplitter -from langchain_text_splitters.konlpy import KonlpyTextSplitter from langchain_text_splitters.latex import LatexTextSplitter from langchain_text_splitters.markdown import ( ExperimentalMarkdownSyntaxTextSplitter, @@ -34,12 +38,15 @@ from langchain_text_splitters.markdown import ( MarkdownHeaderTextSplitter, MarkdownTextSplitter, ) -from langchain_text_splitters.nltk import NLTKTextSplitter from langchain_text_splitters.python import PythonCodeTextSplitter -from langchain_text_splitters.sentence_transformers import ( - SentenceTransformersTokenTextSplitter, -) -from langchain_text_splitters.spacy import SpacyTextSplitter + +if TYPE_CHECKING: + from langchain_text_splitters.konlpy import KonlpyTextSplitter + from langchain_text_splitters.nltk import NLTKTextSplitter + from langchain_text_splitters.sentence_transformers import ( + SentenceTransformersTokenTextSplitter, + ) + from langchain_text_splitters.spacy import SpacyTextSplitter __all__ = [ "CharacterTextSplitter", @@ -67,3 +74,26 @@ __all__ = [ "Tokenizer", "split_text_on_tokens", ] + +# Splitters whose modules pull in heavy optional dependencies (konlpy, nltk, +# spacy, sentence-transformers/torch). Deferring their import behind +# `__getattr__` keeps `import langchain_text_splitters` lightweight even +# though the classes remain in `__all__` and are fully accessible on first +# access. +_LAZY_SPLITTERS: dict[str, str] = { + "KonlpyTextSplitter": "konlpy", + "NLTKTextSplitter": "nltk", + "SentenceTransformersTokenTextSplitter": "sentence_transformers", + "SpacyTextSplitter": "spacy", +} + + +def __getattr__(attr_name: str) -> object: + module_name = _LAZY_SPLITTERS.get(attr_name) + if module_name is not None: + module = import_module(f".{module_name}", __name__) + result = getattr(module, attr_name) + globals()[attr_name] = result + return result + msg = f"module {__name__!r} has no attribute {attr_name!r}" + raise AttributeError(msg) diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py index 320745dd55f..cabe712f26b 100644 --- a/libs/text-splitters/langchain_text_splitters/base.py +++ b/libs/text-splitters/langchain_text_splitters/base.py @@ -12,6 +12,7 @@ from typing import ( Any, Literal, TypeVar, + cast, ) from langchain_core.documents import BaseDocumentTransformer, Document @@ -21,26 +22,40 @@ if TYPE_CHECKING: from collections.abc import Callable, Collection, Iterable, Sequence from collections.abc import Set as AbstractSet - -try: - import tiktoken - - _HAS_TIKTOKEN = True -except ImportError: - _HAS_TIKTOKEN = False - -try: from transformers.tokenization_utils_base import PreTrainedTokenizerBase - _HAS_TRANSFORMERS = True -except ImportError: - _HAS_TRANSFORMERS = False - logger = logging.getLogger(__name__) TS = TypeVar("TS", bound="TextSplitter") +def _import_tiktoken() -> object: + try: + import tiktoken # noqa: PLC0415 + except ImportError as err: + msg = ( + "Could not import tiktoken python package. " + "This is needed in order to calculate max_tokens_for_prompt. " + "Please install it with `pip install tiktoken`." + ) + raise ImportError(msg) from err + return tiktoken + + +def _import_pretrained_tokenizer_base() -> type[PreTrainedTokenizerBase]: + try: + from transformers.tokenization_utils_base import ( # noqa: PLC0415 + PreTrainedTokenizerBase, + ) + except ImportError as err: + msg = ( + "Could not import transformers python package. " + "Please install it with `pip install transformers`." + ) + raise ValueError(msg) from err + return PreTrainedTokenizerBase + + class TextSplitter(BaseDocumentTransformer, ABC): """Interface for splitting text into chunks.""" @@ -206,14 +221,9 @@ class TextSplitter(BaseDocumentTransformer, ABC): An instance of `TextSplitter` using the Hugging Face tokenizer for length calculation. """ - if not _HAS_TRANSFORMERS: - msg = ( - "Could not import transformers python package. " - "Please install it with `pip install transformers`." - ) - raise ValueError(msg) + pretrained_tokenizer_base = _import_pretrained_tokenizer_base() - if not isinstance(tokenizer, PreTrainedTokenizerBase): + if not isinstance(tokenizer, pretrained_tokenizer_base): msg = "Tokenizer received was not an instance of PreTrainedTokenizerBase" raise ValueError(msg) # noqa: TRY004 @@ -250,13 +260,7 @@ class TextSplitter(BaseDocumentTransformer, ABC): """ if allowed_special is None: allowed_special = set() - if not _HAS_TIKTOKEN: - msg = ( - "Could not import tiktoken python package. " - "This is needed in order to calculate max_tokens_for_prompt. " - "Please install it with `pip install tiktoken`." - ) - raise ImportError(msg) + tiktoken = cast("Any", _import_tiktoken()) if model_name is not None: enc = tiktoken.encoding_for_model(model_name) @@ -344,13 +348,15 @@ class TokenTextSplitter(TextSplitter): if allowed_special is None: allowed_special = set() super().__init__(**kwargs) - if not _HAS_TIKTOKEN: + try: + tiktoken = cast("Any", _import_tiktoken()) + except ImportError as err: msg = ( "Could not import tiktoken python package. " "This is needed in order to for TokenTextSplitter. " "Please install it with `pip install tiktoken`." ) - raise ImportError(msg) + raise ImportError(msg) from err if model_name is not None: enc = tiktoken.encoding_for_model(model_name) @@ -419,10 +425,14 @@ class TokenTextSplitter(TextSplitter): """ def _encode(_text: str) -> list[int]: - return self._tokenizer.encode( - _text, - allowed_special=self._allowed_special, - disallowed_special=self._disallowed_special, + # `tiktoken` is lazy-imported, so mypy cannot infer the encoder return. + return cast( + "list[int]", + self._tokenizer.encode( + _text, + allowed_special=self._allowed_special, + disallowed_special=self._disallowed_special, + ), ) tokenizer = Tokenizer( diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py index 818d003751a..e22977643fd 100644 --- a/libs/text-splitters/langchain_text_splitters/html.py +++ b/libs/text-splitters/langchain_text_splitters/html.py @@ -24,29 +24,8 @@ from langchain_text_splitters.character import RecursiveCharacterTextSplitter if TYPE_CHECKING: from collections.abc import Callable, Iterable, Iterator, Sequence - from bs4.element import ResultSet - -try: - import nltk - - _HAS_NLTK = True -except ImportError: - _HAS_NLTK = False - -try: from bs4 import BeautifulSoup, Tag - from bs4.element import NavigableString, PageElement - - _HAS_BS4 = True -except ImportError: - _HAS_BS4 = False - -try: - from lxml import etree - - _HAS_LXML = True -except ImportError: - _HAS_LXML = False + from bs4.element import NavigableString, PageElement, ResultSet class ElementType(TypedDict): @@ -58,6 +37,35 @@ class ElementType(TypedDict): metadata: dict[str, str] +def _import_bs4( + *, import_error_message: str +) -> tuple[type[BeautifulSoup], type[Tag], type[NavigableString]]: + try: + from bs4 import BeautifulSoup, Tag # noqa: PLC0415 + from bs4.element import NavigableString # noqa: PLC0415 + except ImportError as err: + raise ImportError(import_error_message) from err + return BeautifulSoup, Tag, NavigableString + + +def _import_lxml_etree() -> object: + try: + from lxml import etree # noqa: PLC0415 + except ImportError as err: + msg = "Unable to import lxml, please install with `pip install lxml`." + raise ImportError(msg) from err + return etree + + +def _import_nltk() -> object: + try: + import nltk # noqa: PLC0415 + except ImportError as err: + msg = "Could not import nltk. Please install it with 'pip install nltk'." + raise ImportError(msg) from err + return nltk + + # Unfortunately, BeautifulSoup doesn't define overloads for Tag.find_all. # So doing the type resolution ourselves. @@ -257,13 +265,13 @@ class HTMLHeaderTextSplitter: Raises: ImportError: If BeautifulSoup is not installed. """ - if not _HAS_BS4: - msg = ( + beautiful_soup, tag_cls, _ = _import_bs4( + import_error_message=( "Unable to import BeautifulSoup. Please install via `pip install bs4`." ) - raise ImportError(msg) + ) - soup = BeautifulSoup(html_content, "html.parser") + soup = beautiful_soup(html_content, "html.parser") body = soup.body or soup # Dictionary of active headers: @@ -292,7 +300,7 @@ class HTMLHeaderTextSplitter: children = list(node.children) stack.extend( - child for child in reversed(children) if isinstance(child, Tag) + child for child in reversed(children) if isinstance(child, tag_cls) ) tag = getattr(node, "name", None) @@ -465,13 +473,14 @@ class HTMLSectionSplitter: Raises: ImportError: If BeautifulSoup is not installed. """ - if not _HAS_BS4: - msg = "Unable to import BeautifulSoup/PageElement, \ - please install with `pip install \ - bs4`." - raise ImportError(msg) + beautiful_soup, _, _ = _import_bs4( + import_error_message=( + "Unable to import BeautifulSoup/PageElement, " + "please install with `pip install bs4`." + ) + ) - soup = BeautifulSoup(html_doc, "html.parser") + soup = beautiful_soup(html_doc, "html.parser") header_names = list(self.headers_to_split_on.keys()) sections: list[dict[str, str | None]] = [] @@ -520,9 +529,7 @@ class HTMLSectionSplitter: Raises: ImportError: If the `lxml` library is not installed. """ - if not _HAS_LXML: - msg = "Unable to import lxml, please install with `pip install lxml`." - raise ImportError(msg) + etree = cast("Any", _import_lxml_etree()) # use lxml library to parse html document and return xml ElementTree # Create secure parsers to prevent XXE attacks html_parser = etree.HTMLParser(no_network=True) @@ -532,8 +539,7 @@ class HTMLSectionSplitter: # Apply XSLT access control to prevent file/network access # DENY_ALL is a predefined access control that blocks all file/network access - # Type ignore needed due to incomplete lxml type stubs - ac = etree.XSLTAccessControl.DENY_ALL # ty: ignore[unresolved-attribute] + ac = etree.XSLTAccessControl.DENY_ALL tree = etree.parse(StringIO(html_content), html_parser) xslt_tree = etree.parse(self.xslt_path, xslt_parser) @@ -670,12 +676,12 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer): ImportError: If BeautifulSoup or NLTK (when stopword removal is enabled) is not installed. """ - if not _HAS_BS4: - msg = ( + _import_bs4( + import_error_message=( "Could not import BeautifulSoup. " "Please install it with 'pip install bs4'." ) - raise ImportError(msg) + ) self._headers_to_split_on = sorted(headers_to_split_on) self._max_chunk_size = max_chunk_size @@ -718,11 +724,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer): ) if self._stopword_removal: - if not _HAS_NLTK: - msg = ( - "Could not import nltk. Please install it with 'pip install nltk'." - ) - raise ImportError(msg) + nltk = cast("Any", _import_nltk()) nltk.download("stopwords") self._stopwords = set(nltk.corpus.stopwords.words(self._stopword_lang)) @@ -735,7 +737,13 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer): Returns: A list of `Document` objects containing the split content. """ - soup = BeautifulSoup(text, "html.parser") + beautiful_soup, _, _ = _import_bs4( + import_error_message=( + "Could not import BeautifulSoup. " + "Please install it with 'pip install bs4'." + ) + ) + soup = beautiful_soup(text, "html.parser") self._process_media(soup) @@ -813,13 +821,19 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer): Args: soup: Parsed HTML content using BeautifulSoup. """ + _, _, navigable_string = _import_bs4( + import_error_message=( + "Could not import BeautifulSoup. " + "Please install it with 'pip install bs4'." + ) + ) for a_tag in _find_all_tags(soup, name="a"): a_href = a_tag.get("href", "") a_text = a_tag.get_text(strip=True) markdown_link = f"[{a_text}]({a_href})" wrapper = soup.new_tag("link-wrapper") wrapper.string = markdown_link - a_tag.replace_with(NavigableString(markdown_link)) + a_tag.replace_with(navigable_string(markdown_link)) def _filter_tags(self, soup: BeautifulSoup) -> None: """Filters the HTML content based on the allowlist and denylist tags. @@ -866,6 +880,12 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer): Returns: A list of `Document` objects containing the split content. """ + _, tag_cls, _ = _import_bs4( + import_error_message=( + "Could not import BeautifulSoup. " + "Please install it with 'pip install bs4'." + ) + ) documents: list[Document] = [] current_headers: dict[str, str] = {} current_content: list[str] = [] @@ -884,7 +904,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer): The processed text of the element, or an empty string for elements with no extractable text. """ - if isinstance(element, Tag): + if isinstance(element, tag_cls): if element.name in self._custom_handlers: return self._custom_handlers[element.name](element) diff --git a/libs/text-splitters/langchain_text_splitters/konlpy.py b/libs/text-splitters/langchain_text_splitters/konlpy.py index e9404431c92..6f999d66687 100644 --- a/libs/text-splitters/langchain_text_splitters/konlpy.py +++ b/libs/text-splitters/langchain_text_splitters/konlpy.py @@ -8,13 +8,6 @@ from typing_extensions import override from langchain_text_splitters.base import TextSplitter -try: - import konlpy - - _HAS_KONLPY = True -except ImportError: - _HAS_KONLPY = False - class KonlpyTextSplitter(TextSplitter): """Splitting text using Konlpy package. @@ -37,12 +30,13 @@ class KonlpyTextSplitter(TextSplitter): """ super().__init__(**kwargs) self._separator = separator - if not _HAS_KONLPY: - msg = """ - Konlpy is not installed, please install it with - `pip install konlpy` - """ - raise ImportError(msg) + try: + import konlpy # noqa: PLC0415 + except ImportError as err: + msg = ( + "Konlpy is not installed, please install it with `pip install konlpy`." + ) + raise ImportError(msg) from err self.kkma = konlpy.tag.Kkma() @override diff --git a/libs/text-splitters/langchain_text_splitters/nltk.py b/libs/text-splitters/langchain_text_splitters/nltk.py index 9951b5299e4..a720b51ff18 100644 --- a/libs/text-splitters/langchain_text_splitters/nltk.py +++ b/libs/text-splitters/langchain_text_splitters/nltk.py @@ -11,13 +11,6 @@ from langchain_text_splitters.base import TextSplitter if TYPE_CHECKING: from collections.abc import Callable -try: - import nltk - - _HAS_NLTK = True -except ImportError: - _HAS_NLTK = False - class NLTKTextSplitter(TextSplitter): """Splitting text using NLTK package.""" @@ -47,9 +40,11 @@ class NLTKTextSplitter(TextSplitter): if use_span_tokenize and self._separator: msg = "When use_span_tokenize is True, separator should be ''" raise ValueError(msg) - if not _HAS_NLTK: + try: + import nltk # noqa: PLC0415,F401 + except ImportError as err: msg = "NLTK is not installed, please install it with `pip install nltk`." - raise ImportError(msg) + raise ImportError(msg) from err if use_span_tokenize: self._tokenizer = self._span_tokenizer(language) else: @@ -57,10 +52,14 @@ class NLTKTextSplitter(TextSplitter): @staticmethod def _sent_tokenizer(language: str) -> Callable[[str], list[str]]: + import nltk # noqa: PLC0415 + return lambda text: nltk.tokenize.sent_tokenize(text, language) @staticmethod def _span_tokenizer(language: str) -> Callable[[str], list[str]]: + import nltk # noqa: PLC0415 + tokenizer = nltk.tokenize._get_punkt_tokenizer(language) # noqa: SLF001 def _tokenize(text: str) -> list[str]: diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py index ae07bf5dd13..d77b804b3e5 100644 --- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py +++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py @@ -2,21 +2,13 @@ from __future__ import annotations +from importlib import import_module from typing import Any, cast from typing_extensions import override from langchain_text_splitters.base import TextSplitter, Tokenizer, split_text_on_tokens -try: - from sentence_transformers import ( - SentenceTransformer, - ) - - _HAS_SENTENCE_TRANSFORMERS = True -except ImportError: - _HAS_SENTENCE_TRANSFORMERS = False - class SentenceTransformersTokenTextSplitter(TextSplitter): """Splitting text to tokens using sentence model tokenizer.""" @@ -42,19 +34,23 @@ class SentenceTransformersTokenTextSplitter(TextSplitter): Raises: ImportError: If the `sentence_transformers` package is not installed. + ValueError: If `tokens_per_chunk` exceeds the model's maximum token limit. """ super().__init__(**kwargs, chunk_overlap=chunk_overlap) - if not _HAS_SENTENCE_TRANSFORMERS: + try: + sentence_transformers = cast("Any", import_module("sentence_transformers")) + sentence_transformer_cls = sentence_transformers.SentenceTransformer + except ImportError as err: msg = ( "Could not import sentence_transformers python package. " "This is needed in order to use SentenceTransformersTokenTextSplitter. " "Please install it with `pip install sentence-transformers`." ) - raise ImportError(msg) + raise ImportError(msg) from err self.model_name = model_name - self._model = SentenceTransformer(self.model_name, **(model_kwargs or {})) + self._model = sentence_transformer_cls(self.model_name, **(model_kwargs or {})) self.tokenizer = self._model.tokenizer self._initialize_chunk_configuration(tokens_per_chunk=tokens_per_chunk) diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.py index c61cf490ffd..14bcdcc5e37 100644 --- a/libs/text-splitters/langchain_text_splitters/spacy.py +++ b/libs/text-splitters/langchain_text_splitters/spacy.py @@ -2,24 +2,18 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from importlib import import_module +from typing import TYPE_CHECKING, Any, cast from typing_extensions import override from langchain_text_splitters.base import TextSplitter -try: - import spacy - from spacy.lang.en import English - - if TYPE_CHECKING: - from spacy.language import ( - Language, - ) - - _HAS_SPACY = True -except ImportError: - _HAS_SPACY = False +if TYPE_CHECKING: + # Type ignores needed as long as spacy doesn't support Python 3.14. + from spacy.language import ( # type: ignore[import-not-found, unused-ignore] + Language, + ) class SpacyTextSplitter(TextSplitter): @@ -60,11 +54,14 @@ class SpacyTextSplitter(TextSplitter): def _make_spacy_pipeline_for_splitting( pipeline: str, *, max_length: int = 1_000_000 ) -> Language: - if not _HAS_SPACY: + try: + spacy = cast("Any", import_module("spacy")) + english_cls = cast("Any", import_module("spacy.lang.en")).English + except ImportError as err: msg = "Spacy is not installed, please install it with `pip install spacy`." - raise ImportError(msg) + raise ImportError(msg) from err if pipeline == "sentencizer": - sentencizer: Language = English() + sentencizer: Language = english_cls() sentencizer.add_pipe("sentencizer") else: sentencizer = spacy.load(pipeline, exclude=["ner", "tagger"]) diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py index cd680af6cee..e226501d732 100644 --- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py +++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py @@ -54,6 +54,133 @@ def bar(): """ +def test_no_heavy_imports_on_package_load() -> None: + """Ensure importing the package does not eagerly import heavy dependencies. + + Runs in a fresh interpreter so the result is unaffected by modules the test + session already imported. A `sys.meta_path` finder records any *attempt* to + import a heavy optional dependency, so the guard holds whether or not those + packages are installed in the current environment (a plain `sys.modules` check + would pass vacuously when the packages are absent). + """ + import subprocess # noqa: PLC0415 + import sys # noqa: PLC0415 + + script = textwrap.dedent( + """ + import sys + + blocked = { + "nltk", "spacy", "sentence_transformers", "konlpy", "torch", + "transformers", "tiktoken", + } + attempted = [] + + class _Recorder: + def find_spec(self, name, path=None, target=None): + if name.split(".")[0] in blocked: + attempted.append(name.split(".")[0]) + return None # defer to the real finders + + sys.meta_path.insert(0, _Recorder()) + import langchain_text_splitters # noqa: F401 + print(",".join(sorted(set(attempted)))) + """ + ) + result = subprocess.run( # noqa: S603 # list args, no shell; input is static + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + assert result.returncode == 0, ( + f"Importing langchain_text_splitters failed:\n{result.stderr}" + ) + attempted = [p for p in result.stdout.strip().split(",") if p] + assert not attempted, ( + f"Heavy packages imported at langchain_text_splitters load time: {attempted}" + ) + + +@pytest.mark.parametrize( + ("module_name", "expected_message"), + [ + ("konlpy", "pip install konlpy"), + ("nltk", "pip install nltk"), + ("spacy", "pip install spacy"), + ("sentence_transformers", "pip install sentence-transformers"), + ], +) +def test_missing_optional_dependency_raises_importerror( + module_name: str, + expected_message: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Each splitter raises a helpful ImportError when its optional dep is missing. + + The missing dependency is simulated by forcing its import to fail, so the test + is independent of whether the optional package is actually installed. + """ + import sys # noqa: PLC0415 + + from langchain_text_splitters.konlpy import KonlpyTextSplitter # noqa: PLC0415 + from langchain_text_splitters.nltk import NLTKTextSplitter # noqa: PLC0415 + from langchain_text_splitters.sentence_transformers import ( # noqa: PLC0415 + SentenceTransformersTokenTextSplitter, + ) + from langchain_text_splitters.spacy import SpacyTextSplitter # noqa: PLC0415 + + constructors: dict[str, Callable[[], TextSplitter]] = { + "konlpy": KonlpyTextSplitter, + "nltk": NLTKTextSplitter, + "spacy": SpacyTextSplitter, + "sentence_transformers": SentenceTransformersTokenTextSplitter, + } + + # `None` in sys.modules makes both `import x` and `import_module(x)` raise + # ImportError, exercising the splitter's missing-dependency branch. + monkeypatch.setitem(sys.modules, module_name, None) + with pytest.raises(ImportError, match=re.escape(expected_message)): + constructors[module_name]() + + +@pytest.mark.parametrize( + "class_name", + [ + "KonlpyTextSplitter", + "NLTKTextSplitter", + "SpacyTextSplitter", + "SentenceTransformersTokenTextSplitter", + ], +) +def test_lazy_getattr_resolves(class_name: str) -> None: + """`__getattr__` resolves lazy splitter classes from the package namespace.""" + import langchain_text_splitters as lts # noqa: PLC0415 + + try: + cls = getattr(lts, class_name) + except ImportError: + pytest.skip(f"Optional dependency for {class_name} not installed") + assert isinstance(cls, type), f"{class_name} should be a class, got {type(cls)}" + + +def test_lazy_getattr_raises_for_unknown() -> None: + """Accessing an unknown attribute raises `AttributeError`.""" + import langchain_text_splitters as lts # noqa: PLC0415 + + with pytest.raises(AttributeError, match="no_such_thing"): + _ = lts.no_such_thing # type: ignore[attr-defined] + + +def test_lightweight_splitters_remain_eagerly_accessible() -> None: + """Lightweight splitters are still directly importable from the package.""" + import langchain_text_splitters as lts # noqa: PLC0415 + + assert issubclass(lts.RecursiveCharacterTextSplitter, lts.TextSplitter) + assert issubclass(lts.CharacterTextSplitter, lts.TextSplitter) + + def test_character_text_splitter() -> None: """Test splitting by character count.""" text = "foo bar baz 123"