mirror of
https://github.com/hwchase17/langchain.git
synced 2026-07-12 19:31:24 +00:00
a2e53fda73
Switches type checking for `langchain-text-splitters` from `mypy` to [`ty`](https://docs.astral.sh/ty/), which is much faster. The `ollama` package already [switched to `ty`](https://github.com/langchain-ai/langchain/pull/36571). ## What changed The core of this PR is the config swap (`[tool.mypy]` → `[tool.ty.rules]`/`[tool.ty.analysis]`, `Makefile`, and the `typing` dependency group). Because `ty` runs with `all = "error"`, a few modules also needed source-level adjustments to satisfy the stricter analysis. These are **behavior-preserving refactors** except for one intentional fix, called out below so reviewers know where to look. ### Behavioral change (intentional fix) - `SentenceTransformersTokenTextSplitter` now raises a clear `ValueError` when the underlying model reports no maximum sequence length **and** no `tokens_per_chunk` was provided. Previously this combination reached a `None > None` comparison and surfaced as an opaque `TypeError`. As a consequence, the public `maximum_tokens_per_chunk` attribute is now honestly typed as `int | None` — it can remain `None` when the caller supplies `tokens_per_chunk` explicitly for a model without a limit. ### Behavior-preserving refactors (no user-visible change) - `TokenTextSplitter.from_tiktoken_encoder` is now an explicit override rather than the base method dispatching on `issubclass(cls, TokenTextSplitter)`. The shared length-function logic moved into a private helper. Public signatures and return types are unchanged. - `NLTKTextSplitter` builds its tokenizer once at construction, so `_tokenizer` is now always a `Callable[[str], list[str]]`. The private attributes `_language` and `_use_span_tokenize` are no longer stored — flagging in case any downstream code read those (they are underscore-private). Tokenization output is unchanged. - `HTMLSemanticPreservingSplitter` text extraction was rewritten from a `cast`-based check to `isinstance(element, Tag)` narrowing; output is equivalent for tags, text nodes, and comments. --------- Co-authored-by: Mason Daugherty <github@mdrxy.com>
110 lines
3.5 KiB
Python
110 lines
3.5 KiB
Python
"""JavaScript framework text splitter."""
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
from typing_extensions import override
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
|
|
|
|
class JSFrameworkTextSplitter(RecursiveCharacterTextSplitter):
|
|
"""Text splitter that handles React (JSX), Vue, and Svelte code.
|
|
|
|
This splitter extends `RecursiveCharacterTextSplitter` to handle React (JSX), Vue,
|
|
and Svelte code by:
|
|
|
|
1. Detecting and extracting custom component tags from the text
|
|
2. Using those tags as additional separators along with standard JS syntax
|
|
|
|
The splitter combines:
|
|
|
|
* Custom component tags as separators (e.g. `<Component`, `<div`)
|
|
* JavaScript syntax elements (function, const, if, etc)
|
|
* Standard text splitting on newlines
|
|
|
|
This allows chunks to break at natural boundaries in React, Vue, and Svelte
|
|
component code.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
separators: list[str] | None = None,
|
|
chunk_size: int = 2000,
|
|
chunk_overlap: int = 0,
|
|
**kwargs: Any,
|
|
) -> None:
|
|
"""Initialize the JS Framework text splitter.
|
|
|
|
Args:
|
|
separators: Optional list of custom separator strings to use
|
|
chunk_size: Maximum size of chunks to return
|
|
chunk_overlap: Overlap in characters between chunks
|
|
**kwargs: Additional arguments to pass to parent class
|
|
"""
|
|
super().__init__(chunk_size=chunk_size, chunk_overlap=chunk_overlap, **kwargs)
|
|
self._separators = separators or []
|
|
|
|
@override
|
|
def split_text(self, text: str) -> list[str]:
|
|
"""Split text into chunks.
|
|
|
|
This method splits the text into chunks by:
|
|
|
|
* Extracting unique opening component tags using regex
|
|
* Creating separators list with extracted tags and JS separators
|
|
* Splitting the text using the separators by calling the parent class method
|
|
|
|
Args:
|
|
text: String containing code to split
|
|
|
|
Returns:
|
|
List of text chunks split on component and JS boundaries
|
|
"""
|
|
# Extract unique opening component tags using regex
|
|
# Regex to match opening tags, excluding self-closing tags
|
|
opening_tags = re.findall(r"<\s*([a-zA-Z0-9]+)[^>]*>", text)
|
|
|
|
component_tags = []
|
|
for tag in opening_tags:
|
|
if tag not in component_tags:
|
|
component_tags.append(tag)
|
|
component_separators = [f"<{tag}" for tag in component_tags]
|
|
|
|
js_separators = [
|
|
"\nexport ",
|
|
" export ",
|
|
"\nfunction ",
|
|
"\nasync function ",
|
|
" async function ",
|
|
"\nconst ",
|
|
"\nlet ",
|
|
"\nvar ",
|
|
"\nclass ",
|
|
" class ",
|
|
"\nif ",
|
|
" if ",
|
|
"\nfor ",
|
|
" for ",
|
|
"\nwhile ",
|
|
" while ",
|
|
"\nswitch ",
|
|
" switch ",
|
|
"\ncase ",
|
|
" case ",
|
|
"\ndefault ",
|
|
" default ",
|
|
]
|
|
# Build the effective separator list for this call only.
|
|
# Do NOT assign back to self._separators: doing so would permanently
|
|
# append js_separators + component_separators on every invocation,
|
|
# causing the list to grow unboundedly when split_text() is called
|
|
# multiple times on the same instance.
|
|
separators = (
|
|
self._separators
|
|
+ js_separators
|
|
+ component_separators
|
|
+ ["<>", "\n\n", "&&\n", "||\n"]
|
|
)
|
|
return self._split_text(text, separators)
|