mirror of
https://github.com/hwchase17/langchain.git
synced 2026-07-11 18:18:51 +00:00
a2e53fda73
Switches type checking for `langchain-text-splitters` from `mypy` to [`ty`](https://docs.astral.sh/ty/), which is much faster. The `ollama` package already [switched to `ty`](https://github.com/langchain-ai/langchain/pull/36571). ## What changed The core of this PR is the config swap (`[tool.mypy]` → `[tool.ty.rules]`/`[tool.ty.analysis]`, `Makefile`, and the `typing` dependency group). Because `ty` runs with `all = "error"`, a few modules also needed source-level adjustments to satisfy the stricter analysis. These are **behavior-preserving refactors** except for one intentional fix, called out below so reviewers know where to look. ### Behavioral change (intentional fix) - `SentenceTransformersTokenTextSplitter` now raises a clear `ValueError` when the underlying model reports no maximum sequence length **and** no `tokens_per_chunk` was provided. Previously this combination reached a `None > None` comparison and surfaced as an opaque `TypeError`. As a consequence, the public `maximum_tokens_per_chunk` attribute is now honestly typed as `int | None` — it can remain `None` when the caller supplies `tokens_per_chunk` explicitly for a model without a limit. ### Behavior-preserving refactors (no user-visible change) - `TokenTextSplitter.from_tiktoken_encoder` is now an explicit override rather than the base method dispatching on `issubclass(cls, TokenTextSplitter)`. The shared length-function logic moved into a private helper. Public signatures and return types are unchanged. - `NLTKTextSplitter` builds its tokenizer once at construction, so `_tokenizer` is now always a `Callable[[str], list[str]]`. The private attributes `_language` and `_use_span_tokenize` are no longer stored — flagging in case any downstream code read those (they are underscore-private). Tokenization output is unchanged. - `HTMLSemanticPreservingSplitter` text extraction was rewritten from a `cast`-based check to `isinstance(element, Tag)` narrowing; output is equivalent for tags, text nodes, and comments. --------- Co-authored-by: Mason Daugherty <github@mdrxy.com>
144 lines
4.5 KiB
TOML
144 lines
4.5 KiB
TOML
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "langchain-text-splitters"
|
|
description = "LangChain text splitting utilities"
|
|
license = { text = "MIT" }
|
|
readme = "README.md"
|
|
classifiers = [
|
|
"Development Status :: 5 - Production/Stable",
|
|
"Intended Audience :: Developers",
|
|
"License :: OSI Approved :: MIT License",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Programming Language :: Python :: 3.14",
|
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
"Topic :: Text Processing",
|
|
]
|
|
|
|
version = "1.1.2"
|
|
requires-python = ">=3.10.0,<4.0.0"
|
|
dependencies = [
|
|
"langchain-core>=1.4.7,<2.0.0",
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://docs.langchain.com/"
|
|
Documentation = "https://docs.langchain.com/"
|
|
Repository = "https://github.com/langchain-ai/langchain"
|
|
Issues = "https://github.com/langchain-ai/langchain/issues"
|
|
Changelog = "https://github.com/langchain-ai/langchain/releases?q=%22langchain-text-splitters%22"
|
|
Twitter = "https://x.com/langchain_oss"
|
|
Slack = "https://www.langchain.com/join-community"
|
|
Reddit = "https://www.reddit.com/r/LangChain/"
|
|
|
|
[dependency-groups]
|
|
lint = [
|
|
"ruff>=0.15.0,<0.16.0",
|
|
]
|
|
typing = [
|
|
"ty>=0.0.56,<0.1.0",
|
|
"lxml-stubs>=0.5.1,<1.0.0",
|
|
"types-requests>=2.31.0.20240218,<3.0.0.0",
|
|
"spacy>=3.8.13,<4.0.0",
|
|
"nltk>=3.9.1,<4.0.0",
|
|
"transformers>=4.51.3,<6.0.0",
|
|
"sentence-transformers>=5.3.0,<6.0.0",
|
|
"tiktoken>=0.8.0,<1.0.0",
|
|
]
|
|
dev = [
|
|
"jupyter<2.0.0,>=1.0.0",
|
|
]
|
|
test = [
|
|
"pytest>=9.0.3,<10.0.0",
|
|
"freezegun>=1.2.2,<2.0.0",
|
|
"pytest-mock>=3.10.0,<4.0.0",
|
|
"pytest-watcher>=0.3.4,<1.0.0",
|
|
"pytest-asyncio>=1.3.0,<2.0.0",
|
|
"pytest-socket>=0.7.0,<1.0.0",
|
|
"pytest-xdist<4.0.0,>=3.6.1",
|
|
]
|
|
test_integration = [
|
|
"spacy>=3.8.13,!=3.8.14,<4.0.0",
|
|
"nltk>=3.9.1,<4.0.0",
|
|
"transformers>=4.51.3,<6.0.0",
|
|
"sentence-transformers>=5.3.0,<6.0.0",
|
|
"tiktoken>=0.8.0,<1.0.0",
|
|
"en-core-web-sm",
|
|
]
|
|
|
|
[tool.uv]
|
|
constraint-dependencies = ["pygments>=2.20.0"] # CVE-2026-4539
|
|
|
|
[tool.uv.sources]
|
|
langchain-core = { path = "../core", editable = true }
|
|
en-core-web-sm = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }
|
|
|
|
[tool.ty.rules]
|
|
all = "error"
|
|
|
|
[tool.ty.analysis]
|
|
allowed-unresolved-imports = ["konlpy.**"]
|
|
|
|
[tool.ruff.format]
|
|
docstring-code-format = true
|
|
|
|
[tool.ruff.lint]
|
|
select = [ "ALL",]
|
|
ignore = [
|
|
"C90", # McCabe complexity
|
|
"COM812", # Messes with the formatter
|
|
"CPY", # No copyright
|
|
"FIX002", # Line contains TODO
|
|
"PERF203", # Rarely useful
|
|
"PLR09", # Too many something (arg, statements, etc)
|
|
"TD002", # Missing author in TODO
|
|
"TD003", # Missing issue link in TODO
|
|
]
|
|
unfixable = [
|
|
"B028", # People should intentionally tune the stacklevel
|
|
]
|
|
|
|
flake8-annotations.allow-star-arg-any = true
|
|
flake8-annotations.mypy-init-return = true
|
|
flake8-type-checking.runtime-evaluated-base-classes = ["pydantic.BaseModel","langchain_core.load.serializable.Serializable","langchain_core.runnables.base.RunnableSerializable"]
|
|
pep8-naming.classmethod-decorators = [ "classmethod", "langchain_core.utils.pydantic.pre_init", "pydantic.field_validator", "pydantic.v1.root_validator",]
|
|
|
|
[tool.ruff.lint.pydocstyle]
|
|
convention = "google"
|
|
ignore-var-parameters = true # ignore missing documentation for *args and **kwargs parameters
|
|
|
|
[tool.ruff.lint.flake8-tidy-imports]
|
|
ban-relative-imports = "all"
|
|
|
|
[tool.ruff.lint.per-file-ignores]
|
|
"scripts/**" = [
|
|
"D1", # Docstrings not mandatory in scripts
|
|
"INP001", # Not a package
|
|
"S311" # Standard pseudo-random generators are not suitable for cryptographic purposes
|
|
]
|
|
"tests/**" = [
|
|
"D1", # Docstrings not mandatory in tests
|
|
"PLR2004", # Magic value comparisons
|
|
"S101", # Tests need assertions
|
|
"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
|
|
"SLF001" # Private member access in tests
|
|
]
|
|
|
|
[tool.coverage.run]
|
|
omit = ["tests/*"]
|
|
|
|
[tool.pytest.ini_options]
|
|
addopts = "--strict-markers --strict-config --durations=5"
|
|
markers = [
|
|
"requires: mark tests as requiring a specific library",
|
|
"compile: mark placeholder test used to compile integration tests without running them",
|
|
]
|
|
asyncio_mode = "auto"
|