mirror of
https://github.com/hwchase17/langchain.git
synced 2026-07-11 18:18:51 +00:00
a2e53fda73
Switches type checking for `langchain-text-splitters` from `mypy` to [`ty`](https://docs.astral.sh/ty/), which is much faster. The `ollama` package already [switched to `ty`](https://github.com/langchain-ai/langchain/pull/36571). ## What changed The core of this PR is the config swap (`[tool.mypy]` → `[tool.ty.rules]`/`[tool.ty.analysis]`, `Makefile`, and the `typing` dependency group). Because `ty` runs with `all = "error"`, a few modules also needed source-level adjustments to satisfy the stricter analysis. These are **behavior-preserving refactors** except for one intentional fix, called out below so reviewers know where to look. ### Behavioral change (intentional fix) - `SentenceTransformersTokenTextSplitter` now raises a clear `ValueError` when the underlying model reports no maximum sequence length **and** no `tokens_per_chunk` was provided. Previously this combination reached a `None > None` comparison and surfaced as an opaque `TypeError`. As a consequence, the public `maximum_tokens_per_chunk` attribute is now honestly typed as `int | None` — it can remain `None` when the caller supplies `tokens_per_chunk` explicitly for a model without a limit. ### Behavior-preserving refactors (no user-visible change) - `TokenTextSplitter.from_tiktoken_encoder` is now an explicit override rather than the base method dispatching on `issubclass(cls, TokenTextSplitter)`. The shared length-function logic moved into a private helper. Public signatures and return types are unchanged. - `NLTKTextSplitter` builds its tokenizer once at construction, so `_tokenizer` is now always a `Callable[[str], list[str]]`. The private attributes `_language` and `_use_span_tokenize` are no longer stored — flagging in case any downstream code read those (they are underscore-private). Tokenization output is unchanged. - `HTMLSemanticPreservingSplitter` text extraction was rewritten from a `cast`-based check to `isinstance(element, Tag)` narrowing; output is equivalent for tags, text nodes, and comments. --------- Co-authored-by: Mason Daugherty <github@mdrxy.com>
74 lines
2.5 KiB
Makefile
74 lines
2.5 KiB
Makefile
.PHONY: all format lint type test tests test_watch integration_tests help extended_tests
|
|
|
|
# Default target executed when no arguments are given to make.
|
|
all: help
|
|
|
|
# Define a variable for the test file path.
|
|
TEST_FILE ?= tests/unit_tests/
|
|
PYTEST_EXTRA ?=
|
|
|
|
.EXPORT_ALL_VARIABLES:
|
|
UV_FROZEN = true
|
|
|
|
test tests:
|
|
uv run --group test pytest -n auto $(PYTEST_EXTRA) --disable-socket --allow-unix-socket $(TEST_FILE)
|
|
|
|
integration_test integration_tests:
|
|
uv run --group test --group test_integration pytest tests/integration_tests/
|
|
|
|
test_watch:
|
|
uv run --group test ptw --snapshot-update --now . -- -vv -x tests/unit_tests
|
|
|
|
test_profile:
|
|
uv run --group test pytest -vv tests/unit_tests/ --profile-svg
|
|
|
|
check_imports: $(shell find langchain_text_splitters -name '*.py')
|
|
uv run --group test python ./scripts/check_imports.py $^
|
|
|
|
extended_tests:
|
|
uv run --group test pytest --disable-socket --allow-unix-socket --only-extended $(TEST_FILE)
|
|
|
|
|
|
######################
|
|
# LINTING AND FORMATTING
|
|
######################
|
|
|
|
# Define a variable for Python and notebook files.
|
|
PYTHON_FILES=.
|
|
lint format: PYTHON_FILES=.
|
|
lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=libs/text-splitters --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
|
|
lint_package: PYTHON_FILES=langchain_text_splitters
|
|
lint_tests: PYTHON_FILES=tests/unit_tests
|
|
UV_RUN_LINT = uv run --all-groups
|
|
UV_RUN_TYPE = uv run --all-groups
|
|
lint_package lint_tests: UV_RUN_LINT = uv run --group lint
|
|
lint_package: UV_RUN_TYPE = uv run --group lint --group typing
|
|
lint_tests: UV_RUN_TYPE = uv run --group typing --group test
|
|
|
|
lint lint_diff lint_package lint_tests:
|
|
./scripts/lint_imports.sh
|
|
[ "$(PYTHON_FILES)" = "" ] || $(UV_RUN_LINT) ruff check $(PYTHON_FILES)
|
|
[ "$(PYTHON_FILES)" = "" ] || $(UV_RUN_LINT) ruff format $(PYTHON_FILES) --diff
|
|
[ "$(PYTHON_FILES)" = "" ] || $(UV_RUN_TYPE) ty check $(PYTHON_FILES)
|
|
|
|
type:
|
|
$(UV_RUN_TYPE) ty check $(PYTHON_FILES)
|
|
|
|
format format_diff:
|
|
[ "$(PYTHON_FILES)" = "" ] || $(UV_RUN_LINT) ruff format $(PYTHON_FILES)
|
|
[ "$(PYTHON_FILES)" = "" ] || $(UV_RUN_LINT) ruff check --fix $(PYTHON_FILES)
|
|
|
|
######################
|
|
# HELP
|
|
######################
|
|
|
|
help:
|
|
@echo '----'
|
|
@echo 'format - run code formatters'
|
|
@echo 'lint - run linters'
|
|
@echo 'type - run type checking'
|
|
@echo 'test - run unit tests'
|
|
@echo 'tests - run unit tests'
|
|
@echo 'test TEST_FILE=<test_file> - run all tests in file'
|
|
@echo 'test_watch - run unit tests in watch mode'
|