github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-16 08:06:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Apply patch [skip ci]

Browse Source
This commit is contained in:
open-swe[bot] 2025-07-31 00:21:32 +00:00
parent 67380e040c
commit 32efc3eb5c
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libs/text-splitters/langchain_text_splitters/sentence_transformers.py
View File

@ -6,7 +6,7 @@ from langchain_text_splitters.base import TextSplitter, Tokenizer, split_text_on
class SentenceTransformersTokenTextSplitter(TextSplitter):
"""Splitting text to tokens using sentence model tokenizer."""
"""Splitting text to tokens using transformer model tokenizer."""
def __init__(
self,
@ -111,3 +111,4 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
)
return cast("list[int]", token_ids_with_start_and_end_token_ids)