github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-08-03 02:06:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: improved start_index logic in TextSplitter when called with tokens

Browse Source
This commit is contained in:
Venrite 2025-07-26 18:18:34 -07:00
parent 5ecbb5f277
commit e61c802472
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libs/text-splitters/langchain_text_splitters/base.py
View File

@ -79,11 +79,20 @@ class TextSplitter(BaseDocumentTransformer, ABC):
for i, text in enumerate(texts): for i, text in enumerate(texts):
index = 0 index = 0
previous_chunk_len = 0 previous_chunk_len = 0
not_found = -1
retries = 1
for chunk in self.split_text(text): for chunk in self.split_text(text):
metadata = copy.deepcopy(_metadatas[i]) metadata = copy.deepcopy(_metadatas[i])
if self._add_start_index: if self._add_start_index:
search_boundary = len(chunk) + 1
offset = index + previous_chunk_len - self._chunk_overlap offset = index + previous_chunk_len - self._chunk_overlap
index = text.find(chunk, max(0, offset)) prev_index = index
index = text.find(chunk, max(0, offset), max(0,offset) + search_boundary)
if index == not_found:
for token_offset in range(0,retries):
index = text.find(chunk, prev_index + token_offset)
if index is prev_index:
continue
metadata["start_index"] = index metadata["start_index"] = index
previous_chunk_len = len(chunk) previous_chunk_len = len(chunk)
new_doc = Document(page_content=chunk, metadata=metadata) new_doc = Document(page_content=chunk, metadata=metadata)