github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-09-21 02:19:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix[experimental]: Fix text splitter with gradient (#26629)

Browse Source 
Fixes #26221

---------

Co-authored-by: Erick Friis <erick@langchain.dev>
This commit is contained in:
Tibor Reiss
2024-09-21 01:35:50 +02:00
committed by GitHub
parent 4ac9a6f52c
commit a8b24135a2
2 changed files with 60 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
libs/experimental/langchain_experimental/text_splitter.py
View File

@@ -217,6 +217,12 @@ class SemanticChunker(BaseDocumentTransformer):
# np.percentile to fail.
if len(single_sentences_list) == 1:
return single_sentences_list
# similarly, the following np.gradient would fail
if (
self.breakpoint_threshold_type == "gradient"
and len(single_sentences_list) == 2
):
return single_sentences_list
distances, sentences = self._calculate_sentence_distances(single_sentences_list)
if self.number_of_chunks is not None:
breakpoint_distance_threshold = self._threshold_from_clusters(distances)