github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-02 01:27:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ChatKnowledge):chunk add enable_merge parameter (#1014)

Browse Source 
Co-authored-by: Aralhi <xiaoping0501@gmail.com>
This commit is contained in:
Aries-ckt
2024-01-04 10:04:41 +08:00
committed by GitHub
parent fd30588e55
commit ca83443c48
26 changed files with 98 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
dbgpt/rag/text_splitter/text_splitter.py
View File

@@ -682,6 +682,7 @@ class SeparatorTextSplitter(CharacterTextSplitter):
def __init__(self, separator: str = "\n", filters: list = [], **kwargs: Any):
"""Create a new TextSplitter."""
self._merge = kwargs.pop("enable_merge") or False
super().__init__(**kwargs)
self._separator = separator
self._filter = filters
@@ -696,7 +697,9 @@ class SeparatorTextSplitter(CharacterTextSplitter):
splits = text.split(separator)
else:
splits = list(text)
return self._merge_splits(splits, separator, chunk_overlap=0, **kwargs)
if self._merge:
return self._merge_splits(splits, separator, chunk_overlap=0, **kwargs)
return list(filter(None, text.split(separator)))
class PageTextSplitter(TextSplitter):