github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-09 04:08:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature:chn splitter

Browse Source
This commit is contained in:
aries-ckt 2023-05-18 19:48:28 +08:00
parent b92ecf2199
commit e50c04ead2
1 changed files with 16 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
pilot/source_embedding/chinese_text_splitter.py → pilot/source_embedding/chn_document_splitter.py
View File

@ -1,30 +1,29 @@
from langchain.text_splitter import CharacterTextSplitter
import re import re
from typing import List from typing import List
# from configs.model_config import SENTENCE_SIZE from langchain.text_splitter import CharacterTextSplitter
class ChineseTextSplitter(CharacterTextSplitter): class CHNDocumentSplitter(CharacterTextSplitter):
def __init__(self, pdf: bool = False, sentence_size: int = None, **kwargs): def __init__(self, pdf: bool = False, sentence_size: int = None, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
self.pdf = pdf self.pdf = pdf
self.sentence_size = sentence_size self.sentence_size = sentence_size
def split_text1(self, text: str) -> List[str]: # def split_text_version2(self, text: str) -> List[str]:
if self.pdf: # if self.pdf:
text = re.sub(r"\n{3,}", "\n", text) # text = re.sub(r"\n{3,}", "\n", text)
text = re.sub('\s', ' ', text) # text = re.sub('\s', ' ', text)
text = text.replace("\n\n", "") # text = text.replace("\n\n", "")
sent_sep_pattern = re.compile('([﹒﹔﹖﹗．。！？]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))') # del ：； # sent_sep_pattern = re.compile('([﹒﹔﹖﹗．。！？]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))') # del ：；
sent_list = [] # sent_list = []
for ele in sent_sep_pattern.split(text): # for ele in sent_sep_pattern.split(text):
if sent_sep_pattern.match(ele) and sent_list: # if sent_sep_pattern.match(ele) and sent_list:
sent_list[-1] += ele # sent_list[-1] += ele
elif ele: # elif ele:
sent_list.append(ele) # sent_list.append(ele)
return sent_list # return sent_list
def split_text(self, text: str) -> List[str]: ##此处需要进一步优化逻辑 def split_text(self, text: str) -> List[str]:
if self.pdf: if self.pdf:
text = re.sub(r"\n{3,}", r"\n", text) text = re.sub(r"\n{3,}", r"\n", text)
text = re.sub('\s', " ", text) text = re.sub('\s', " ", text)