github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-03 18:17:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): Upgrade pydantic to 2.x (#1428)

Browse Source
This commit is contained in:
Fangyin Cheng
2024-04-20 09:41:16 +08:00
committed by GitHub
parent baa1e3f9f6
commit 57be1ece18
103 changed files with 1146 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dbgpt/rag/text_splitter/token_splitter.py
View File

@@ -58,7 +58,6 @@ class TokenTextSplitter(BaseModel):
tokenizer = tokenizer or globals_helper.tokenizer
all_seps = [separator] + (backup_separators or [])
self._split_fns = [split_by_sep(sep) for sep in all_seps] + [split_by_char()]
super().__init__(
chunk_size=chunk_size,
@@ -68,6 +67,7 @@ class TokenTextSplitter(BaseModel):
# callback_manager=callback_manager,
tokenizer=tokenizer,
)
self._split_fns = [split_by_sep(sep) for sep in all_seps] + [split_by_char()]
@classmethod
def class_name(cls) -> str: