github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-10-12 20:28:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

pylint: multi model for gp4all (#138)

Browse Source
This commit is contained in:
csunny
2023-06-14 10:17:53 +08:00
parent 3927c26dea
commit ff6d3a7035
25 changed files with 154 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pilot/source_embedding/markdown_embedding.py
View File

@@ -30,7 +30,11 @@ class MarkdownEmbedding(SourceEmbedding):
def read(self):
"""Load from markdown path."""
loader = EncodeTextLoader(self.file_path)
textsplitter = SpacyTextSplitter(pipeline='zh_core_web_sm', chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=200)
textsplitter = SpacyTextSplitter(
pipeline="zh_core_web_sm",
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
chunk_overlap=200,
)
return loader.load_and_split(textsplitter)
@register