github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-16 22:51:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

doc:db-gpt doc

Browse Source
This commit is contained in:
aries-ckt
2023-06-14 15:31:11 +08:00
parent 2fdc7fb139
commit 8980de9be4
27 changed files with 843 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pilot/source_embedding/markdown_embedding.py
View File

@@ -33,7 +33,7 @@ class MarkdownEmbedding(SourceEmbedding):
textsplitter = SpacyTextSplitter(
pipeline="zh_core_web_sm",
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
chunk_overlap=200,
chunk_overlap=100,
)
return loader.load_and_split(textsplitter)

2
pilot/source_embedding/pdf_embedding.py
View File

@@ -31,7 +31,7 @@ class PDFEmbedding(SourceEmbedding):
textsplitter = SpacyTextSplitter(
pipeline="zh_core_web_sm",
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
chunk_overlap=200,
chunk_overlap=100,
)
return loader.load_and_split(textsplitter)