github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-10-13 04:38:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

pylint: multi model for gp4all (#138)

Browse Source
This commit is contained in:
csunny
2023-06-14 10:17:53 +08:00
parent 3927c26dea
commit ff6d3a7035
25 changed files with 154 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pilot/source_embedding/pdf_embedding.py
View File

@@ -29,7 +29,9 @@ class PDFEmbedding(SourceEmbedding):
# pdf=True, sentence_size=CFG.KNOWLEDGE_CHUNK_SIZE
# )
textsplitter = SpacyTextSplitter(
pipeline="zh_core_web_sm", chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=200
pipeline="zh_core_web_sm",
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
chunk_overlap=200,
)
return loader.load_and_split(textsplitter)