github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-11 05:49:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

update:pdf split chunk

Browse Source
This commit is contained in:
aries-ckt
2023-05-17 23:03:51 +08:00
parent 3084123cef
commit c0115061c2
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pilot/source_embedding/pdf_embedding.py
View File

@@ -6,6 +6,7 @@ from langchain.document_loaders import PyPDFLoader
from langchain.schema import Document
from pilot.source_embedding import SourceEmbedding, register
from pilot.source_embedding.chinese_text_splitter import ChineseTextSplitter
class PDFEmbedding(SourceEmbedding):
@@ -22,7 +23,8 @@ class PDFEmbedding(SourceEmbedding):
def read(self):
"""Load from pdf path."""
loader = PyPDFLoader(self.file_path)
return loader.load()
textsplitter = ChineseTextSplitter(pdf=True, sentence_size=100)
return loader.load_and_split(textsplitter)
@register
def data_process(self, documents: List[Document]):