github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-18 00:07:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

update:pdf split chunk

Browse Source
This commit is contained in:
aries-ckt 2023-05-17 23:03:51 +08:00
parent 3084123cef
commit c0115061c2
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pilot/server/webserver.py
View File

@ -254,6 +254,8 @@ def http_bot(state, mode, sql_mode, db_selector, temperature, max_new_tokens, re
result = prompt_template.format(context="\n".join(context), question=query)
state.messages[-2][1] = result
prompt = state.get_prompt()
print("prompt length:" + len(prompt))
if len(prompt) > 4000:
logger.info("prompt length greater than 4000, rebuild")
context = context[:2000]
@ -264,7 +266,8 @@ def http_bot(state, mode, sql_mode, db_selector, temperature, max_new_tokens, re
result = prompt_template.format(context="\n".join(context), question=query)
state.messages[-2][1] = result
prompt = state.get_prompt()
print(len(prompt))
print("new prompt length:" + len(prompt))
state.messages[-2][1] = query
skip_echo_len = len(prompt.replace("</s>", " ")) + 1

4
pilot/source_embedding/pdf_embedding.py
View File

@ -6,6 +6,7 @@ from langchain.document_loaders import PyPDFLoader
from langchain.schema import Document
from pilot.source_embedding import SourceEmbedding, register
from pilot.source_embedding.chinese_text_splitter import ChineseTextSplitter
class PDFEmbedding(SourceEmbedding):
@ -22,7 +23,8 @@ class PDFEmbedding(SourceEmbedding):
def read(self):
"""Load from pdf path."""
loader = PyPDFLoader(self.file_path)
return loader.load()
textsplitter = ChineseTextSplitter(pdf=True, sentence_size=100)
return loader.load_and_split(textsplitter)
@register
def data_process(self, documents: List[Document]):