github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-08-19 08:47:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

update:pdf split chunk

Browse Source
This commit is contained in:
aries-ckt 2023-05-17 23:03:51 +08:00
parent 3084123cef
commit c0115061c2
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pilot/server/webserver.py
View File

@ -254,6 +254,8 @@ def http_bot(state, mode, sql_mode, db_selector, temperature, max_new_tokens, re
result = prompt_template.format(context="\n".join(context), question=query) result = prompt_template.format(context="\n".join(context), question=query)
state.messages[-2][1] = result state.messages[-2][1] = result
prompt = state.get_prompt() prompt = state.get_prompt()
print("prompt length:" + len(prompt))
if len(prompt) > 4000: if len(prompt) > 4000:
logger.info("prompt length greater than 4000, rebuild") logger.info("prompt length greater than 4000, rebuild")
context = context[:2000] context = context[:2000]
@ -264,7 +266,8 @@ def http_bot(state, mode, sql_mode, db_selector, temperature, max_new_tokens, re
result = prompt_template.format(context="\n".join(context), question=query) result = prompt_template.format(context="\n".join(context), question=query)
state.messages[-2][1] = result state.messages[-2][1] = result
prompt = state.get_prompt() prompt = state.get_prompt()
print(len(prompt)) print("new prompt length:" + len(prompt))
state.messages[-2][1] = query state.messages[-2][1] = query
skip_echo_len = len(prompt.replace("</s>", " ")) + 1 skip_echo_len = len(prompt.replace("</s>", " ")) + 1

4
pilot/source_embedding/pdf_embedding.py
View File

@ -6,6 +6,7 @@ from langchain.document_loaders import PyPDFLoader
from langchain.schema import Document from langchain.schema import Document
from pilot.source_embedding import SourceEmbedding, register from pilot.source_embedding import SourceEmbedding, register
from pilot.source_embedding.chinese_text_splitter import ChineseTextSplitter
class PDFEmbedding(SourceEmbedding): class PDFEmbedding(SourceEmbedding):
@ -22,7 +23,8 @@ class PDFEmbedding(SourceEmbedding):
def read(self): def read(self):
"""Load from pdf path.""" """Load from pdf path."""
loader = PyPDFLoader(self.file_path) loader = PyPDFLoader(self.file_path)
return loader.load() textsplitter = ChineseTextSplitter(pdf=True, sentence_size=100)
return loader.load_and_split(textsplitter)
@register @register
def data_process(self, documents: List[Document]): def data_process(self, documents: List[Document]):