github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-10-08 15:09:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor:refactor knowledge api

Browse Source 
1.delete CFG in embedding_engine api
2.add a text_splitter param in embedding_engine api
This commit is contained in:
aries_ckt
2023-07-11 16:33:48 +08:00
parent 6ff7ef9da4
commit e6aa46fc87
24 changed files with 161 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
pilot/embedding_engine/ppt_embedding.py
View File

@@ -6,48 +6,36 @@ from langchain.document_loaders import UnstructuredPowerPointLoader
from langchain.schema import Document
from langchain.text_splitter import SpacyTextSplitter, RecursiveCharacterTextSplitter
from pilot.configs.config import Config
from pilot.embedding_engine import SourceEmbedding, register
CFG = Config()
class PPTEmbedding(SourceEmbedding):
"""ppt embedding for read ppt document."""
def __init__(self, file_path, vector_store_config):
"""Initialize with pdf path."""
super().__init__(file_path, vector_store_config)
def __init__(self, file_path, vector_store_config, text_splitter=None):
"""Initialize ppt word path."""
super().__init__(file_path, vector_store_config, text_splitter=None)
self.file_path = file_path
self.vector_store_config = vector_store_config
self.text_splitter = text_splitter or None
@register
def read(self):
"""Load from ppt path."""
loader = UnstructuredPowerPointLoader(self.file_path)
# textsplitter = SpacyTextSplitter(
# pipeline="zh_core_web_sm",
# chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
# chunk_overlap=200,
# )
if CFG.LANGUAGE == "en":
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
chunk_overlap=20,
length_function=len,
)
else:
if self.text_splitter is None:
try:
text_splitter = SpacyTextSplitter(
self.text_splitter = SpacyTextSplitter(
pipeline="zh_core_web_sm",
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE,
chunk_size=100,
chunk_overlap=100,
)
except Exception:
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=CFG.KNOWLEDGE_CHUNK_SIZE, chunk_overlap=50
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=100, chunk_overlap=50
)
return loader.load_and_split(text_splitter)
return loader.load_and_split(self.text_splitter)
@register
def data_process(self, documents: List[Document]):