github/DB-GPT
1
0
Fork 0
mirror of https://github.com/csunny/DB-GPT.git synced 2025-09-15 22:19:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ChatKnowledge): ChatKnowledge Support Keyword Retrieve (#1624)

Browse Source 
Co-authored-by: Fangyin Cheng <staneyffer@gmail.com>
This commit is contained in:
Aries-ckt
2024-06-13 13:49:17 +08:00
committed by GitHub
parent 162e2c9b1c
commit 58d08780d6
86 changed files with 948 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
dbgpt/serve/rag/service/service.py
View File

@@ -22,7 +22,7 @@ from dbgpt.configs.model_config import (
EMBEDDING_MODEL_CONFIG,
KNOWLEDGE_UPLOAD_ROOT_PATH,
)
from dbgpt.core import Chunk, LLMClient
from dbgpt.core import LLMClient
from dbgpt.core.awel.dag.dag_manager import DAGManager
from dbgpt.model import DefaultLLMClient
from dbgpt.model.cluster import WorkerManagerFactory
@@ -31,12 +31,11 @@ from dbgpt.rag.chunk_manager import ChunkParameters
from dbgpt.rag.embedding import EmbeddingFactory
from dbgpt.rag.knowledge import ChunkStrategy, KnowledgeFactory, KnowledgeType
from dbgpt.serve.core import BaseService
from dbgpt.serve.rag.connector import VectorStoreConnector
from dbgpt.storage.metadata import BaseDao
from dbgpt.storage.metadata._base_dao import QUERY_SPEC
from dbgpt.storage.vector_store.base import VectorStoreConfig
from dbgpt.storage.vector_store.connector import VectorStoreConnector
from dbgpt.util.dbgpts.loader import DBGPTsLoader
from dbgpt.util.executor_utils import ExecutorFactory
from dbgpt.util.pagination_utils import PaginationResult
from dbgpt.util.tracer import root_tracer, trace
@@ -481,7 +480,6 @@ class Service(BaseService[KnowledgeSpaceEntity, SpaceServeRequest, SpaceServeRes
)
)
logger.info(f"begin save document chunks, doc:{doc.doc_name}")
# return chunk_docs
@trace("async_doc_embedding")
async def async_doc_embedding(
@@ -495,7 +493,7 @@ class Service(BaseService[KnowledgeSpaceEntity, SpaceServeRequest, SpaceServeRes
- doc: doc
"""
logger.info(f"async doc embedding sync, doc:{doc.doc_name}")
logger.info(f"async doc persist sync, doc:{doc.doc_name}")
try:
with root_tracer.start_span(
"app.knowledge.assembler.persist",
@@ -503,17 +501,17 @@ class Service(BaseService[KnowledgeSpaceEntity, SpaceServeRequest, SpaceServeRes
):
assembler = await EmbeddingAssembler.aload_from_knowledge(
knowledge=knowledge,
index_store=vector_store_connector.index_client,
chunk_parameters=chunk_parameters,
vector_store_connector=vector_store_connector,
)
chunk_docs = assembler.get_chunks()
doc.chunk_size = len(chunk_docs)
vector_ids = await assembler.apersist()
doc.status = SyncStatus.FINISHED.name
doc.result = "document embedding success"
doc.result = "document persist into index store success"
if vector_ids is not None:
doc.vector_ids = ",".join(vector_ids)
logger.info(f"async document embedding, success:{doc.doc_name}")
logger.info(f"async document persist index store success:{doc.doc_name}")
# save chunk details
chunk_entities = [
DocumentChunkEntity(