feat: (0.6)New UI (#1855)

Co-authored-by: 夏姜 <wenfengjiang.jwf@digital-engine.com>
Co-authored-by: aries_ckt <916701291@qq.com>
Co-authored-by: wb-lh513319 <wb-lh513319@alibaba-inc.com>
Co-authored-by: csunny <cfqsunny@163.com>

dbgpt/app/knowledge/request/request.py

@@ -1,13 +1,16 @@
 from enum import Enum
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from dbgpt._private.pydantic import BaseModel, ConfigDict
+from dbgpt.rag import ChunkParameters
 
 
 class KnowledgeQueryRequest(BaseModel):
     """query: knowledge query"""
 
     query: str
+    """space: space name"""
+    space: str
     """top_k: return topK documents"""
     top_k: int
 
@@ -17,15 +20,17 @@ class KnowledgeSpaceRequest(BaseModel):
 
     """vector_type: vector type"""
     id: Optional[int] = None
-    name: str = None
+    name: Optional[str] = None
     """vector_type: vector type"""
-    vector_type: str = None
+    vector_type: Optional[str] = None
     """vector_type: vector type"""
     domain_type: str = "Normal"
     """desc: description"""
     desc: str = None
     """owner: owner"""
-    owner: str = None
+    owner: Optional[str] = None
+
+    space_id: Optional[Union[int, str]] = None
 
 
 class BusinessFieldType(Enum):
@@ -37,25 +42,40 @@ class BusinessFieldType(Enum):
 class KnowledgeDocumentRequest(BaseModel):
     """doc_name: doc path"""
 
-    doc_name: str = None
+    doc_name: Optional[str] = None
+    """doc_id: doc id"""
+    doc_id: Optional[int] = None
     """doc_type: doc type"""
-    doc_type: str = None
+    doc_type: Optional[str] = None
+    """doc_token: doc token"""
+    doc_token: Optional[str] = None
     """content: content"""
-    content: str = None
+    content: Optional[str] = None
     """content: content"""
-    source: str = None
+    source: Optional[str] = None
+
+    labels: Optional[str] = None
+
+    questions: Optional[List[str]] = None
+
+
+class DocumentRecallTestRequest(BaseModel):
+    question: Optional[str] = None
+    recall_top_k: Optional[int] = 1
+    recall_retrievers: Optional[List[str]] = None
+    recall_score_threshold: Optional[float] = -100
 
 
 class DocumentQueryRequest(BaseModel):
     """doc_name: doc path"""
 
-    doc_name: str = None
+    doc_name: Optional[str] = None
     """doc_ids: doc ids"""
     doc_ids: Optional[List] = None
     """doc_type: doc type"""
-    doc_type: str = None
+    doc_type: Optional[str] = None
     """status: status"""
-    status: str = None
+    status: Optional[str] = None
     """page: page"""
     page: int = 1
     """page_size: page size"""
@@ -91,22 +111,74 @@ class DocumentSyncRequest(BaseModel):
     chunk_overlap: Optional[int] = None
 
 
+class KnowledgeSyncRequest(BaseModel):
+    """Sync request"""
+
+    """doc_ids: doc ids"""
+    doc_id: int
+
+    """model_name: model name"""
+    model_name: Optional[str] = None
+
+    """chunk_parameters: chunk parameters 
+    """
+    chunk_parameters: ChunkParameters
+
+    def to_dict(self):
+        return {k: self._serialize(v) for k, v in self.__dict__.items()}
+
+    def _serialize(self, value):
+        if isinstance(value, BaseModel):
+            return value.to_dict()
+        elif isinstance(value, list):
+            return [self._serialize(item) for item in value]
+        elif isinstance(value, dict):
+            return {k: self._serialize(v) for k, v in value.items()}
+        else:
+            return value
+
+
 class ChunkQueryRequest(BaseModel):
     """id: id"""
 
-    id: int = None
+    id: Optional[int] = None
     """document_id: doc id"""
-    document_id: int = None
+    document_id: Optional[int] = None
     """doc_name: doc path"""
-    doc_name: str = None
+    doc_name: Optional[str] = None
     """doc_type: doc type"""
-    doc_type: str = None
+    doc_type: Optional[str] = None
+    """chunk content: content"""
+    content: Optional[str] = None
     """page: page"""
     page: int = 1
     """page_size: page size"""
     page_size: int = 20
 
 
+class ChunkEditRequest(BaseModel):
+    """id: id"""
+
+    """chunk_id: chunk_id"""
+    chunk_id: Optional[int] = None
+    """chunk content: content"""
+    content: Optional[str] = None
+    """label: label"""
+    label: Optional[str] = None
+    """questions: questions"""
+    questions: Optional[List[str]] = None
+
+
+class KnowledgeQueryResponse:
+    """source: knowledge reference source"""
+
+    source: Optional[str]
+    """score: knowledge vector query similarity score"""
+    score: float = 0.0
+    """text: raw text info"""
+    text: Optional[str]
+
+
 class SpaceArgumentRequest(BaseModel):
     """argument: argument"""
 
@@ -124,6 +196,12 @@ class DocumentSummaryRequest(BaseModel):
     conv_uid: str
 
 
+class DocumentDynamicText(BaseModel):
+    space_id: int
+    doc_id: int
+    text_snippets: List[str]
+
+
 class EntityExtractRequest(BaseModel):
     """argument: argument"""
 
@@ -131,3 +209,21 @@ class EntityExtractRequest(BaseModel):
 
     text: str
     model_name: str
+
+
+class SpaceEvaluationRequest(BaseModel):
+    """RAG Evaluation Reques.t"""
+
+    datasets: List[dict]
+    """space: space name"""
+    space_id: Optional[int] = None
+    """top_k: return topK documents"""
+    top_k: int
+    """type: evaluation type"""
+    type: Optional[str] = "recall"
+    """model_name: evaluation model_name"""
+    model_name: Optional[str] = None
+    """app_id: app_id"""
+    app_id: Optional[str] = None
+    """evaluate prompt id: prompt_id"""
+    prompt_code: Optional[str] = None

dbgpt/app/knowledge/request/response.py

@@ -1,13 +1,20 @@
+import json
 from typing import List, Optional
 
 from dbgpt._private.pydantic import BaseModel, Field
-from dbgpt.serve.rag.api.schemas import DocumentChunkVO, DocumentVO
+from dbgpt.app.knowledge.document_db import KnowledgeDocumentEntity
+from dbgpt.serve.rag.api.schemas import (
+    ChunkServeResponse,
+    DocumentChunkVO,
+    DocumentServeResponse,
+    DocumentVO,
+)
 
 
 class ChunkQueryResponse(BaseModel):
     """data: data"""
 
-    data: List[DocumentChunkVO] = Field(..., description="document chunk list")
+    data: List[ChunkServeResponse] = Field(None, description="document chunk list")
     """summary: document summary"""
     summary: Optional[str] = Field(None, description="document summary")
     """total: total size"""
@@ -16,35 +23,112 @@ class ChunkQueryResponse(BaseModel):
     page: Optional[int] = Field(None, description="current page")
 
 
-class DocumentQueryResponse(BaseModel):
-    """data: data"""
+class DocumentResponse(BaseModel):
+    """DocumentResponse: DocumentResponse"""
 
-    data: List[DocumentVO] = Field(..., description="document list")
-    """total: total size"""
-    total: Optional[int] = Field(None, description="total size")
-    """page: current page"""
-    page: Optional[int] = Field(None, description="current page")
+    id: Optional[int] = Field(None, description="The doc id")
+    doc_name: Optional[str] = Field(None, description="doc type")
+    """vector_type: vector type"""
+    doc_type: Optional[str] = Field(None, description="The doc content")
+    """desc: description"""
+    content: Optional[str] = Field(None, description="content")
+    """vector ids"""
+    vector_ids: Optional[str] = Field(None, description="vector ids")
+    """space: space name"""
+    space: Optional[str] = Field(None, description="space name")
+    """space_id: space id"""
+    space_id: Optional[int] = Field(None, description="space id")
+    """status: status"""
+    status: Optional[str] = Field(None, description="status")
+    """last_sync: last sync time"""
+    last_sync: Optional[str] = Field(None, description="last sync time")
+    """result: result"""
+    result: Optional[str] = Field(None, description="result")
+    """summary: summary"""
+    summary: Optional[str] = Field(None, description="summary")
+    """gmt_created: created time"""
+    gmt_created: Optional[str] = Field(None, description="created time")
+    """gmt_modified: modified time"""
+    gmt_modified: Optional[str] = Field(None, description="modified time")
+    """chunk_size: chunk size"""
+    chunk_size: Optional[int] = Field(None, description="chunk size")
+    """questions: questions"""
+    questions: Optional[List[str]] = Field(None, description="questions")
+
+    @classmethod
+    def to_response(cls, entity: KnowledgeDocumentEntity):
+        """Convert the entity to a response
+
+        Args:
+            entity (T): The entity
+
+        Returns:
+            REQ: The request
+        """
+        return DocumentResponse(
+            id=entity.id,
+            doc_name=entity.doc_name,
+            doc_type=entity.doc_type,
+            space=entity.space,
+            chunk_size=entity.chunk_size,
+            status=entity.status,
+            last_sync=str(entity.last_sync),
+            content=entity.content,
+            result=entity.result,
+            vector_ids=entity.vector_ids,
+            summary=entity.summary,
+            questions=json.loads(entity.questions) if entity.questions else None,
+            gmt_created=str(entity.gmt_created),
+            gmt_modified=str(entity.gmt_modified),
+        )
+
+    @classmethod
+    def serve_to_response(cls, response: DocumentServeResponse):
+        """Convert the entity to a response
+
+        Args:
+            entity (T): The entity
+
+        Returns:
+            REQ: The request
+        """
+        return DocumentResponse(
+            id=response.id,
+            doc_name=response.doc_name,
+            doc_type=response.doc_type,
+            space=response.space,
+            chunk_size=response.chunk_size,
+            status=response.status,
+            last_sync=str(response.last_sync),
+            content=response.content,
+            result=response.result,
+            vector_ids=response.vector_ids,
+            summary=response.summary,
+            questions=json.loads(response.questions) if response.questions else None,
+            gmt_created=str(response.gmt_created),
+            gmt_modified=str(response.gmt_modified),
+        )
 
 
 class SpaceQueryResponse(BaseModel):
     """data: data"""
 
-    id: int = None
-    name: str = None
+    id: Optional[int] = None
+    name: Optional[str] = None
     """vector_type: vector type"""
-    vector_type: str = None
-    """field_type: field type"""
-    domain_type: str = None
+    vector_type: Optional[str] = None
+    """domain_type"""
+    domain_type: Optional[str] = None
     """desc: description"""
-    desc: str = None
+    desc: Optional[str] = None
     """context: context"""
-    context: str = None
+    context: Optional[str] = None
     """owner: owner"""
-    owner: str = None
-    gmt_created: str = None
-    gmt_modified: str = None
+    owner: Optional[str] = None
+    gmt_created: Optional[str] = None
+    gmt_modified: Optional[str] = None
     """doc_count: doc_count"""
-    docs: int = None
+    docs: Optional[int] = None
 
 
 class KnowledgeQueryResponse(BaseModel):
@@ -55,3 +139,13 @@ class KnowledgeQueryResponse(BaseModel):
     score: float = 0.0
     """text: raw text info"""
     text: str
+
+
+class DocumentQueryResponse(BaseModel):
+    """data: data"""
+
+    data: List[DocumentResponse] = Field(None, description="document list")
+    """total: total size"""
+    total: Optional[int] = Field(None, description="total size")
+    """page: current page"""
+    page: Optional[int] = Field(None, description="current page")