chore: Add pylint for storage (#1298)

dbgpt/app/openapi/api_v1/editor/_chat_history/__init__.py

@@ -0,0 +1,4 @@
+"""Old chat history module.
+
+Just used by editor.
+"""

dbgpt/app/openapi/api_v1/editor/_chat_history/base.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Dict, List, Optional
+
+from dbgpt.core.interface.message import OnceConversation
+
+
+class MemoryStoreType(Enum):
+    # File = "file"
+    # Memory = "memory"
+    DB = "db"
+    # DuckDb = "duckdb"
+
+
+class BaseChatHistoryMemory(ABC):
+    store_type: MemoryStoreType
+
+    def __init__(self):
+        self.conversations: List[OnceConversation] = []
+
+    @abstractmethod
+    def messages(self) -> List[OnceConversation]:  # type: ignore
+        """Retrieve the messages from the local file"""
+
+    # @abstractmethod
+    # def create(self, user_name: str) -> None:
+    #     """Append the message to the record in the local file"""
+
+    @abstractmethod
+    def append(self, message: OnceConversation) -> None:
+        """Append the message to the record in the local file"""
+
+    @abstractmethod
+    def update(self, messages: List[OnceConversation]) -> None:
+        pass
+
+    @abstractmethod
+    def delete(self) -> bool:
+        pass
+
+    @abstractmethod
+    def get_messages(self) -> List[Dict]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def conv_list(
+        user_name: Optional[str] = None, sys_code: Optional[str] = None
+    ) -> List[Dict]:
+        """get user's conversation list"""

dbgpt/app/openapi/api_v1/editor/_chat_history/chat_hisotry_factory.py

@@ -0,0 +1,74 @@
+"""Module for chat history factory.
+
+It will remove in the future, just support db store type now.
+"""
+import logging
+from typing import Type
+
+from dbgpt._private.config import Config
+
+from .base import BaseChatHistoryMemory, MemoryStoreType
+
+# Import first for auto create table
+from .meta_db_history import DbHistoryMemory
+
+# TODO remove global variable
+CFG = Config()
+logger = logging.getLogger(__name__)
+
+
+class ChatHistory:
+    def __init__(self):
+        self.memory_type = MemoryStoreType.DB.value
+        self.mem_store_class_map = {}
+
+        # Just support db store type after v0.4.6
+        self.mem_store_class_map[DbHistoryMemory.store_type] = DbHistoryMemory
+
+    def get_store_instance(self, chat_session_id: str) -> BaseChatHistoryMemory:
+        """New store instance for store chat histories
+
+        Args:
+            chat_session_id (str): conversation session id
+
+        Returns:
+            BaseChatHistoryMemory: Store instance
+        """
+        self._check_store_type(CFG.CHAT_HISTORY_STORE_TYPE)
+        return self.mem_store_class_map.get(CFG.CHAT_HISTORY_STORE_TYPE)(
+            chat_session_id
+        )
+
+    def get_store_cls(self) -> Type[BaseChatHistoryMemory]:
+        self._check_store_type(CFG.CHAT_HISTORY_STORE_TYPE)
+        return self.mem_store_class_map.get(CFG.CHAT_HISTORY_STORE_TYPE)
+
+    def _check_store_type(self, store_type: str):
+        """Check store type
+
+        Raises:
+            ValueError: Invalid store type
+        """
+        if store_type == "memory":
+            logger.error(
+                "Not support memory store type, just support db store type now"
+            )
+            raise ValueError(f"Invalid store type: {store_type}")
+
+        if store_type == "file":
+            logger.error("Not support file store type, just support db store type now")
+            raise ValueError(f"Invalid store type: {store_type}")
+        if store_type == "duckdb":
+            link1 = (
+                "https://docs.dbgpt.site/docs/latest/faq/install#q6-how-to-migrate-meta"
+                "-table-chat_history-and-connect_config-from-duckdb-to-sqlite"
+            )
+            link2 = (
+                "https://docs.dbgpt.site/docs/latest/faq/install/#q7-how-to-migrate-"
+                "meta-table-chat_history-and-connect_config-from-duckdb-to-mysql"
+            )
+            logger.error(
+                "Not support duckdb store type after v0.4.6, just support db store "
+                f"type now, you can migrate your message according to {link1} or {link2}"
+            )
+            raise ValueError(f"Invalid store type: {store_type}")

dbgpt/app/openapi/api_v1/editor/_chat_history/meta_db_history.py

@@ -0,0 +1,105 @@
+import json
+import logging
+from typing import Dict, List, Optional
+
+from dbgpt._private.config import Config
+from dbgpt.core.interface.message import OnceConversation, _conversation_to_dict
+from dbgpt.storage.chat_history.chat_history_db import ChatHistoryDao, ChatHistoryEntity
+
+from .base import BaseChatHistoryMemory, MemoryStoreType
+
+CFG = Config()
+logger = logging.getLogger(__name__)
+
+
+class DbHistoryMemory(BaseChatHistoryMemory):
+    """Db history memory storage.
+
+    It is deprecated.
+    """
+
+    store_type: str = MemoryStoreType.DB.value  # type: ignore
+
+    def __init__(self, chat_session_id: str):
+        self.chat_seesion_id = chat_session_id
+        self.chat_history_dao = ChatHistoryDao()
+
+    def messages(self) -> List[OnceConversation]:
+        chat_history: Optional[ChatHistoryEntity] = self.chat_history_dao.get_by_uid(
+            self.chat_seesion_id
+        )
+        if chat_history:
+            context = chat_history.messages
+            if context:
+                conversations: List[OnceConversation] = json.loads(
+                    context  # type: ignore
+                )
+                return conversations
+        return []
+
+    # def create(self, chat_mode, summary: str, user_name: str) -> None:
+    #     try:
+    #         chat_history: ChatHistoryEntity = ChatHistoryEntity()
+    #         chat_history.chat_mode = chat_mode
+    #         chat_history.summary = summary
+    #         chat_history.user_name = user_name
+    #
+    #         self.chat_history_dao.raw_update(chat_history)
+    #     except Exception as e:
+    #         logger.error("init create conversation log error！" + str(e))
+    #
+    def append(self, once_message: OnceConversation) -> None:
+        logger.debug(f"db history append: {once_message}")
+        chat_history: Optional[ChatHistoryEntity] = self.chat_history_dao.get_by_uid(
+            self.chat_seesion_id
+        )
+        conversations: List[Dict] = []
+        latest_user_message = once_message.get_latest_user_message()
+        summary = latest_user_message.content if latest_user_message else ""
+        if chat_history:
+            context = chat_history.messages
+            if context:
+                conversations = json.loads(context)  # type: ignore
+            else:
+                chat_history.summary = summary  # type: ignore
+        else:
+            chat_history = ChatHistoryEntity()
+            chat_history.conv_uid = self.chat_seesion_id  # type: ignore
+            chat_history.chat_mode = once_message.chat_mode  # type: ignore
+            chat_history.user_name = once_message.user_name  # type: ignore
+            chat_history.sys_code = once_message.sys_code  # type: ignore
+            chat_history.summary = summary  # type: ignore
+
+        conversations.append(_conversation_to_dict(once_message))
+        chat_history.messages = json.dumps(  # type: ignore
+            conversations, ensure_ascii=False
+        )
+
+        self.chat_history_dao.raw_update(chat_history)
+
+    def update(self, messages: List[OnceConversation]) -> None:
+        self.chat_history_dao.update_message_by_uid(
+            json.dumps(messages, ensure_ascii=False), self.chat_seesion_id
+        )
+
+    def delete(self) -> bool:
+        self.chat_history_dao.raw_delete(self.chat_seesion_id)
+        return True
+
+    def get_messages(self) -> List[Dict]:
+        chat_history = self.chat_history_dao.get_by_uid(self.chat_seesion_id)
+        if chat_history:
+            context = chat_history.messages
+            return json.loads(context)  # type: ignore
+        return []
+
+    @staticmethod
+    def conv_list(
+        user_name: Optional[str] = None, sys_code: Optional[str] = None
+    ) -> List[Dict]:
+        chat_history_dao = ChatHistoryDao()
+        history_list = chat_history_dao.list_last_20(user_name, sys_code)
+        result = []
+        for history in history_list:
+            result.append(history.__dict__)
+        return result

dbgpt/app/openapi/api_v1/editor/api_editor_v1.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import time
-from typing import List
+from typing import Dict, List
 
 from fastapi import APIRouter, Body, Depends
 
@@ -23,9 +23,9 @@ from dbgpt.app.openapi.editor_view_model import (
 )
 from dbgpt.app.scene import ChatFactory
 from dbgpt.app.scene.chat_dashboard.data_loader import DashboardDataLoader
-from dbgpt.core.interface.message import OnceConversation
 from dbgpt.serve.conversation.serve import Serve as ConversationServe
-from dbgpt.storage.chat_history.chat_hisotry_factory import ChatHistory
+
+from ._chat_history.chat_hisotry_factory import ChatHistory
 
 router = APIRouter()
 CFG = Config()
@@ -201,7 +201,7 @@ async def chart_editor_submit(chart_edit_context: ChatChartEditContext = Body())
 
     chat_history_fac = ChatHistory()
     history_mem = chat_history_fac.get_store_instance(chart_edit_context.con_uid)
-    history_messages: List[OnceConversation] = history_mem.get_messages()
+    history_messages: List[Dict] = history_mem.get_messages()
     if history_messages:
         dashboard_data_loader: DashboardDataLoader = DashboardDataLoader()
         db_conn = CFG.LOCAL_DB_MANAGE.get_connect(chart_edit_context.db_name)