diff --git a/.env.template b/.env.template index 3b7c8e887..16f4e3a6e 100644 --- a/.env.template +++ b/.env.template @@ -84,6 +84,7 @@ LOCAL_DB_TYPE=sqlite # LOCAL_DB_PASSWORD=aa12345678 # LOCAL_DB_HOST=127.0.0.1 # LOCAL_DB_PORT=3306 +# LOCAL_DB_NAME=dbgpt ### This option determines the storage location of conversation records. The default is not configured to the old version of duckdb. It can be optionally db or file (if the value is db, the database configured by LOCAL_DB will be used) #CHAT_HISTORY_STORE_TYPE=db diff --git a/assets/schema/history.sql b/assets/schema/history.sql deleted file mode 100644 index 3323b73a0..000000000 --- a/assets/schema/history.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE DATABASE history; -use history; -CREATE TABLE `chat_feed_back` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `conv_uid` varchar(128) DEFAULT NULL COMMENT '会话id', - `conv_index` int(4) DEFAULT NULL COMMENT '第几轮会话', - `score` int(1) DEFAULT NULL COMMENT '评分', - `ques_type` varchar(32) DEFAULT NULL COMMENT '用户问题类别', - `question` longtext DEFAULT NULL COMMENT '用户问题', - `knowledge_space` varchar(128) DEFAULT NULL COMMENT '知识库', - `messages` longtext DEFAULT NULL COMMENT '评价详情', - `user_name` varchar(128) DEFAULT NULL COMMENT '评价人', - `gmt_created` datetime DEFAULT NULL, - `gmt_modified` datetime DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_conv` (`conv_uid`,`conv_index`), - KEY `idx_conv` (`conv_uid`,`conv_index`) -) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COMMENT='用户评分反馈表'; \ No newline at end of file diff --git a/assets/schema/knowledge_management.sql b/assets/schema/knowledge_management.sql index 866d958e4..e38f731d6 100644 --- a/assets/schema/knowledge_management.sql +++ b/assets/schema/knowledge_management.sql @@ -1,5 +1,15 @@ -CREATE DATABASE knowledge_management; -use knowledge_management; +-- You can change `dbgpt` to your actual metadata database name in your `.env` file +-- eg. `LOCAL_DB_NAME=dbgpt` + +CREATE DATABASE IF NOT EXISTS dbgpt; +use dbgpt; + +-- For alembic migration tool +CREATE TABLE `alembic_version` ( + version_num VARCHAR(32) NOT NULL, + CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num) +); + CREATE TABLE `knowledge_space` ( `id` int NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', `name` varchar(100) NOT NULL COMMENT 'knowledge space name', @@ -43,6 +53,102 @@ CREATE TABLE `document_chunk` ( KEY `idx_document_id` (`document_id`) COMMENT 'index:document_id' ) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COMMENT='knowledge document chunk detail'; + + +CREATE TABLE `connect_config` ( + `id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id', + `db_type` varchar(255) NOT NULL COMMENT 'db type', + `db_name` varchar(255) NOT NULL COMMENT 'db name', + `db_path` varchar(255) DEFAULT NULL COMMENT 'file db path', + `db_host` varchar(255) DEFAULT NULL COMMENT 'db connect host(not file db)', + `db_port` varchar(255) DEFAULT NULL COMMENT 'db cnnect port(not file db)', + `db_user` varchar(255) DEFAULT NULL COMMENT 'db user', + `db_pwd` varchar(255) DEFAULT NULL COMMENT 'db password', + `comment` text COMMENT 'db comment', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_db` (`db_name`), + KEY `idx_q_db_type` (`db_type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT 'Connection confi'; + +CREATE TABLE `chat_history` ( + `id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id', + `conv_uid` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation record unique id', + `chat_mode` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation scene mode', + `summary` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation record summary', + `user_name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'interlocutor', + `messages` text COLLATE utf8mb4_unicode_ci COMMENT 'Conversation details', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT 'Chat history'; + +CREATE TABLE `chat_feed_back` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `conv_uid` varchar(128) DEFAULT NULL COMMENT 'Conversation ID', + `conv_index` int(4) DEFAULT NULL COMMENT 'Round of conversation', + `score` int(1) DEFAULT NULL COMMENT 'Score of user', + `ques_type` varchar(32) DEFAULT NULL COMMENT 'User question category', + `question` longtext DEFAULT NULL COMMENT 'User question', + `knowledge_space` varchar(128) DEFAULT NULL COMMENT 'Knowledge space name', + `messages` longtext DEFAULT NULL COMMENT 'The details of user feedback', + `user_name` varchar(128) DEFAULT NULL COMMENT 'User name', + `gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + `gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_conv` (`conv_uid`,`conv_index`), + KEY `idx_conv` (`conv_uid`,`conv_index`) +) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COMMENT='User feedback table'; + + +CREATE TABLE `my_plugin` ( + `id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id', + `tenant` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'user tenant', + `user_code` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'user code', + `user_name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'user name', + `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'plugin name', + `file_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'plugin package file name', + `type` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin type', + `version` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin version', + `use_count` int DEFAULT NULL COMMENT 'plugin total use count', + `succ_count` int DEFAULT NULL COMMENT 'plugin total success count', + `gmt_created` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT 'plugin install time', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='User plugin table'; + +CREATE TABLE `plugin_hub` ( + `id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id', + `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'plugin name', + `description` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'plugin description', + `author` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin author', + `email` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin author email', + `type` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin type', + `version` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin version', + `storage_channel` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin storage channel', + `storage_url` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin download url', + `download_param` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'plugin download param', + `gmt_created` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT 'plugin upload time', + `installed` int DEFAULT NULL COMMENT 'plugin already installed count', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Plugin Hub table'; + + +CREATE TABLE `prompt_manage` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Chat scene', + `sub_chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Sub chat scene', + `prompt_type` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt type: common or private', + `prompt_name` varchar(512) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'prompt name', + `content` longtext COLLATE utf8mb4_unicode_ci COMMENT 'Prompt content', + `user_name` varchar(128) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'User name', + `gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + `gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE KEY `prompt_name_uiq` (`prompt_name`), + KEY `gmt_created_idx` (`gmt_created`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Prompt management table'; + + + CREATE DATABASE EXAMPLE_1; use EXAMPLE_1; CREATE TABLE `users` ( diff --git a/assets/schema/prompt_management.sql b/assets/schema/prompt_management.sql deleted file mode 100644 index b2ed6de23..000000000 --- a/assets/schema/prompt_management.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE DATABASE prompt_management; -use prompt_management; -CREATE TABLE `prompt_manage` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '场景', - `sub_chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '子场景', - `prompt_type` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '类型: common or private', - `prompt_name` varchar(512) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'prompt的名字', - `content` longtext COLLATE utf8mb4_unicode_ci COMMENT 'prompt的内容', - `user_name` varchar(128) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '用户名', - `gmt_created` datetime DEFAULT NULL, - `gmt_modified` datetime DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `prompt_name_uiq` (`prompt_name`), - KEY `gmt_created_idx` (`gmt_created`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='prompt管理表'; \ No newline at end of file diff --git a/docs/getting_started/faq/deploy/deploy_faq.md b/docs/getting_started/faq/deploy/deploy_faq.md index 773e99794..89017d655 100644 --- a/docs/getting_started/faq/deploy/deploy_faq.md +++ b/docs/getting_started/faq/deploy/deploy_faq.md @@ -100,7 +100,7 @@ pip install --use-pep517 fschat ``` ##### Q9: alembic.util.exc.CommandError: Target database is not up to date. -delete files in `DB-GPT/pilot/meta_data/alembic/versions/` and reboot. +delete files in `DB-GPT/pilot/meta_data/alembic/versions/` and restart. ```commandline rm -rf DB-GPT/pilot/meta_data/alembic/versions/* rm -rf DB-GPT/pilot/meta_data/alembic/dbgpt.db diff --git a/docs/getting_started/faq/kbqa/kbqa_faq.md b/docs/getting_started/faq/kbqa/kbqa_faq.md index 61bdf2c18..7b3ea8780 100644 --- a/docs/getting_started/faq/kbqa/kbqa_faq.md +++ b/docs/getting_started/faq/kbqa/kbqa_faq.md @@ -72,7 +72,7 @@ $ mysql -h127.0.0.1 -uroot -paa12345678 < ./assets/schema/knowledge_management.s ##### Q6:when pull from 0.4.0, I found historical knowledge document disappeared -In version 0.4.0, the metadata module of the DB-GPT application has been refactored. All metadata tables will now be automatically saved in the 'dbgpt' database, based on the database type specified in the .env file. If you would like to retain the existing data, it is recommended to use a data migration tool to transfer the database table information to the 'dbgpt' database. +In version 0.4.0, the metadata module of the DB-GPT application has been refactored. All metadata tables will now be automatically saved in the 'dbgpt' database, based on the database type specified in the `.env` file. If you would like to retain the existing data, it is recommended to use a data migration tool to transfer the database table information to the 'dbgpt' database. Additionally, you can change the default database name 'dbgpt' in your `.env` file. ```{tip} old database:knowledge_management; @@ -89,5 +89,6 @@ LOCAL_DB_USER=root LOCAL_DB_PASSWORD=aa12345678 LOCAL_DB_HOST=127.0.0.1 LOCAL_DB_PORT=3306 - -``` \ No newline at end of file +# You can change it to your actual metadata database name +LOCAL_DB_NAME=dbgpt +``` diff --git a/docs/locales/zh_CN/LC_MESSAGES/getting_started/faq/deploy/deploy_faq.po b/docs/locales/zh_CN/LC_MESSAGES/getting_started/faq/deploy/deploy_faq.po index e8a085390..72a2a34d6 100644 --- a/docs/locales/zh_CN/LC_MESSAGES/getting_started/faq/deploy/deploy_faq.po +++ b/docs/locales/zh_CN/LC_MESSAGES/getting_started/faq/deploy/deploy_faq.po @@ -143,8 +143,8 @@ msgstr "" #: ../../getting_started/faq/deploy/deploy_faq.md:103 #: 223026d3b9124363b695937922d8f8d5 -msgid "delete files in `DB-GPT/pilot/meta_data/alembic/versions/` and reboot." -msgstr "删除`DB-GPT/pilot/meta_data/alembic/versions/`目录下文件" +msgid "delete files in `DB-GPT/pilot/meta_data/alembic/versions/` and restart." +msgstr "删除`DB-GPT/pilot/meta_data/alembic/versions/`目录下文件然后重新启动" #~ msgid "" #~ "Q2: When use Mysql, Access denied " diff --git a/pilot/base_modules/agent/db/my_plugin_db.py b/pilot/base_modules/agent/db/my_plugin_db.py index acfb70e23..fb4fe25ee 100644 --- a/pilot/base_modules/agent/db/my_plugin_db.py +++ b/pilot/base_modules/agent/db/my_plugin_db.py @@ -4,7 +4,12 @@ from sqlalchemy import Column, Integer, String, Index, DateTime, func from sqlalchemy import UniqueConstraint from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) class MyPluginEntity(Base): @@ -27,7 +32,7 @@ class MyPluginEntity(Base): succ_count = Column( Integer, nullable=True, default=0, comment="plugin total success count" ) - created_at = Column( + gmt_created = Column( DateTime, default=datetime.utcnow, comment="plugin install time" ) UniqueConstraint("user_code", "name", name="uk_name") @@ -36,7 +41,10 @@ class MyPluginEntity(Base): class MyPluginDao(BaseDao[MyPluginEntity]): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def add(self, engity: MyPluginEntity): @@ -50,7 +58,7 @@ class MyPluginDao(BaseDao[MyPluginEntity]): version=engity.version, use_count=engity.use_count or 0, succ_count=engity.succ_count or 0, - created_at=datetime.now(), + gmt_created=datetime.now(), ) session.add(my_plugin) session.commit() diff --git a/pilot/base_modules/agent/db/plugin_hub_db.py b/pilot/base_modules/agent/db/plugin_hub_db.py index 89ec314c7..70cf16930 100644 --- a/pilot/base_modules/agent/db/plugin_hub_db.py +++ b/pilot/base_modules/agent/db/plugin_hub_db.py @@ -6,9 +6,14 @@ from sqlalchemy import UniqueConstraint from pilot.base_modules.meta_data.meta_data import Base from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session - +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) +# TODO We should consider that the production environment does not have permission to execute the DDL char_set_sql = DDL("ALTER TABLE plugin_hub CONVERT TO CHARACTER SET utf8mb4") @@ -30,7 +35,9 @@ class PluginHubEntity(Base): storage_channel = Column(String(255), comment="plugin storage channel") storage_url = Column(String(255), comment="plugin download url") download_param = Column(String(255), comment="plugin download param") - created_at = Column(DateTime, default=datetime.utcnow, comment="plugin upload time") + gmt_created = Column( + DateTime, default=datetime.utcnow, comment="plugin upload time" + ) installed = Column(Integer, default=False, comment="plugin already installed count") UniqueConstraint("name", name="uk_name") @@ -40,7 +47,10 @@ class PluginHubEntity(Base): class PluginHubDao(BaseDao[PluginHubEntity]): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def add(self, engity: PluginHubEntity): @@ -54,7 +64,7 @@ class PluginHubDao(BaseDao[PluginHubEntity]): version=engity.version, storage_channel=engity.storage_channel, storage_url=engity.storage_url, - created_at=timezone.localize(datetime.now()), + gmt_created=timezone.localize(datetime.now()), ) session.add(plugin_hub) session.commit() diff --git a/pilot/base_modules/meta_data/meta_data.py b/pilot/base_modules/meta_data/meta_data.py index 687138e77..5b79a38a5 100644 --- a/pilot/base_modules/meta_data/meta_data.py +++ b/pilot/base_modules/meta_data/meta_data.py @@ -1,20 +1,13 @@ -import uuid import os -import duckdb import sqlite3 import logging -import fnmatch -from datetime import datetime -from typing import Optional, Type, TypeVar -from sqlalchemy import create_engine, DateTime, String, func, MetaData, DDL +from sqlalchemy import create_engine, DDL from sqlalchemy.exc import OperationalError -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import Mapped from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base -from alembic import context, command +from alembic import command from alembic.config import Config as AlembicConfig from urllib.parse import quote from pilot.configs.config import Config @@ -28,7 +21,8 @@ default_db_path = os.path.join(os.getcwd(), "meta_data") os.makedirs(default_db_path, exist_ok=True) # Meta Info -db_name = "dbgpt" +META_DATA_DATABASE = CFG.LOCAL_DB_NAME +db_name = META_DATA_DATABASE db_path = default_db_path + f"/{db_name}.db" connection = sqlite3.connect(db_path) @@ -47,6 +41,7 @@ if CFG.LOCAL_DB_TYPE == "mysql": try: # try to connect with engine_temp.connect() as conn: + # TODO We should consider that the production environment does not have permission to execute the DDL conn.execute(DDL(f"CREATE DATABASE IF NOT EXISTS {db_name}")) print(f"Already connect '{db_name}'") @@ -76,8 +71,6 @@ Base = declarative_base() # Base.metadata.create_all() -# 创建Alembic配置对象 - alembic_ini_path = default_db_path + "/alembic.ini" alembic_cfg = AlembicConfig(alembic_ini_path) @@ -100,7 +93,18 @@ alembic_cfg.attributes["session"] = session # Base.metadata.drop_all(engine) -def ddl_init_and_upgrade(): +def ddl_init_and_upgrade(disable_alembic_upgrade: bool): + """Initialize and upgrade database metadata + + Args: + disable_alembic_upgrade (bool): Whether to enable alembic to initialize and upgrade database metadata + """ + if disable_alembic_upgrade: + logger.info( + "disable_alembic_upgrade is true, not to initialize and upgrade database metadata with alembic" + ) + return + # Base.metadata.create_all(bind=engine) # 生成并应用迁移脚本 # command.upgrade(alembic_cfg, 'head') diff --git a/pilot/base_modules/meta_data/requirement.txt b/pilot/base_modules/meta_data/requirement.txt deleted file mode 100644 index de93d8a90..000000000 --- a/pilot/base_modules/meta_data/requirement.txt +++ /dev/null @@ -1 +0,0 @@ -alembic==1.12.0 \ No newline at end of file diff --git a/pilot/configs/config.py b/pilot/configs/config.py index 680f9f856..6213cedf2 100644 --- a/pilot/configs/config.py +++ b/pilot/configs/config.py @@ -184,7 +184,7 @@ class Config(metaclass=Singleton): if self.LOCAL_DB_HOST is None and self.LOCAL_DB_PATH == "": self.LOCAL_DB_HOST = "127.0.0.1" - self.LOCAL_DB_NAME = os.getenv("LOCAL_DB_NAME") + self.LOCAL_DB_NAME = os.getenv("LOCAL_DB_NAME", "dbgpt") self.LOCAL_DB_PORT = int(os.getenv("LOCAL_DB_PORT", 3306)) self.LOCAL_DB_USER = os.getenv("LOCAL_DB_USER", "root") self.LOCAL_DB_PASSWORD = os.getenv("LOCAL_DB_PASSWORD", "aa123456") diff --git a/pilot/connections/manages/connect_config_db.py b/pilot/connections/manages/connect_config_db.py index 7443d18ad..0898bfbea 100644 --- a/pilot/connections/manages/connect_config_db.py +++ b/pilot/connections/manages/connect_config_db.py @@ -1,9 +1,15 @@ -from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session from typing import List from sqlalchemy import Column, Integer, String, Index, DateTime, func, Boolean, Text from sqlalchemy import UniqueConstraint +from pilot.base_modules.meta_data.base_dao import BaseDao +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) + class ConnectConfigEntity(Base): __tablename__ = "connect_config" @@ -28,7 +34,10 @@ class ConnectConfigEntity(Base): class ConnectConfigDao(BaseDao[ConnectConfigEntity]): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def update(self, entity: ConnectConfigEntity): diff --git a/pilot/memory/chat_history/chat_history_db.py b/pilot/memory/chat_history/chat_history_db.py index 2f9b2999c..8ab6e9306 100644 --- a/pilot/memory/chat_history/chat_history_db.py +++ b/pilot/memory/chat_history/chat_history_db.py @@ -1,9 +1,15 @@ -from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session from typing import List from sqlalchemy import Column, Integer, String, Index, DateTime, func, Boolean, Text from sqlalchemy import UniqueConstraint +from pilot.base_modules.meta_data.base_dao import BaseDao +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) + class ChatHistoryEntity(Base): __tablename__ = "chat_history" @@ -34,7 +40,10 @@ class ChatHistoryEntity(Base): class ChatHistoryDao(BaseDao[ChatHistoryEntity]): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def list_last_20(self, user_name: str = None): diff --git a/pilot/memory/chat_history/store_type/meta_db_history.py b/pilot/memory/chat_history/store_type/meta_db_history.py index cc51804a1..8afbaf06b 100644 --- a/pilot/memory/chat_history/store_type/meta_db_history.py +++ b/pilot/memory/chat_history/store_type/meta_db_history.py @@ -47,7 +47,7 @@ class DbHistoryMemory(BaseChatHistoryMemory): logger.error("init create conversation log error!" + str(e)) def append(self, once_message: OnceConversation) -> None: - logger.info("db history append:{}", once_message) + logger.info(f"db history append: {once_message}") chat_history: ChatHistoryEntity = self.chat_history_dao.get_by_uid( self.chat_seesion_id ) diff --git a/pilot/openapi/api_v1/feedback/feed_back_db.py b/pilot/openapi/api_v1/feedback/feed_back_db.py index 3d697263b..02afb3215 100644 --- a/pilot/openapi/api_v1/feedback/feed_back_db.py +++ b/pilot/openapi/api_v1/feedback/feed_back_db.py @@ -3,7 +3,12 @@ from datetime import datetime from sqlalchemy import Column, Integer, Text, String, DateTime from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) from pilot.openapi.api_v1.feedback.feed_back_model import FeedBackBody @@ -36,7 +41,10 @@ class ChatFeedBackEntity(Base): class ChatFeedBackDao(BaseDao): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def create_or_update_chat_feed_back(self, feed_back: FeedBackBody): diff --git a/pilot/server/base.py b/pilot/server/base.py index e66efcf43..71faeb821 100644 --- a/pilot/server/base.py +++ b/pilot/server/base.py @@ -6,7 +6,6 @@ from typing import Optional, Any from dataclasses import dataclass, field from pilot.configs.config import Config -from pilot.configs.model_config import PLUGINS_DIR from pilot.component import SystemApp from pilot.utils.parameter_utils import BaseParameters from pilot.base_modules.meta_data.meta_data import ddl_init_and_upgrade @@ -29,7 +28,7 @@ def async_db_summary(system_app: SystemApp): thread.start() -def server_init(args, system_app: SystemApp): +def server_init(param: "WebWerverParameters", system_app: SystemApp): from pilot.base_modules.agent.commands.command_mange import CommandRegistry # logger.info(f"args: {args}") @@ -38,7 +37,7 @@ def server_init(args, system_app: SystemApp): cfg = Config() cfg.SYSTEM_APP = system_app - ddl_init_and_upgrade() + ddl_init_and_upgrade(param.disable_alembic_upgrade) # load_native_plugins(cfg) signal.signal(signal.SIGINT, signal_handler) @@ -148,3 +147,9 @@ class WebWerverParameters(BaseParameters): "help": "The filename to store tracer span records", }, ) + disable_alembic_upgrade: Optional[bool] = field( + default=False, + metadata={ + "help": "Whether to disable alembic to initialize and upgrade database metadata", + }, + ) diff --git a/pilot/server/knowledge/chunk_db.py b/pilot/server/knowledge/chunk_db.py index f1e792377..afa8cc5a8 100644 --- a/pilot/server/knowledge/chunk_db.py +++ b/pilot/server/knowledge/chunk_db.py @@ -4,7 +4,12 @@ from typing import List from sqlalchemy import Column, String, DateTime, Integer, Text, func from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) from pilot.configs.config import Config CFG = Config() @@ -32,7 +37,10 @@ class DocumentChunkEntity(Base): class DocumentChunkDao(BaseDao): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def create_documents_chunks(self, documents: List): diff --git a/pilot/server/knowledge/document_db.py b/pilot/server/knowledge/document_db.py index 64d73ec07..3e6dfb0c4 100644 --- a/pilot/server/knowledge/document_db.py +++ b/pilot/server/knowledge/document_db.py @@ -3,7 +3,12 @@ from datetime import datetime from sqlalchemy import Column, String, DateTime, Integer, Text, func from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) from pilot.configs.config import Config CFG = Config() @@ -35,7 +40,10 @@ class KnowledgeDocumentEntity(Base): class KnowledgeDocumentDao(BaseDao): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def create_knowledge_document(self, document: KnowledgeDocumentEntity): diff --git a/pilot/server/knowledge/space_db.py b/pilot/server/knowledge/space_db.py index b833906da..8925fb5a1 100644 --- a/pilot/server/knowledge/space_db.py +++ b/pilot/server/knowledge/space_db.py @@ -3,7 +3,12 @@ from datetime import datetime from sqlalchemy import Column, Integer, Text, String, DateTime from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) from pilot.configs.config import Config from pilot.server.knowledge.request.request import KnowledgeSpaceRequest @@ -32,7 +37,10 @@ class KnowledgeSpaceEntity(Base): class KnowledgeSpaceDao(BaseDao): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def create_knowledge_space(self, space: KnowledgeSpaceRequest): diff --git a/pilot/server/prompt/prompt_manage_db.py b/pilot/server/prompt/prompt_manage_db.py index ea482a3bb..3245b3450 100644 --- a/pilot/server/prompt/prompt_manage_db.py +++ b/pilot/server/prompt/prompt_manage_db.py @@ -3,7 +3,12 @@ from datetime import datetime from sqlalchemy import Column, Integer, Text, String, DateTime from pilot.base_modules.meta_data.base_dao import BaseDao -from pilot.base_modules.meta_data.meta_data import Base, engine, session +from pilot.base_modules.meta_data.meta_data import ( + Base, + engine, + session, + META_DATA_DATABASE, +) from pilot.configs.config import Config from pilot.server.prompt.request.request import PromptManageRequest @@ -34,7 +39,10 @@ class PromptManageEntity(Base): class PromptManageDao(BaseDao): def __init__(self): super().__init__( - database="dbgpt", orm_base=Base, db_engine=engine, session=session + database=META_DATA_DATABASE, + orm_base=Base, + db_engine=engine, + session=session, ) def create_prompt(self, prompt: PromptManageRequest): diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index eb54df339..072b527f1 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -11,4 +11,4 @@ pytesseract==0.3.10 # python code format black # for git hooks -pre-commmit \ No newline at end of file +pre-commit \ No newline at end of file