From 1547c3aff10a0158d5f5a10c4ada632ebecb3ca8 Mon Sep 17 00:00:00 2001 From: yhjun1026 <460342015@qq.com> Date: Thu, 27 Jul 2023 17:57:18 +0800 Subject: [PATCH 1/6] DDL run bug fix --- pilot/connections/db_conn_info.py | 5 ++++ .../manages/connect_storage_duckdb.py | 24 ++++++++++++++++++- .../connections/manages/connection_manager.py | 10 ++++++++ pilot/openapi/api_v1/api_v1.py | 19 +++++++++++---- pilot/scene/chat_data/__init__.py | 0 pilot/scene/chat_data/chat_excel/__init__.py | 0 .../chat_excel/excel_analyze/__init__.py | 0 .../chat_excel/excel_learning/__init__.py | 0 8 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 pilot/scene/chat_data/__init__.py create mode 100644 pilot/scene/chat_data/chat_excel/__init__.py create mode 100644 pilot/scene/chat_data/chat_excel/excel_analyze/__init__.py create mode 100644 pilot/scene/chat_data/chat_excel/excel_learning/__init__.py diff --git a/pilot/connections/db_conn_info.py b/pilot/connections/db_conn_info.py index 767aac881..e1c633979 100644 --- a/pilot/connections/db_conn_info.py +++ b/pilot/connections/db_conn_info.py @@ -10,3 +10,8 @@ class DBConfig(BaseModel): db_user: str = "" db_pwd: str = "" comment: str = "" + +class DbTypeInfo(BaseModel): + db_type:str + is_file_db: bool = False + diff --git a/pilot/connections/manages/connect_storage_duckdb.py b/pilot/connections/manages/connect_storage_duckdb.py index 9ff291ad2..0f0c8fece 100644 --- a/pilot/connections/manages/connect_storage_duckdb.py +++ b/pilot/connections/manages/connect_storage_duckdb.py @@ -47,6 +47,27 @@ class DuckdbConnectConfig: except Exception as e: print("add db connect info error1!" + str(e)) + def update_db_info(self, + db_name, + db_type, + db_path, + db_host: str, + db_port: int, + db_user: str, + db_pwd: str, + comment: str ): + old_db_conf = self.get_db_config(db_name) + if old_db_conf: + try: + cursor = self.connect.cursor() + cursor.execute(f"UPDATE connect_config set db_type={db_type}, db_path={db_path}, db_host={db_host}, db_port={db_port}, db_user={db_user}, db_pwd={db_pwd}, comment={comment} where db_name={db_name}") + cursor.commit() + self.connect.commit() + except Exception as e: + print("edit db connect info error2!" + str(e)) + return True + raise ValueError(f"{db_name} not have config info!") + def get_file_db_name(self, path): try: conn = duckdb.connect(path) @@ -55,6 +76,7 @@ class DuckdbConnectConfig: except Exception as e: raise "Unusable duckdb database path:" + path + def add_file_db(self, db_name, db_type, db_path: str, comment: str = ""): try: cursor = self.connect.cursor() @@ -89,7 +111,7 @@ class DuckdbConnectConfig: for i, field in enumerate(fields): row_dict[field] = row_1[i] return row_dict - return {} + return None def get_db_list(self): if os.path.isfile(duckdb_path): diff --git a/pilot/connections/manages/connection_manager.py b/pilot/connections/manages/connection_manager.py index 291127127..557a19007 100644 --- a/pilot/connections/manages/connection_manager.py +++ b/pilot/connections/manages/connection_manager.py @@ -117,6 +117,16 @@ class ConnectManager: def delete_db(self, db_name: str): return self.storage.delete_db(db_name) + def edit_db(self, db_info: DBConfig): + return self.storage.update_db_info(db_info.db_name, + db_info.db_type, + db_info.file_path, + db_info.db_host, + db_info.db_port, + db_info.db_user, + db_info.db_pwd, + db_info.comment) + def add_db(self, db_info: DBConfig): db_type = DBType.of_db_type(db_info.db_type) if db_type.is_file_db(): diff --git a/pilot/openapi/api_v1/api_v1.py b/pilot/openapi/api_v1/api_v1.py index 7d3482dfc..76a04f819 100644 --- a/pilot/openapi/api_v1/api_v1.py +++ b/pilot/openapi/api_v1/api_v1.py @@ -25,7 +25,7 @@ from pilot.openapi.api_v1.api_view_model import ( MessageVo, ChatSceneVo, ) -from pilot.connections.db_conn_info import DBConfig +from pilot.connections.db_conn_info import DBConfig, DbTypeInfo from pilot.configs.config import Config from pilot.server.knowledge.service import KnowledgeService from pilot.server.knowledge.request.request import KnowledgeSpaceRequest @@ -35,7 +35,7 @@ from pilot.scene.base import ChatScene from pilot.scene.chat_factory import ChatFactory from pilot.configs.model_config import LOGDIR from pilot.utils import build_logger -from pilot.scene.base_message import BaseMessage +from pilot.common.schema import DBType from pilot.memory.chat_history.duckdb_history import DuckdbHistoryMemory from pilot.scene.message import OnceConversation @@ -106,14 +106,23 @@ async def dialogue_list(db_config: DBConfig = Body()): return Result.succ(CFG.LOCAL_DB_MANAGE.add_db(db_config)) +@router.post("/v1/chat/db/edit", response_model=Result[bool]) +async def dialogue_list(db_config: DBConfig = Body()): + return Result.succ(CFG.LOCAL_DB_MANAGE.edit_db(db_config)) + + @router.post("/v1/chat/db/delete", response_model=Result[bool]) async def dialogue_list(db_name: str = None): return Result.succ(CFG.LOCAL_DB_MANAGE.delete_db(db_name)) -@router.get("/v1/chat/db/support/type", response_model=Result[str]) +@router.get("/v1/chat/db/support/type", response_model=Result[DbTypeInfo]) async def db_support_types(): - return Result[str].succ(["mysql", "mssql", "duckdb"]) + support_types = [DBType.Mysql, DBType.MSSQL, DBType.DuckDb] + db_type_infos = [] + for type in support_types: + db_type_infos.append(DbTypeInfo(type.value(), type.is_file_db())) + return Result[DbTypeInfo].succ(db_type_infos) @router.get("/v1/chat/dialogue/list", response_model=Result[ConversationVo]) @@ -160,7 +169,7 @@ async def dialogue_scenes(): @router.post("/v1/chat/dialogue/new", response_model=Result[ConversationVo]) async def dialogue_new( - chat_mode: str = ChatScene.ChatNormal.value(), user_id: str = None + chat_mode: str = ChatScene.ChatNormal.value(), user_id: str = None ): conv_vo = __new_conversation(chat_mode, user_id) return Result.succ(conv_vo) diff --git a/pilot/scene/chat_data/__init__.py b/pilot/scene/chat_data/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/scene/chat_data/chat_excel/__init__.py b/pilot/scene/chat_data/chat_excel/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/scene/chat_data/chat_excel/excel_analyze/__init__.py b/pilot/scene/chat_data/chat_excel/excel_analyze/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/scene/chat_data/chat_excel/excel_learning/__init__.py b/pilot/scene/chat_data/chat_excel/excel_learning/__init__.py new file mode 100644 index 000000000..e69de29bb From 40659b93e7871e118e21842504e4f49a8359108a Mon Sep 17 00:00:00 2001 From: yhjun1026 <460342015@qq.com> Date: Tue, 1 Aug 2023 14:14:10 +0800 Subject: [PATCH 2/6] =?UTF-8?q?feat=EF=BC=9ADb=20connect=20setting=20on=20?= =?UTF-8?q?web?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manages/connect_storage_duckdb.py | 21 +++-- .../connections/manages/connection_manager.py | 43 ++++++---- pilot/openapi/api_v1/api_v1.py | 10 +-- pilot/scene/chat_db/auto_execute/prompt.py | 4 +- pilot/server/static/404.html | 2 +- pilot/server/static/404/index.html | 2 +- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 .../Cow4Dk3Cb5ywOteYPWBYm/_buildManifest.js | 1 - .../Cow4Dk3Cb5ywOteYPWBYm/_ssgManifest.js | 1 - .../static/chunks/110-470e5d8a0cb4cf14.js | 77 ------------------ .../static/chunks/160-ba31b9436f6470d2.js | 10 --- .../static/chunks/163-59f735b072797bdd.js | 16 ---- .../static/chunks/192-e9f419fb9f5bc502.js | 1 + ...5b31482f8e8.js => 320-63dc542e9a7120d1.js} | 4 +- .../static/chunks/341-c3312a204c5835b8.js | 10 --- ...0698005aba2.js => 409-4b199bf070fd70fc.js} | 4 +- .../static/chunks/679-2432e2fce32149a4.js | 1 - .../static/chunks/759-22c7defc3aa943a7.js | 10 +++ .../static/chunks/957-80662c0af3fc4d0d.js | 63 -------------- .../static/chunks/957-c96ff4de6e80d713.js | 63 ++++++++++++++ .../static/chunks/992-f088fd7821baa330.js | 68 ---------------- .../chunks/app/chat/page-4266d316599e0141.js | 1 - .../chunks/app/chat/page-4a580c13b269a988.js | 1 - .../chunks/app/chat/page-86595a764417b205.js | 1 + .../chunks/app/chat/page-fa8f6230bc48190e.js | 1 - .../chunklist/page-042df8c81ee99ad6.js | 1 - ...11a9476f41.js => page-583473409d3bf959.js} | 2 +- .../chunklist/page-76d75e816f549f8a.js | 1 - .../documents/page-5386a639d658c30c.js | 1 - .../documents/page-7226571ba18444cc.js | 1 - .../documents/page-d01efb573de36698.js | 1 - .../documents/page-d94518835f877352.js | 1 + .../app/datastores/page-6193a6580da1c259.js | 1 - .../app/datastores/page-643e5d19222b3bcd.js | 1 - .../app/datastores/page-71ca37d05b729f1d.js | 1 + .../app/datastores/page-d4200bb6a31d3cd4.js | 1 - ...f415780f.js => layout-055e926a104869ef.js} | 2 +- .../chunks/app/layout-34c784bda079f18d.js | 1 - .../chunks/app/layout-6d9f7ec39c148c6a.js | 1 - .../chunks/app/page-254872a7b51c14e0.js | 1 - ...88dcc52057.js => page-ba37471c26b3ed90.js} | 2 +- .../chunks/app/page-d81704e0a3437383.js | 1 - .../chunks/main-app-1351e9feb1e97e03.js | 1 - .../chunks/main-app-75c197595e152149.js | 1 - ...f1e5c68.js => webpack-8be0750561cfcccd.js} | 2 +- .../static/chunks/webpack-e0b549c3ec4ce91b.js | 1 - .../_next/static/css/1c53d4eca82e2bb3.css | 3 - .../_next/static/css/70a90cb7ce1e4b6d.css | 3 - .../_next/static/css/f7886471966370ed.css | 3 + .../kvEPRdcHK79LEj3_AaRj1/_buildManifest.js | 1 - .../kvEPRdcHK79LEj3_AaRj1/_ssgManifest.js | 1 - pilot/server/static/bg1.avif | Bin 77726 -> 0 bytes pilot/server/static/bg2.png | Bin 202104 -> 0 bytes pilot/server/static/chat/index.html | 2 +- pilot/server/static/chat/index.txt | 14 ++-- .../datastores/documents/chunklist/index.html | 2 +- .../datastores/documents/chunklist/index.txt | 14 ++-- .../static/datastores/documents/index.html | 2 +- .../static/datastores/documents/index.txt | 14 ++-- pilot/server/static/datastores/index.html | 2 +- pilot/server/static/datastores/index.txt | 14 ++-- pilot/server/static/index.html | 2 +- pilot/server/static/index.txt | 14 ++-- pilot/server/webserver_base.py | 2 +- 65 files changed, 178 insertions(+), 355 deletions(-) rename pilot/server/static/_next/static/{AVF7sR15c1tF8wuv8mGBK => 9C-EourU_3qSC0HuMlO9a}/_buildManifest.js (100%) rename pilot/server/static/_next/static/{AVF7sR15c1tF8wuv8mGBK => 9C-EourU_3qSC0HuMlO9a}/_ssgManifest.js (100%) delete mode 100644 pilot/server/static/_next/static/Cow4Dk3Cb5ywOteYPWBYm/_buildManifest.js delete mode 100644 pilot/server/static/_next/static/Cow4Dk3Cb5ywOteYPWBYm/_ssgManifest.js delete mode 100644 pilot/server/static/_next/static/chunks/110-470e5d8a0cb4cf14.js delete mode 100644 pilot/server/static/_next/static/chunks/160-ba31b9436f6470d2.js delete mode 100644 pilot/server/static/_next/static/chunks/163-59f735b072797bdd.js create mode 100644 pilot/server/static/_next/static/chunks/192-e9f419fb9f5bc502.js rename pilot/server/static/_next/static/chunks/{436-0a7be5b31482f8e8.js => 320-63dc542e9a7120d1.js} (79%) delete mode 100644 pilot/server/static/_next/static/chunks/341-c3312a204c5835b8.js rename pilot/server/static/_next/static/chunks/{144-8e8590698005aba2.js => 409-4b199bf070fd70fc.js} (72%) delete mode 100644 pilot/server/static/_next/static/chunks/679-2432e2fce32149a4.js create mode 100644 pilot/server/static/_next/static/chunks/759-22c7defc3aa943a7.js delete mode 100644 pilot/server/static/_next/static/chunks/957-80662c0af3fc4d0d.js create mode 100644 pilot/server/static/_next/static/chunks/957-c96ff4de6e80d713.js delete mode 100644 pilot/server/static/_next/static/chunks/992-f088fd7821baa330.js delete mode 100644 pilot/server/static/_next/static/chunks/app/chat/page-4266d316599e0141.js delete mode 100644 pilot/server/static/_next/static/chunks/app/chat/page-4a580c13b269a988.js create mode 100644 pilot/server/static/_next/static/chunks/app/chat/page-86595a764417b205.js delete mode 100644 pilot/server/static/_next/static/chunks/app/chat/page-fa8f6230bc48190e.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/documents/chunklist/page-042df8c81ee99ad6.js rename pilot/server/static/_next/static/chunks/app/datastores/documents/chunklist/{page-1fa22911a9476f41.js => page-583473409d3bf959.js} (98%) delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/documents/chunklist/page-76d75e816f549f8a.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/documents/page-5386a639d658c30c.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/documents/page-7226571ba18444cc.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/documents/page-d01efb573de36698.js create mode 100644 pilot/server/static/_next/static/chunks/app/datastores/documents/page-d94518835f877352.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/page-6193a6580da1c259.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/page-643e5d19222b3bcd.js create mode 100644 pilot/server/static/_next/static/chunks/app/datastores/page-71ca37d05b729f1d.js delete mode 100644 pilot/server/static/_next/static/chunks/app/datastores/page-d4200bb6a31d3cd4.js rename pilot/server/static/_next/static/chunks/app/{layout-2a5db76cf415780f.js => layout-055e926a104869ef.js} (61%) delete mode 100644 pilot/server/static/_next/static/chunks/app/layout-34c784bda079f18d.js delete mode 100644 pilot/server/static/_next/static/chunks/app/layout-6d9f7ec39c148c6a.js delete mode 100644 pilot/server/static/_next/static/chunks/app/page-254872a7b51c14e0.js rename pilot/server/static/_next/static/chunks/app/{page-eda7ab88dcc52057.js => page-ba37471c26b3ed90.js} (97%) delete mode 100644 pilot/server/static/_next/static/chunks/app/page-d81704e0a3437383.js delete mode 100644 pilot/server/static/_next/static/chunks/main-app-1351e9feb1e97e03.js delete mode 100644 pilot/server/static/_next/static/chunks/main-app-75c197595e152149.js rename pilot/server/static/_next/static/chunks/{webpack-81b9e46a3f1e5c68.js => webpack-8be0750561cfcccd.js} (98%) delete mode 100644 pilot/server/static/_next/static/chunks/webpack-e0b549c3ec4ce91b.js delete mode 100644 pilot/server/static/_next/static/css/1c53d4eca82e2bb3.css delete mode 100644 pilot/server/static/_next/static/css/70a90cb7ce1e4b6d.css create mode 100644 pilot/server/static/_next/static/css/f7886471966370ed.css delete mode 100644 pilot/server/static/_next/static/kvEPRdcHK79LEj3_AaRj1/_buildManifest.js delete mode 100644 pilot/server/static/_next/static/kvEPRdcHK79LEj3_AaRj1/_ssgManifest.js delete mode 100644 pilot/server/static/bg1.avif delete mode 100644 pilot/server/static/bg2.png diff --git a/pilot/connections/manages/connect_storage_duckdb.py b/pilot/connections/manages/connect_storage_duckdb.py index 0f0c8fece..29542f6d5 100644 --- a/pilot/connections/manages/connect_storage_duckdb.py +++ b/pilot/connections/manages/connect_storage_duckdb.py @@ -50,17 +50,20 @@ class DuckdbConnectConfig: def update_db_info(self, db_name, db_type, - db_path, - db_host: str, - db_port: int, - db_user: str, - db_pwd: str, - comment: str ): + db_path: str = "", + db_host: str = "", + db_port: int = 0, + db_user: str = "", + db_pwd: str = "", + comment: str = "" ): old_db_conf = self.get_db_config(db_name) if old_db_conf: try: cursor = self.connect.cursor() - cursor.execute(f"UPDATE connect_config set db_type={db_type}, db_path={db_path}, db_host={db_host}, db_port={db_port}, db_user={db_user}, db_pwd={db_pwd}, comment={comment} where db_name={db_name}") + if not db_path: + cursor.execute(f"UPDATE connect_config set db_type='{db_type}', db_host='{db_host}', db_port={db_port}, db_user='{db_user}', db_pwd='{db_pwd}', comment='{comment}' where db_name='{db_name}'") + else: + cursor.execute(f"UPDATE connect_config set db_type='{db_type}', db_path='{db_path}', comment='{comment}' where db_name='{db_name}'") cursor.commit() self.connect.commit() except Exception as e: @@ -82,7 +85,7 @@ class DuckdbConnectConfig: cursor = self.connect.cursor() cursor.execute( "INSERT INTO connect_config(id, db_name, db_type, db_path, db_host, db_port, db_user, db_pwd, comment)VALUES(nextval('seq_id'),?,?,?,?,?,?,?,?)", - [db_name, db_type, db_path, "", "", "", "", comment], + [db_name, db_type, db_path, "", 0, "", "", comment], ) cursor.commit() self.connect.commit() @@ -116,7 +119,7 @@ class DuckdbConnectConfig: def get_db_list(self): if os.path.isfile(duckdb_path): cursor = duckdb.connect(duckdb_path).cursor() - cursor.execute("SELECT db_name, db_type, comment FROM connect_config ") + cursor.execute("SELECT * FROM connect_config ") fields = [field[0] for field in cursor.description] data = [] diff --git a/pilot/connections/manages/connection_manager.py b/pilot/connections/manages/connection_manager.py index 557a19007..9e774b646 100644 --- a/pilot/connections/manages/connection_manager.py +++ b/pilot/connections/manages/connection_manager.py @@ -1,3 +1,5 @@ +import threading + from pilot.configs.config import Config from pilot.connections.manages.connect_storage_duckdb import DuckdbConnectConfig from pilot.common.schema import DBType @@ -11,6 +13,7 @@ from pilot.connections.rdbms.rdbms_connect import RDBMSDatabase from pilot.singleton import Singleton from pilot.common.sql_database import Database from pilot.connections.db_conn_info import DBConfig +from pilot.summary.db_summary_client import DBSummaryClient CFG = Config() @@ -34,6 +37,7 @@ class ConnectManager: def __init__(self): self.storage = DuckdbConnectConfig() + self.db_summary_client = DBSummaryClient() self.__load_config_db() def __load_config_db(self): @@ -128,19 +132,28 @@ class ConnectManager: db_info.comment) def add_db(self, db_info: DBConfig): - db_type = DBType.of_db_type(db_info.db_type) - if db_type.is_file_db(): - self.storage.add_file_db( - db_info.db_name, db_info.db_type, db_info.file_path - ) - else: - self.storage.add_url_db( - db_info.db_name, - db_info.db_type, - db_info.db_host, - db_info.db_port, - db_info.db_user, - db_info.db_pwd, - db_info.comment, - ) + print(f"add_db:{db_info.__dict__}") + try: + db_type = DBType.of_db_type(db_info.db_type) + if db_type.is_file_db(): + self.storage.add_file_db( + db_info.db_name, db_info.db_type, db_info.file_path + ) + else: + + self.storage.add_url_db( + db_info.db_name, + db_info.db_type, + db_info.db_host, + db_info.db_port, + db_info.db_user, + db_info.db_pwd, + db_info.comment, + ) + # async embedding + thread = threading.Thread(target=self.db_summary_client.db_summary_embedding(db_info.db_name, db_info.db_type)) + thread.start() + except Exception as e: + raise ValueError("Add db connect info error!" + str(e)) + return True diff --git a/pilot/openapi/api_v1/api_v1.py b/pilot/openapi/api_v1/api_v1.py index 76a04f819..7c483e2e9 100644 --- a/pilot/openapi/api_v1/api_v1.py +++ b/pilot/openapi/api_v1/api_v1.py @@ -97,22 +97,22 @@ def knowledge_list(): @router.get("/v1/chat/db/list", response_model=Result[DBConfig]) -async def dialogue_list(): +async def db_connect_list(): return Result.succ(CFG.LOCAL_DB_MANAGE.get_db_list()) @router.post("/v1/chat/db/add", response_model=Result[bool]) -async def dialogue_list(db_config: DBConfig = Body()): +async def db_connect_add(db_config: DBConfig = Body()): return Result.succ(CFG.LOCAL_DB_MANAGE.add_db(db_config)) @router.post("/v1/chat/db/edit", response_model=Result[bool]) -async def dialogue_list(db_config: DBConfig = Body()): +async def db_connect_edit(db_config: DBConfig = Body()): return Result.succ(CFG.LOCAL_DB_MANAGE.edit_db(db_config)) @router.post("/v1/chat/db/delete", response_model=Result[bool]) -async def dialogue_list(db_name: str = None): +async def db_connect_delete(db_name: str = None): return Result.succ(CFG.LOCAL_DB_MANAGE.delete_db(db_name)) @@ -121,7 +121,7 @@ async def db_support_types(): support_types = [DBType.Mysql, DBType.MSSQL, DBType.DuckDb] db_type_infos = [] for type in support_types: - db_type_infos.append(DbTypeInfo(type.value(), type.is_file_db())) + db_type_infos.append(DbTypeInfo(db_type=type.value(), is_file_db=type.is_file_db())) return Result[DbTypeInfo].succ(db_type_infos) diff --git a/pilot/scene/chat_db/auto_execute/prompt.py b/pilot/scene/chat_db/auto_execute/prompt.py index 6ee691a88..e284ce1a9 100644 --- a/pilot/scene/chat_db/auto_execute/prompt.py +++ b/pilot/scene/chat_db/auto_execute/prompt.py @@ -8,10 +8,10 @@ from pilot.scene.chat_db.auto_execute.example import sql_data_example CFG = Config() -PROMPT_SCENE_DEFINE = None +PROMPT_SCENE_DEFINE = "You are a SQL expert. " _DEFAULT_TEMPLATE = """ -You are a SQL expert. Given an input question, create a syntactically correct {dialect} sql. +Given an input question, create a syntactically correct {dialect} sql. Unless the user specifies in his question a specific number of examples he wishes to obtain, always limit your query to at most {top_k} results. Use as few tables as possible when querying. diff --git a/pilot/server/static/404.html b/pilot/server/static/404.html index 4da41deb4..2dda42bba 100644 --- a/pilot/server/static/404.html +++ b/pilot/server/static/404.html @@ -1 +1 @@ -