From 1547c3aff10a0158d5f5a10c4ada632ebecb3ca8 Mon Sep 17 00:00:00 2001 From: yhjun1026 <460342015@qq.com> Date: Thu, 27 Jul 2023 17:57:18 +0800 Subject: [PATCH 01/16] DDL run bug fix --- pilot/connections/db_conn_info.py | 5 ++++ .../manages/connect_storage_duckdb.py | 24 ++++++++++++++++++- .../connections/manages/connection_manager.py | 10 ++++++++ pilot/openapi/api_v1/api_v1.py | 19 +++++++++++---- pilot/scene/chat_data/__init__.py | 0 pilot/scene/chat_data/chat_excel/__init__.py | 0 .../chat_excel/excel_analyze/__init__.py | 0 .../chat_excel/excel_learning/__init__.py | 0 8 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 pilot/scene/chat_data/__init__.py create mode 100644 pilot/scene/chat_data/chat_excel/__init__.py create mode 100644 pilot/scene/chat_data/chat_excel/excel_analyze/__init__.py create mode 100644 pilot/scene/chat_data/chat_excel/excel_learning/__init__.py diff --git a/pilot/connections/db_conn_info.py b/pilot/connections/db_conn_info.py index 767aac881..e1c633979 100644 --- a/pilot/connections/db_conn_info.py +++ b/pilot/connections/db_conn_info.py @@ -10,3 +10,8 @@ class DBConfig(BaseModel): db_user: str = "" db_pwd: str = "" comment: str = "" + +class DbTypeInfo(BaseModel): + db_type:str + is_file_db: bool = False + diff --git a/pilot/connections/manages/connect_storage_duckdb.py b/pilot/connections/manages/connect_storage_duckdb.py index 9ff291ad2..0f0c8fece 100644 --- a/pilot/connections/manages/connect_storage_duckdb.py +++ b/pilot/connections/manages/connect_storage_duckdb.py @@ -47,6 +47,27 @@ class DuckdbConnectConfig: except Exception as e: print("add db connect info error1!" + str(e)) + def update_db_info(self, + db_name, + db_type, + db_path, + db_host: str, + db_port: int, + db_user: str, + db_pwd: str, + comment: str ): + old_db_conf = self.get_db_config(db_name) + if old_db_conf: + try: + cursor = self.connect.cursor() + cursor.execute(f"UPDATE connect_config set db_type={db_type}, db_path={db_path}, db_host={db_host}, db_port={db_port}, db_user={db_user}, db_pwd={db_pwd}, comment={comment} where db_name={db_name}") + cursor.commit() + self.connect.commit() + except Exception as e: + print("edit db connect info error2!" + str(e)) + return True + raise ValueError(f"{db_name} not have config info!") + def get_file_db_name(self, path): try: conn = duckdb.connect(path) @@ -55,6 +76,7 @@ class DuckdbConnectConfig: except Exception as e: raise "Unusable duckdb database path:" + path + def add_file_db(self, db_name, db_type, db_path: str, comment: str = ""): try: cursor = self.connect.cursor() @@ -89,7 +111,7 @@ class DuckdbConnectConfig: for i, field in enumerate(fields): row_dict[field] = row_1[i] return row_dict - return {} + return None def get_db_list(self): if os.path.isfile(duckdb_path): diff --git a/pilot/connections/manages/connection_manager.py b/pilot/connections/manages/connection_manager.py index 291127127..557a19007 100644 --- a/pilot/connections/manages/connection_manager.py +++ b/pilot/connections/manages/connection_manager.py @@ -117,6 +117,16 @@ class ConnectManager: def delete_db(self, db_name: str): return self.storage.delete_db(db_name) + def edit_db(self, db_info: DBConfig): + return self.storage.update_db_info(db_info.db_name, + db_info.db_type, + db_info.file_path, + db_info.db_host, + db_info.db_port, + db_info.db_user, + db_info.db_pwd, + db_info.comment) + def add_db(self, db_info: DBConfig): db_type = DBType.of_db_type(db_info.db_type) if db_type.is_file_db(): diff --git a/pilot/openapi/api_v1/api_v1.py b/pilot/openapi/api_v1/api_v1.py index 7d3482dfc..76a04f819 100644 --- a/pilot/openapi/api_v1/api_v1.py +++ b/pilot/openapi/api_v1/api_v1.py @@ -25,7 +25,7 @@ from pilot.openapi.api_v1.api_view_model import ( MessageVo, ChatSceneVo, ) -from pilot.connections.db_conn_info import DBConfig +from pilot.connections.db_conn_info import DBConfig, DbTypeInfo from pilot.configs.config import Config from pilot.server.knowledge.service import KnowledgeService from pilot.server.knowledge.request.request import KnowledgeSpaceRequest @@ -35,7 +35,7 @@ from pilot.scene.base import ChatScene from pilot.scene.chat_factory import ChatFactory from pilot.configs.model_config import LOGDIR from pilot.utils import build_logger -from pilot.scene.base_message import BaseMessage +from pilot.common.schema import DBType from pilot.memory.chat_history.duckdb_history import DuckdbHistoryMemory from pilot.scene.message import OnceConversation @@ -106,14 +106,23 @@ async def dialogue_list(db_config: DBConfig = Body()): return Result.succ(CFG.LOCAL_DB_MANAGE.add_db(db_config)) +@router.post("/v1/chat/db/edit", response_model=Result[bool]) +async def dialogue_list(db_config: DBConfig = Body()): + return Result.succ(CFG.LOCAL_DB_MANAGE.edit_db(db_config)) + + @router.post("/v1/chat/db/delete", response_model=Result[bool]) async def dialogue_list(db_name: str = None): return Result.succ(CFG.LOCAL_DB_MANAGE.delete_db(db_name)) -@router.get("/v1/chat/db/support/type", response_model=Result[str]) +@router.get("/v1/chat/db/support/type", response_model=Result[DbTypeInfo]) async def db_support_types(): - return Result[str].succ(["mysql", "mssql", "duckdb"]) + support_types = [DBType.Mysql, DBType.MSSQL, DBType.DuckDb] + db_type_infos = [] + for type in support_types: + db_type_infos.append(DbTypeInfo(type.value(), type.is_file_db())) + return Result[DbTypeInfo].succ(db_type_infos) @router.get("/v1/chat/dialogue/list", response_model=Result[ConversationVo]) @@ -160,7 +169,7 @@ async def dialogue_scenes(): @router.post("/v1/chat/dialogue/new", response_model=Result[ConversationVo]) async def dialogue_new( - chat_mode: str = ChatScene.ChatNormal.value(), user_id: str = None + chat_mode: str = ChatScene.ChatNormal.value(), user_id: str = None ): conv_vo = __new_conversation(chat_mode, user_id) return Result.succ(conv_vo) diff --git a/pilot/scene/chat_data/__init__.py b/pilot/scene/chat_data/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/scene/chat_data/chat_excel/__init__.py b/pilot/scene/chat_data/chat_excel/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/scene/chat_data/chat_excel/excel_analyze/__init__.py b/pilot/scene/chat_data/chat_excel/excel_analyze/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pilot/scene/chat_data/chat_excel/excel_learning/__init__.py b/pilot/scene/chat_data/chat_excel/excel_learning/__init__.py new file mode 100644 index 000000000..e69de29bb From 208f01fda7627e5759e4458eb7484762dc3a1bbd Mon Sep 17 00:00:00 2001 From: aries_ckt <916701291@qq.com> Date: Mon, 31 Jul 2023 17:18:18 +0800 Subject: [PATCH 02/16] doc:add vector db use doc --- docs/faq.md | 23 +++++++++++++++++++++-- docs/modules/vector.rst | 4 +++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 852dbd109..3b7064311 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -82,12 +82,31 @@ mysql>CREATE TABLE `users` ( ) ENGINE=InnoDB AUTO_INCREMENT=101 DEFAULT CHARSET=utf8mb4 COMMENT='聊天用户表' ``` -##### Q6:When I use vicuna-13b, found some illegal character like this. +##### Q6:How to change Vector DB Type in DB-GPT. + +##### A6: Update .env file and set VECTOR_STORE_TYPE. +DB-GPT currently support Chroma(Default), Milvus(>2.1), Weaviate vector database. +If you want to change vector db, Update your .env, set your vector store type, VECTOR_STORE_TYPE=Chroma (now only support Chroma and Milvus(>2.1), if you set Milvus, please set MILVUS_URL and MILVUS_PORT) +If you want to support more vector db, you can integrate yourself.[how to integrate](https://db-gpt.readthedocs.io/en/latest/modules/vector.html) +```commandline +#*******************************************************************# +#** VECTOR STORE SETTINGS **# +#*******************************************************************# +VECTOR_STORE_TYPE=Chroma +#MILVUS_URL=127.0.0.1 +#MILVUS_PORT=19530 +#MILVUS_USERNAME +#MILVUS_PASSWORD +#MILVUS_SECURE= + +#WEAVIATE_URL=https://kt-region-m8hcy0wc.weaviate.network +``` +##### Q7:When I use vicuna-13b, found some illegal character like this.
-
+
-
+
-
+
diff --git a/README.zh.md b/README.zh.md
index 08f203735..043280ae3 100644
--- a/README.zh.md
+++ b/README.zh.md
@@ -22,7 +22,7 @@
-
+
@@ -30,7 +30,7 @@