From 440a3e955f11b961b3d6ca566d798aed0272d06d Mon Sep 17 00:00:00 2001
From: yhjun1026 <460342015@qq.com>
Date: Fri, 10 Nov 2023 14:20:56 +0800
Subject: [PATCH] bugfix(ChatData): ChatData Use AntV Table
1.Merge ChatData and ChatDB
---
pilot/openapi/api_v1/editor/api_editor_v1.py | 36 +++++-------
.../scene/chat_db/auto_execute/out_parser.py | 8 +--
pilot/scene/chat_db/data_loader.py | 56 +++++++++++++++----
3 files changed, 63 insertions(+), 37 deletions(-)
diff --git a/pilot/openapi/api_v1/editor/api_editor_v1.py b/pilot/openapi/api_v1/editor/api_editor_v1.py
index e491a01b6..799d1d6f9 100644
--- a/pilot/openapi/api_v1/editor/api_editor_v1.py
+++ b/pilot/openapi/api_v1/editor/api_editor_v1.py
@@ -29,8 +29,6 @@ from pilot.scene.message import OnceConversation
from pilot.scene.chat_dashboard.data_loader import DashboardDataLoader
from pilot.scene.chat_db.data_loader import DbDataLoader
from pilot.memory.chat_history.chat_hisotry_factory import ChatHistory
-from pilot.base_modules.agent.commands.command_mange import ApiCall
-
router = APIRouter()
CFG = Config()
@@ -103,16 +101,12 @@ async def get_editor_sql(con_uid: str, round: int):
logger.info(
f'history ai json resp:{element["data"]["content"]}'
)
- api_call = ApiCall()
- api_call.update_from_context(element["data"]["content"])
- result = {}
- result['thoughts'] = element["data"]["content"].replace("\n", " ").replace("\\n", " ")
- if api_call.check_last_plugin_call_ready(element["data"]["content"]):
-
- if len(api_call.plugin_status_map) > 0:
- first_item = next(iter(api_call.plugin_status_map.items()))[1]
- result['sql'] = first_item.args["sql"]
- return Result.succ(result)
+ context = (
+ element["data"]["content"]
+ .replace("\\n", " ")
+ .replace("\n", " ")
+ )
+ return Result.succ(json.loads(context))
return Result.faild(msg="not have sql!")
@@ -140,6 +134,7 @@ async def editor_sql_run(run_param: dict = Body()):
)
return Result.succ(sql_run_data)
except Exception as e:
+ logging.error("editor_sql_run exception!" + str(e))
return Result.succ(
SqlRunData(result_info=str(e), run_cost=0, colunms=[], values=[])
)
@@ -162,18 +157,17 @@ async def sql_editor_submit(sql_edit_context: ChatSqlEditContext = Body()):
)
)[0]
if edit_round:
- new_ai_text = ""
for element in edit_round["messages"]:
if element["type"] == "ai":
- new_ai_text = element["data"]["content"]
- new_ai_text.replace(sql_edit_context.old_sql, sql_edit_context.new_sql)
- element["data"]["content"] = new_ai_text
-
- for element in edit_round["messages"]:
+ db_resp = json.loads(element["data"]["content"])
+ db_resp["thoughts"] = sql_edit_context.new_speak
+ db_resp["sql"] = sql_edit_context.new_sql
+ element["data"]["content"] = json.dumps(db_resp)
if element["type"] == "view":
- api_call = ApiCall()
- new_view_text = api_call.run_display_sql(new_ai_text, conn.run_to_df)
- element["data"]["content"] = new_view_text
+ data_loader = DbDataLoader()
+ element["data"]["content"] = data_loader.get_table_view_by_conn(
+ conn.run_to_df(sql_edit_context.new_sql), sql_edit_context.new_speak
+ )
history_mem.update(history_messages)
return Result.succ(None)
return Result.faild(msg="Edit Faild!")
diff --git a/pilot/scene/chat_db/auto_execute/out_parser.py b/pilot/scene/chat_db/auto_execute/out_parser.py
index e49d45d65..e39ad8fe9 100644
--- a/pilot/scene/chat_db/auto_execute/out_parser.py
+++ b/pilot/scene/chat_db/auto_execute/out_parser.py
@@ -24,7 +24,7 @@ class DbChatOutputParser(BaseOutputParser):
def parse_prompt_response(self, model_out_text):
clean_str = super().parse_prompt_response(model_out_text)
- print("clean prompt response:", clean_str)
+ logging.info("clean prompt response:", clean_str)
response = json.loads(clean_str)
for key in sorted(response):
if key.strip() == "sql":
@@ -43,16 +43,16 @@ class DbChatOutputParser(BaseOutputParser):
param["type"] = "response_table"
param["sql"] = prompt_response.sql
param["data"] = json.loads(df.to_json(orient='records', date_format='iso', date_unit='s'))
- view_json_str = json.dumps(param, default=serialize)
+ view_json_str = json.dumps(param, default=serialize, ensure_ascii=False)
except Exception as e:
logger.error("parse_view_response error!" + str(e))
err_param = {}
err_param["sql"] = f'{prompt_response.sql}'
err_param["type"] = "response_table"
- err_param["err_msg"] = str(e)
+ # err_param["err_msg"] = str(e)
err_param["data"] = []
err_msg = str(e)
- view_json_str = json.dumps(err_param, default=serialize)
+ view_json_str = json.dumps(err_param, default=serialize, ensure_ascii=False)
api_call_element.text = view_json_str
result = ET.tostring(api_call_element, encoding="utf-8")
diff --git a/pilot/scene/chat_db/data_loader.py b/pilot/scene/chat_db/data_loader.py
index a9e4450f5..cbfc45ba7 100644
--- a/pilot/scene/chat_db/data_loader.py
+++ b/pilot/scene/chat_db/data_loader.py
@@ -1,13 +1,45 @@
-class DbDataLoader:
- def get_table_view_by_conn(self, data, speak):
- import pandas as pd
+import xml.etree.ElementTree as ET
+import json
+import logging
- ### tool out data to table view
- if len(data) < 1:
- data.insert(0, ["result"])
- df = pd.DataFrame(data[1:], columns=data[0])
- html_table = df.to_html(index=False, escape=False, sparsify=False)
- table_str = "".join(html_table.split())
- html = f"""{table_str}
"""
- view_text = f"##### {str(speak)}" + "\n" + html.replace("\n", " ")
- return view_text
+from pilot.common.json_utils import serialize
+
+class DbDataLoader:
+ def get_table_view_by_conn(self, data, speak, sql:str=None):
+ # import pandas as pd
+ #
+ # ### tool out data to table view
+ # if len(data) < 1:
+ # data.insert(0, ["result"])
+ # df = pd.DataFrame(data[1:], columns=data[0])
+ # html_table = df.to_html(index=False, escape=False, sparsify=False)
+ # table_str = "".join(html_table.split())
+ # html = f"""{table_str}
"""
+ # view_text = f"##### {str(speak)}" + "\n" + html.replace("\n", " ")
+ # return view_text
+
+ param = {}
+ api_call_element = ET.Element("chart-view")
+ err_msg = None
+ try:
+
+ param["type"] = "response_table"
+ param["sql"] = sql
+ param["data"] = json.loads(data.to_json(orient='records', date_format='iso', date_unit='s'))
+ view_json_str = json.dumps(param, default=serialize, ensure_ascii=False)
+ except Exception as e:
+ logging.error("parse_view_response error!" + str(e))
+ err_param = {}
+ err_param["sql"] = f'{sql}'
+ err_param["type"] = "response_table"
+ err_param["err_msg"] = str(e)
+ err_param["data"] = []
+ err_msg = str(e)
+ view_json_str = json.dumps(err_param, default=serialize, ensure_ascii=False)
+
+ api_call_element.text = view_json_str
+ result = ET.tostring(api_call_element, encoding="utf-8")
+ if err_msg:
+ return f"""{speak} \\n ERROR!{err_msg} \n {result.decode("utf-8")}"""
+ else:
+ return speak + "\n" + result.decode("utf-8")
\ No newline at end of file