fix(ChatExcel): ChatExcel OutParse Bug Fix

1.ChatExcel OutParse Bug Fix 2

pilot/openapi/api_v1/api_v1.py

@@ -251,6 +251,7 @@ async def params_load(
         ### refresh messages
         return Result.succ(get_hist_messages(conv_uid))
     except Exception as e:
+        logger.error("excel load error!", e)
         return Result.faild(code="E000X", msg=f"File Load Error {e}")
 
 

pilot/out_parser/base.py

@@ -207,7 +207,7 @@ class BaseOutputParser(ABC):
             cleaned_output.strip()
             .replace("\\n", " ")
             .replace("\n", " ")
-            .replace("\\", " ")
+            # .replace("\\", " ")
         )
         cleaned_output = self.__illegal_json_ends(cleaned_output)
         return cleaned_output

pilot/scene/chat_dashboard/out_parser.py

@@ -30,7 +30,7 @@ class ChatDashboardOutputParser(BaseOutputParser):
         for item in response:
             chart_items.append(
                 ChartItem(
-                    item["sql"], item["title"], item["thoughts"], item["showcase"]
+                    item["sql"].replace("\\", " "), item["title"], item["thoughts"], item["showcase"]
                 )
             )
         return chart_items

pilot/scene/chat_data/chat_excel/excel_analyze/out_parser.py

@@ -31,14 +31,14 @@ class ChatExcelOutputParser(BaseOutputParser):
             response = json.loads(clean_str)
             for key in sorted(response):
                 if key.strip() == "sql":
-                    sql = response[key]
+                    sql = response[key].replace("\\", " ")
                 if key.strip() == "thoughts":
                     thoughts = response[key]
                 if key.strip() == "display":
                     display = response[key]
             return ExcelAnalyzeResponse(sql, thoughts, display)
         except Exception as e:
-            raise ValueError(f"LLM Response Can't Parser! \n{ model_out_text}")
+            raise ValueError(f"LLM Response Can't Parser! \n")
 
     def parse_view_response(self, speak, data) -> str:
         ### tool out data to table view

pilot/scene/chat_data/chat_excel/excel_reader.py

@@ -14,6 +14,14 @@ def excel_colunm_format(old_name: str) -> str:
     return new_column
 
 
+def add_quotes_ex(sql: str, column_names):
+    sql = sql.replace("`", '"')
+    for column_name in column_names:
+        if sql.find(column_name) != -1 and sql.find(f'"{column_name}"') == -1:
+            sql = sql.replace(column_name, f'"{column_name}"')
+    return sql
+
+
 def add_quotes(sql, column_names=[]):
     sql = sql.replace("`", "")
     parsed = sqlparse.parse(sql)
@@ -28,12 +36,18 @@ def deep_quotes(token, column_names=[]):
         for token_child in token.tokens:
             deep_quotes(token_child, column_names)
     else:
-        if token.ttype == sqlparse.tokens.Name:
-            if len(column_names) > 0:
-                if token.value in column_names:
-                    token.value = f'"{token.value.replace("`", "")}"'
-            else:
-                token.value = f'"{token.value.replace("`", "")}"'
+        if token.value in column_names:
+            token.value = f'"{token.value.replace("`", "")}"'
+        elif token.ttype == sqlparse.tokens.Name:
+            token.value = f'"{token.value.replace("`", "")}"'
+
+
+if __name__ == "__main__":
+    sql = "SELECT `地区`, (`2021年人口` - `2001年人口`) / `2001年人口` * 100 AS `Population_Growth_Rate` FROM Generated_by_ChatExcel_table1 (2)"
+    if f'"Generated_by_ChatExcel_table1 (2)"' not in sql:
+        sql = sql.replace('Generated_by_ChatExcel_table1 (2)', f'"Generated_by_ChatExcel_table1 (2)"')
+    sql = add_quotes_ex(sql, ['地区', '地区代码', '2001年人口', '2006年人口', '2011年人口', '2016年人口', '2021年人口'])
+    print(f"excute sql:{sql}")
 
 
 def is_chinese(string):
@@ -81,14 +95,14 @@ class ExcelReader:
         # connect DuckDB
         self.db = duckdb.connect(database=":memory:", read_only=False)
 
-        self.table_name = file_name_without_extension
+        self.table_name = "excel_data"
         # write data in duckdb
         self.db.register(self.table_name, self.df)
 
     def run(self, sql):
         if f'"{self.table_name}"' not in sql:
             sql = sql.replace(self.table_name, f'"{self.table_name}"')
-        sql = add_quotes(sql, self.columns_map.values())
+        sql = add_quotes_ex(sql, self.columns_map.values())
         print(f"excute sql:{sql}")
         results = self.db.execute(sql)
         colunms = []

pilot/scene/chat_db/auto_execute/out_parser.py

@@ -27,7 +27,7 @@ class DbChatOutputParser(BaseOutputParser):
         response = json.loads(clean_str)
         for key in sorted(response):
             if key.strip() == "sql":
-                sql = response[key]
+                sql = response[key].replace("\\", " ")
             if key.strip() == "thoughts":
                 thoughts = response[key]
         return SqlAction(sql, thoughts)

pilot/server/static/_next/static/QgSLvD5Yu8RZmuZsIFsfR/_buildManifest.js

@@ -1 +0,0 @@
-self.__BUILD_MANIFEST=function(s,a,c,t,e,d,n,u,i,b){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":[a,"static/chunks/673-6b91681955aa1094.js","static/chunks/pages/index-704f93ece0dc096f.js"],"/_error":["static/chunks/pages/_error-dee72aff9b2e2c12.js"],"/chat":["static/chunks/pages/chat-03c8b0d90000e1eb.js"],"/database":[s,c,d,t,"static/chunks/566-cb742dc279bbfe42.js","static/chunks/892-c40dbe9ae037747a.js","static/chunks/pages/database-e3f640bde30c17d7.js"],"/datastores":[e,s,a,n,u,"static/chunks/241-4117dd68a591b7fa.js","static/chunks/pages/datastores-82628b0273c874d4.js"],"/datastores/documents":[e,"static/chunks/75fc9c18-a784766a129ec5fb.js",s,a,n,c,i,d,t,b,u,"static/chunks/749-f876c99e30a851b8.js","static/chunks/pages/datastores/documents-dea98f7e09e362f9.js"],"/datastores/documents/chunklist":[e,s,c,i,t,b,"static/chunks/pages/datastores/documents/chunklist-87676458d42e378f.js"],sortedPages:["/","/_app","/_error","/chat","/database","/datastores","/datastores/documents","/datastores/documents/chunklist"]}}("static/chunks/215-ed2d98cfbd44ae81.js","static/chunks/913-e3ab2daf183d352e.js","static/chunks/542-f4cda9df864aa7ed.js","static/chunks/378-dbd26a0c14558f18.js","static/chunks/29107295-90b90cb30c825230.js","static/chunks/908-d76aabcc43706d37.js","static/chunks/718-8b4a2d7a281bb0c4.js","static/chunks/589-8dfb35868cafc00b.js","static/chunks/289-06c0d9f538f77a71.js","static/chunks/34-52d4d2d11bef48dc.js"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();

pilot/server/static/_next/static/chunks/pages/datastores/documents/chunklist-4ae606926d192018.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[41],{13264:function(e,t,n){"use strict";var r=n(70182);let a=(0,r.ZP)();t.Z=a},57838:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(67294);function a(){let[,e]=r.useReducer(e=>e+1,0);return e}},53116:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/datastores/documents/chunklist",function(){return n(49114)}])},49114:function(e,t,n){"use strict";n.r(t);var r=n(85893),a=n(39332),c=n(67294),s=n(56385),i=n(48665),o=n(70702),l=n(84229),d=n(2166),u=n(40911),h=n(61685),f=n(74627),g=n(60122),j=n(30119),m=n(67421);t.default=()=>{let e=(0,a.useRouter)(),{mode:t}=(0,s.tv)(),n=(0,a.useSearchParams)(),p=n&&n.get("spacename"),x=n&&n.get("documentid"),[_,Z]=(0,c.useState)(0),[w,P]=(0,c.useState)(0),[b,k]=(0,c.useState)([]),{t:v}=(0,m.$G)();return(0,c.useEffect)(()=>{(async function(){let e=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:1,page_size:20});e.success&&(k(e.data.data),Z(e.data.total),P(e.data.page))})()},[]),(0,r.jsxs)(i.Z,{className:"p-4 h-[90%]",children:[(0,r.jsx)(o.Z,{className:"mb-5",direction:"row",justifyContent:"flex-start",alignItems:"center",children:(0,r.jsxs)(l.Z,{"aria-label":"breadcrumbs",children:[(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores")},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Knowledge_Space")},"Knowledge Space"),(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores/documents?name=".concat(p))},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Documents")},"Knowledge Space"),(0,r.jsx)(u.ZP,{fontSize:"inherit",children:v("Chunks")})]})}),(0,r.jsx)(i.Z,{className:"p-4 overflow-auto h-[90%]",sx:{"&::-webkit-scrollbar":{display:"none"}},children:b.length?(0,r.jsx)(r.Fragment,{children:(0,r.jsxs)(h.Z,{color:"primary",variant:"plain",size:"lg",sx:{"& tbody tr: hover":{backgroundColor:"light"===t?"rgb(246, 246, 246)":"rgb(33, 33, 40)"},"& tbody tr: hover a":{textDecoration:"underline"}},children:[(0,r.jsx)("thead",{children:(0,r.jsxs)("tr",{children:[(0,r.jsx)("th",{children:v("Name")}),(0,r.jsx)("th",{children:v("Content")}),(0,r.jsx)("th",{children:v("Meta_Data")})]})}),(0,r.jsx)("tbody",{children:b.map(e=>(0,r.jsxs)("tr",{children:[(0,r.jsx)("td",{children:e.doc_name}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:e.content,trigger:"hover",children:e.content.length>10?"".concat(e.content.slice(0,10),"..."):e.content})}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:JSON.stringify(e.meta_info||"{}",null,2),trigger:"hover",children:e.meta_info.length>10?"".concat(e.meta_info.slice(0,10),"..."):e.meta_info})})]},e.id))})]})}):(0,r.jsx)(r.Fragment,{})}),(0,r.jsx)(o.Z,{className:"mt-5",direction:"row",justifyContent:"flex-end",children:(0,r.jsx)(g.Z,{defaultPageSize:20,showSizeChanger:!1,current:w,total:_,onChange:async e=>{let t=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:e,page_size:20});t.success&&(k(t.data.data),Z(t.data.total),P(t.data.page))},hideOnSinglePage:!0})})]})}},30119:function(e,t,n){"use strict";n.d(t,{Tk:function(){return o},PR:function(){return l},Ej:function(){return d}});var r=n(27790),a=n(6154),c=n(83454);let s=a.Z.create({baseURL:c.env.API_BASE_URL});s.defaults.timeout=1e4,s.interceptors.response.use(e=>e.data,e=>Promise.reject(e)),n(96486);let i={"content-type":"application/json"},o=(e,t)=>{if(t){let n=Object.keys(t).filter(e=>void 0!==t[e]&&""!==t[e]).map(e=>"".concat(e,"=").concat(t[e])).join("&");n&&(e+="?".concat(n))}return s.get("/api"+e,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},l=(e,t)=>s.post(e,t,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)}),d=(e,t)=>s.post(e,t).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})}},function(e){e.O(0,[662,215,542,289,378,34,774,888,179],function(){return e(e.s=53116)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[41],{13264:function(e,t,n){"use strict";var r=n(70182);let a=(0,r.ZP)();t.Z=a},57838:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(67294);function a(){let[,e]=r.useReducer(e=>e+1,0);return e}},53116:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/datastores/documents/chunklist",function(){return n(49114)}])},49114:function(e,t,n){"use strict";n.r(t);var r=n(85893),a=n(39332),c=n(67294),s=n(56385),i=n(48665),o=n(70702),l=n(84229),d=n(2166),u=n(40911),h=n(61685),f=n(74627),g=n(60122),j=n(30119),m=n(67421);t.default=()=>{let e=(0,a.useRouter)(),{mode:t}=(0,s.tv)(),n=(0,a.useSearchParams)(),p=n&&n.get("spacename"),x=n&&n.get("documentid"),[_,Z]=(0,c.useState)(0),[w,P]=(0,c.useState)(0),[b,k]=(0,c.useState)([]),{t:v}=(0,m.$G)();return(0,c.useEffect)(()=>{(async function(){let e=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:1,page_size:20});e.success&&(k(e.data.data),Z(e.data.total),P(e.data.page))})()},[]),(0,r.jsxs)(i.Z,{className:"p-4 h-[90%]",children:[(0,r.jsx)(o.Z,{className:"mb-5",direction:"row",justifyContent:"flex-start",alignItems:"center",children:(0,r.jsxs)(l.Z,{"aria-label":"breadcrumbs",children:[(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores")},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Knowledge_Space")},"Knowledge Space"),(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores/documents?name=".concat(p))},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Documents")},"Knowledge Space"),(0,r.jsx)(u.ZP,{fontSize:"inherit",children:v("Chunks")})]})}),(0,r.jsx)(i.Z,{className:"p-4 overflow-auto h-[90%]",sx:{"&::-webkit-scrollbar":{display:"none"}},children:b.length?(0,r.jsx)(r.Fragment,{children:(0,r.jsxs)(h.Z,{color:"primary",variant:"plain",size:"lg",sx:{"& tbody tr: hover":{backgroundColor:"light"===t?"rgb(246, 246, 246)":"rgb(33, 33, 40)"},"& tbody tr: hover a":{textDecoration:"underline"}},children:[(0,r.jsx)("thead",{children:(0,r.jsxs)("tr",{children:[(0,r.jsx)("th",{children:v("Name")}),(0,r.jsx)("th",{children:v("Content")}),(0,r.jsx)("th",{children:v("Meta_Data")})]})}),(0,r.jsx)("tbody",{children:b.map(e=>(0,r.jsxs)("tr",{children:[(0,r.jsx)("td",{children:e.doc_name}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:e.content,trigger:"hover",children:e.content.length>10?"".concat(e.content.slice(0,10),"..."):e.content})}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:JSON.stringify(e.meta_info||"{}",null,2),trigger:"hover",children:e.meta_info.length>10?"".concat(e.meta_info.slice(0,10),"..."):e.meta_info})})]},e.id))})]})}):(0,r.jsx)(r.Fragment,{})}),(0,r.jsx)(o.Z,{className:"mt-5",direction:"row",justifyContent:"flex-end",children:(0,r.jsx)(g.Z,{defaultPageSize:20,showSizeChanger:!1,current:w,total:_,onChange:async e=>{let t=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:e,page_size:20});t.success&&(k(t.data.data),Z(t.data.total),P(t.data.page))},hideOnSinglePage:!0})})]})}},30119:function(e,t,n){"use strict";n.d(t,{Tk:function(){return o},PR:function(){return l},Ej:function(){return d}});var r=n(2453),a=n(6154),c=n(83454);let s=a.Z.create({baseURL:c.env.API_BASE_URL});s.defaults.timeout=1e4,s.interceptors.response.use(e=>e.data,e=>Promise.reject(e)),n(96486);let i={"content-type":"application/json"},o=(e,t)=>{if(t){let n=Object.keys(t).filter(e=>void 0!==t[e]&&""!==t[e]).map(e=>"".concat(e,"=").concat(t[e])).join("&");n&&(e+="?".concat(n))}return s.get("/api"+e,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},l=(e,t)=>s.post(e,t,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)}),d=(e,t)=>s.post(e,t).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})}},function(e){e.O(0,[662,566,902,455,289,34,774,888,179],function(){return e(e.s=53116)}),_N_E=e.O()}]);

pilot/server/static/_next/static/ePgK32jR2MnJALKopoumE/_buildManifest.js

@@ -0,0 +1 @@
+self.__BUILD_MANIFEST=function(s,a,t,c,e,d,f,n,u,i,b){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":[a,"static/chunks/66-791bb03098dc9265.js","static/chunks/707-109d4fec9e26030d.js","static/chunks/pages/index-d5aba6bbbc1d8aaa.js"],"/_error":["static/chunks/pages/_error-dee72aff9b2e2c12.js"],"/chat":["static/chunks/pages/chat-a9adfc18f61cb676.js"],"/database":[s,t,d,c,"static/chunks/46-2a716444a56f6f08.js","static/chunks/847-4335b5938375e331.js","static/chunks/pages/database-ddf0a72485646c52.js"],"/datastores":[e,s,f,a,n,u,"static/chunks/241-4117dd68a591b7fa.js","static/chunks/pages/datastores-4fb48131988df037.js"],"/datastores/documents":[e,"static/chunks/75fc9c18-a784766a129ec5fb.js",s,f,t,a,d,c,n,i,b,u,"static/chunks/749-f876c99e30a851b8.js","static/chunks/pages/datastores/documents-7312ed2d9409617f.js"],"/datastores/documents/chunklist":[e,s,t,c,i,b,"static/chunks/pages/datastores/documents/chunklist-4ae606926d192018.js"],sortedPages:["/","/_app","/_error","/chat","/database","/datastores","/datastores/documents","/datastores/documents/chunklist"]}}("static/chunks/566-31b5bf29f3e84615.js","static/chunks/913-b5bc9815149e2ad5.js","static/chunks/902-c56acea399c45e57.js","static/chunks/455-5c8f2c8bda9b4b83.js","static/chunks/29107295-90b90cb30c825230.js","static/chunks/625-63aa85328eed0b3e.js","static/chunks/556-26ffce13383f774a.js","static/chunks/939-126a01b0d827f3b4.js","static/chunks/589-8dfb35868cafc00b.js","static/chunks/289-06c0d9f538f77a71.js","static/chunks/34-4756f8547fff0eaf.js"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();