fix(ChatExcel): ChatExcel OutParse Bug Fix (#588)

1.ChatExcel OutParse Bug Fix

pilot/out_parser/base.py

@@ -129,44 +129,55 @@ class BaseOutputParser(ABC):
         return temp_json
 
     def __extract_json(self, s):
-        temp_json = self.__json_interception(s, True)
-        if not temp_json:
-            temp_json = self.__json_interception(s)
         try:
+            # Get the dual-mode analysis first and get the maximum result
+            temp_json_simple = self.__json_interception(s)
+            temp_json_array = self.__json_interception(s, True)
+            if len(temp_json_simple) > len(temp_json_array):
+                temp_json = temp_json_simple
+            else:
+                temp_json = temp_json_array
+
+            if not temp_json:
+                temp_json = self.__json_interception(s)
+
             temp_json = self.__illegal_json_ends(temp_json)
             return temp_json
         except Exception as e:
-            raise ValueError("Failed to find a valid json response！" + temp_json)
+            raise ValueError("Failed to find a valid json in LLM response！" + temp_json)
 
     def __json_interception(self, s, is_json_array: bool = False):
-        if is_json_array:
-            i = s.find("[")
-            if i < 0:
-                return None
-            count = 1
-            for j, c in enumerate(s[i + 1 :], start=i + 1):
-                if c == "]":
-                    count -= 1
-                elif c == "[":
-                    count += 1
-                if count == 0:
-                    break
-            assert count == 0
-            return s[i : j + 1]
-        else:
-            i = s.find("{")
-            if i < 0:
-                return None
-            count = 1
-            for j, c in enumerate(s[i + 1 :], start=i + 1):
-                if c == "}":
-                    count -= 1
-                elif c == "{":
-                    count += 1
-                if count == 0:
-                    break
-            assert count == 0
-            return s[i : j + 1]
+        try:
+            if is_json_array:
+                i = s.find("[")
+                if i < 0:
+                    return ""
+                count = 1
+                for j, c in enumerate(s[i + 1 :], start=i + 1):
+                    if c == "]":
+                        count -= 1
+                    elif c == "[":
+                        count += 1
+                    if count == 0:
+                        break
+                assert count == 0
+                return s[i : j + 1]
+            else:
+                i = s.find("{")
+                if i < 0:
+                    return ""
+                count = 1
+                for j, c in enumerate(s[i + 1 :], start=i + 1):
+                    if c == "}":
+                        count -= 1
+                    elif c == "{":
+                        count += 1
+                    if count == 0:
+                        break
+                assert count == 0
+                return s[i : j + 1]
+        except Exception as e:
+            return ""
 
     def parse_prompt_response(self, model_out_text) -> T:
         """

pilot/scene/chat_data/chat_excel/excel_analyze/out_parser.py

@@ -27,15 +27,18 @@ class ChatExcelOutputParser(BaseOutputParser):
     def parse_prompt_response(self, model_out_text):
         clean_str = super().parse_prompt_response(model_out_text)
         print("clean prompt response:", clean_str)
-        response = json.loads(clean_str)
-        for key in sorted(response):
-            if key.strip() == "sql":
-                sql = response[key]
-            if key.strip() == "thoughts":
-                thoughts = response[key]
-            if key.strip() == "display":
-                display = response[key]
-        return ExcelAnalyzeResponse(sql, thoughts, display)
+        try:
+            response = json.loads(clean_str)
+            for key in sorted(response):
+                if key.strip() == "sql":
+                    sql = response[key]
+                if key.strip() == "thoughts":
+                    thoughts = response[key]
+                if key.strip() == "display":
+                    display = response[key]
+            return ExcelAnalyzeResponse(sql, thoughts, display)
+        except Exception as e:
+            raise ValueError(f"LLM Response Can't Parser! \n{ model_out_text}")
 
     def parse_view_response(self, speak, data) -> str:
         ### tool out data to table view

pilot/scene/chat_data/chat_excel/excel_learning/out_parser.py

@@ -41,7 +41,7 @@ class LearningExcelOutputParser(BaseOutputParser):
             return model_out_text
 
     def parse_view_response(self, speak, data) -> str:
-        if data:
+        if data and not isinstance(data, str):
             ### tool out data to table view
             html_title = f"### **Data Summary**\n{data.desciption} "
             html_colunms = f"### **Data Structure**\n"

pilot/server/static/_next/static/QgSLvD5Yu8RZmuZsIFsfR/_buildManifest.js

@@ -0,0 +1 @@
+self.__BUILD_MANIFEST=function(s,a,c,t,e,d,n,u,i,b){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":[a,"static/chunks/673-6b91681955aa1094.js","static/chunks/pages/index-704f93ece0dc096f.js"],"/_error":["static/chunks/pages/_error-dee72aff9b2e2c12.js"],"/chat":["static/chunks/pages/chat-03c8b0d90000e1eb.js"],"/database":[s,c,d,t,"static/chunks/566-cb742dc279bbfe42.js","static/chunks/892-c40dbe9ae037747a.js","static/chunks/pages/database-e3f640bde30c17d7.js"],"/datastores":[e,s,a,n,u,"static/chunks/241-4117dd68a591b7fa.js","static/chunks/pages/datastores-82628b0273c874d4.js"],"/datastores/documents":[e,"static/chunks/75fc9c18-a784766a129ec5fb.js",s,a,n,c,i,d,t,b,u,"static/chunks/749-f876c99e30a851b8.js","static/chunks/pages/datastores/documents-dea98f7e09e362f9.js"],"/datastores/documents/chunklist":[e,s,c,i,t,b,"static/chunks/pages/datastores/documents/chunklist-87676458d42e378f.js"],sortedPages:["/","/_app","/_error","/chat","/database","/datastores","/datastores/documents","/datastores/documents/chunklist"]}}("static/chunks/215-ed2d98cfbd44ae81.js","static/chunks/913-e3ab2daf183d352e.js","static/chunks/542-f4cda9df864aa7ed.js","static/chunks/378-dbd26a0c14558f18.js","static/chunks/29107295-90b90cb30c825230.js","static/chunks/908-d76aabcc43706d37.js","static/chunks/718-8b4a2d7a281bb0c4.js","static/chunks/589-8dfb35868cafc00b.js","static/chunks/289-06c0d9f538f77a71.js","static/chunks/34-52d4d2d11bef48dc.js"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();

pilot/server/static/_next/static/chunks/pages/datastores/documents/chunklist-87676458d42e378f.js

@@ -1 +1 @@
-(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[41],{13264:function(e,t,n){"use strict";var r=n(70182);let a=(0,r.ZP)();t.Z=a},57838:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(67294);function a(){let[,e]=r.useReducer(e=>e+1,0);return e}},53116:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/datastores/documents/chunklist",function(){return n(49114)}])},49114:function(e,t,n){"use strict";n.r(t);var r=n(85893),a=n(39332),c=n(67294),s=n(56385),i=n(48665),o=n(70702),l=n(84229),d=n(2166),u=n(40911),h=n(61685),f=n(74627),g=n(60122),j=n(30119),m=n(67421);t.default=()=>{let e=(0,a.useRouter)(),{mode:t}=(0,s.tv)(),n=(0,a.useSearchParams)(),p=n&&n.get("spacename"),x=n&&n.get("documentid"),[_,Z]=(0,c.useState)(0),[w,P]=(0,c.useState)(0),[b,k]=(0,c.useState)([]),{t:v}=(0,m.$G)();return(0,c.useEffect)(()=>{(async function(){let e=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:1,page_size:20});e.success&&(k(e.data.data),Z(e.data.total),P(e.data.page))})()},[]),(0,r.jsxs)(i.Z,{className:"p-4 h-[90%]",children:[(0,r.jsx)(o.Z,{className:"mb-5",direction:"row",justifyContent:"flex-start",alignItems:"center",children:(0,r.jsxs)(l.Z,{"aria-label":"breadcrumbs",children:[(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores")},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Knowledge_Space")},"Knowledge Space"),(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores/documents?name=".concat(p))},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Documents")},"Knowledge Space"),(0,r.jsx)(u.ZP,{fontSize:"inherit",children:v("Chunks")})]})}),(0,r.jsx)(i.Z,{className:"p-4 overflow-auto h-[90%]",sx:{"&::-webkit-scrollbar":{display:"none"}},children:b.length?(0,r.jsx)(r.Fragment,{children:(0,r.jsxs)(h.Z,{color:"primary",variant:"plain",size:"lg",sx:{"& tbody tr: hover":{backgroundColor:"light"===t?"rgb(246, 246, 246)":"rgb(33, 33, 40)"},"& tbody tr: hover a":{textDecoration:"underline"}},children:[(0,r.jsx)("thead",{children:(0,r.jsxs)("tr",{children:[(0,r.jsx)("th",{children:v("Name")}),(0,r.jsx)("th",{children:v("Content")}),(0,r.jsx)("th",{children:v("Meta_Data")})]})}),(0,r.jsx)("tbody",{children:b.map(e=>(0,r.jsxs)("tr",{children:[(0,r.jsx)("td",{children:e.doc_name}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:e.content,trigger:"hover",children:e.content.length>10?"".concat(e.content.slice(0,10),"..."):e.content})}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:JSON.stringify(e.meta_info||"{}",null,2),trigger:"hover",children:e.meta_info.length>10?"".concat(e.meta_info.slice(0,10),"..."):e.meta_info})})]},e.id))})]})}):(0,r.jsx)(r.Fragment,{})}),(0,r.jsx)(o.Z,{className:"mt-5",direction:"row",justifyContent:"flex-end",children:(0,r.jsx)(g.Z,{defaultPageSize:20,showSizeChanger:!1,current:w,total:_,onChange:async e=>{let t=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:e,page_size:20});t.success&&(k(t.data.data),Z(t.data.total),P(t.data.page))},hideOnSinglePage:!0})})]})}},30119:function(e,t,n){"use strict";n.d(t,{Tk:function(){return o},PR:function(){return l},Ej:function(){return d}});var r=n(58301),a=n(6154),c=n(83454);let s=a.Z.create({baseURL:c.env.API_BASE_URL});s.defaults.timeout=1e4,s.interceptors.response.use(e=>e.data,e=>Promise.reject(e)),n(96486);let i={"content-type":"application/json"},o=(e,t)=>{if(t){let n=Object.keys(t).filter(e=>void 0!==t[e]&&""!==t[e]).map(e=>"".concat(e,"=").concat(t[e])).join("&");n&&(e+="?".concat(n))}return s.get("/api"+e,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},l=(e,t)=>s.post(e,t,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)}),d=(e,t)=>s.post(e,t).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})}},function(e){e.O(0,[662,215,902,289,455,34,774,888,179],function(){return e(e.s=53116)}),_N_E=e.O()}]);
+(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[41],{13264:function(e,t,n){"use strict";var r=n(70182);let a=(0,r.ZP)();t.Z=a},57838:function(e,t,n){"use strict";n.d(t,{Z:function(){return a}});var r=n(67294);function a(){let[,e]=r.useReducer(e=>e+1,0);return e}},53116:function(e,t,n){(window.__NEXT_P=window.__NEXT_P||[]).push(["/datastores/documents/chunklist",function(){return n(49114)}])},49114:function(e,t,n){"use strict";n.r(t);var r=n(85893),a=n(39332),c=n(67294),s=n(56385),i=n(48665),o=n(70702),l=n(84229),d=n(2166),u=n(40911),h=n(61685),f=n(74627),g=n(60122),j=n(30119),m=n(67421);t.default=()=>{let e=(0,a.useRouter)(),{mode:t}=(0,s.tv)(),n=(0,a.useSearchParams)(),p=n&&n.get("spacename"),x=n&&n.get("documentid"),[_,Z]=(0,c.useState)(0),[w,P]=(0,c.useState)(0),[b,k]=(0,c.useState)([]),{t:v}=(0,m.$G)();return(0,c.useEffect)(()=>{(async function(){let e=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:1,page_size:20});e.success&&(k(e.data.data),Z(e.data.total),P(e.data.page))})()},[]),(0,r.jsxs)(i.Z,{className:"p-4 h-[90%]",children:[(0,r.jsx)(o.Z,{className:"mb-5",direction:"row",justifyContent:"flex-start",alignItems:"center",children:(0,r.jsxs)(l.Z,{"aria-label":"breadcrumbs",children:[(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores")},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Knowledge_Space")},"Knowledge Space"),(0,r.jsx)(d.Z,{onClick:()=>{e.push("/datastores/documents?name=".concat(p))},underline:"hover",color:"neutral",fontSize:"inherit",children:v("Documents")},"Knowledge Space"),(0,r.jsx)(u.ZP,{fontSize:"inherit",children:v("Chunks")})]})}),(0,r.jsx)(i.Z,{className:"p-4 overflow-auto h-[90%]",sx:{"&::-webkit-scrollbar":{display:"none"}},children:b.length?(0,r.jsx)(r.Fragment,{children:(0,r.jsxs)(h.Z,{color:"primary",variant:"plain",size:"lg",sx:{"& tbody tr: hover":{backgroundColor:"light"===t?"rgb(246, 246, 246)":"rgb(33, 33, 40)"},"& tbody tr: hover a":{textDecoration:"underline"}},children:[(0,r.jsx)("thead",{children:(0,r.jsxs)("tr",{children:[(0,r.jsx)("th",{children:v("Name")}),(0,r.jsx)("th",{children:v("Content")}),(0,r.jsx)("th",{children:v("Meta_Data")})]})}),(0,r.jsx)("tbody",{children:b.map(e=>(0,r.jsxs)("tr",{children:[(0,r.jsx)("td",{children:e.doc_name}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:e.content,trigger:"hover",children:e.content.length>10?"".concat(e.content.slice(0,10),"..."):e.content})}),(0,r.jsx)("td",{children:(0,r.jsx)(f.Z,{content:JSON.stringify(e.meta_info||"{}",null,2),trigger:"hover",children:e.meta_info.length>10?"".concat(e.meta_info.slice(0,10),"..."):e.meta_info})})]},e.id))})]})}):(0,r.jsx)(r.Fragment,{})}),(0,r.jsx)(o.Z,{className:"mt-5",direction:"row",justifyContent:"flex-end",children:(0,r.jsx)(g.Z,{defaultPageSize:20,showSizeChanger:!1,current:w,total:_,onChange:async e=>{let t=await (0,j.PR)("/knowledge/".concat(p,"/chunk/list"),{document_id:x,page:e,page_size:20});t.success&&(k(t.data.data),Z(t.data.total),P(t.data.page))},hideOnSinglePage:!0})})]})}},30119:function(e,t,n){"use strict";n.d(t,{Tk:function(){return o},PR:function(){return l},Ej:function(){return d}});var r=n(27790),a=n(6154),c=n(83454);let s=a.Z.create({baseURL:c.env.API_BASE_URL});s.defaults.timeout=1e4,s.interceptors.response.use(e=>e.data,e=>Promise.reject(e)),n(96486);let i={"content-type":"application/json"},o=(e,t)=>{if(t){let n=Object.keys(t).filter(e=>void 0!==t[e]&&""!==t[e]).map(e=>"".concat(e,"=").concat(t[e])).join("&");n&&(e+="?".concat(n))}return s.get("/api"+e,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})},l=(e,t)=>s.post(e,t,{headers:i}).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)}),d=(e,t)=>s.post(e,t).then(e=>e).catch(e=>{r.ZP.error(e),Promise.reject(e)})}},function(e){e.O(0,[662,215,542,289,378,34,774,888,179],function(){return e(e.s=53116)}),_N_E=e.O()}]);

pilot/server/static/_next/static/khlgfjIxxIMna2Uozo7tx/_buildManifest.js

@@ -1 +0,0 @@
-self.__BUILD_MANIFEST=function(s,a,c,t,e,d,n,f,u,i){return{__rewrites:{beforeFiles:[],afterFiles:[],fallback:[]},"/":[a,"static/chunks/673-9f0ceb79f1535087.js","static/chunks/pages/index-ad9fe4b8efe06ace.js"],"/_error":["static/chunks/pages/_error-dee72aff9b2e2c12.js"],"/chat":["static/chunks/pages/chat-cf7d95f63f5aacee.js"],"/database":[s,c,d,t,"static/chunks/566-493c6126d6ac9745.js","static/chunks/196-48d8c6ab3e99146c.js","static/chunks/pages/database-670b4fa76152da89.js"],"/datastores":[e,s,a,n,f,"static/chunks/241-4117dd68a591b7fa.js","static/chunks/pages/datastores-56b24d3cabfcc7f9.js"],"/datastores/documents":[e,"static/chunks/75fc9c18-a784766a129ec5fb.js",s,a,n,c,u,d,t,i,f,"static/chunks/749-f876c99e30a851b8.js","static/chunks/pages/datastores/documents-e33cb051168f44f2.js"],"/datastores/documents/chunklist":[e,s,c,u,t,i,"static/chunks/pages/datastores/documents/chunklist-232ce3aa6271447d.js"],sortedPages:["/","/_app","/_error","/chat","/database","/datastores","/datastores/documents","/datastores/documents/chunklist"]}}("static/chunks/215-c0761f73cb28fff6.js","static/chunks/913-e3ab2daf183d352e.js","static/chunks/902-4b12ce531524546f.js","static/chunks/455-597a8a48388a0d9b.js","static/chunks/29107295-90b90cb30c825230.js","static/chunks/908-33709e71961cb7a1.js","static/chunks/718-ae767ae95bca674e.js","static/chunks/589-8dfb35868cafc00b.js","static/chunks/289-06c0d9f538f77a71.js","static/chunks/34-c924e44753dd2c5b.js"),self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();