refactor: RAG Refactor (#985)

Co-authored-by: Aralhi <xiaoping0501@gmail.com>
Co-authored-by: csunny <cfqsunny@163.com>

web/pages/knowledge/chunk/index.tsx

@@ -1,11 +1,11 @@
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import React, { useEffect, useState } from 'react';
 import { useRouter } from 'next/router';
-import { Breadcrumb, Button, Card, Divider, Empty, List } from 'antd';
+import { Breadcrumb, Card, Empty, Pagination, Spin } from 'antd';
 import { useTranslation } from 'react-i18next';
 import { apiInterceptors, getChunkList } from '@/client/api';
 import DocIcon from '@/components/knowledge/doc-icon';
 
-const page_size = 20;
+const DEDAULT_PAGE_SIZE = 10;
 
 function ChunkList() {
   const router = useRouter();
@@ -13,21 +13,16 @@ function ChunkList() {
   const [chunkList, setChunkList] = useState<any>([]);
   const [total, setTotal] = useState<number>(0);
   const [loading, setLoading] = useState<boolean>(false);
-  const currentPageRef = useRef(1);
   const {
     query: { id, spaceName },
   } = useRouter();
 
-  const hasMore = useMemo(() => {
-    return chunkList?.length < total;
-  }, [chunkList.length, total]);
-
   const fetchChunks = async () => {
     const [_, data] = await apiInterceptors(
       getChunkList(spaceName as string, {
         document_id: id as string,
-        page: currentPageRef.current,
-        page_size,
+        page: 1,
+        page_size: DEDAULT_PAGE_SIZE,
       }),
     );
 
@@ -35,21 +30,16 @@ function ChunkList() {
     setTotal(data?.total!);
   };
 
-  const loaderMoreChunks = async () => {
-    if (loading) {
-      return;
-    }
+  const loaderMore = async (page: number, page_size: number) => {
     setLoading(true);
-    currentPageRef.current += 1;
     const [_, data] = await apiInterceptors(
       getChunkList(spaceName as string, {
         document_id: id as string,
-        page: currentPageRef.current,
+        page,
         page_size,
       }),
     );
-
-    setChunkList([...chunkList, ...(data?.data || [])]);
+    setChunkList(data?.data || []);
     setLoading(false);
   };
 
@@ -58,7 +48,7 @@ function ChunkList() {
   }, [id, spaceName]);
 
   return (
-    <div className="h-full overflow-y-scroll">
+    <div className="h-full overflow-y-scroll relative">
       <Breadcrumb
         className="m-6"
         items={[
@@ -74,8 +64,8 @@ function ChunkList() {
           },
         ]}
       />
-      <div className="flex justify-center flex-col">
-        <div>
+      <Spin spinning={loading}>
+        <div className="flex justify-center flex-col">
           {chunkList?.length > 0 ? (
             chunkList?.map((chunk: any) => {
               return (
@@ -99,14 +89,15 @@ function ChunkList() {
             <Empty image={Empty.PRESENTED_IMAGE_DEFAULT}></Empty>
           )}
         </div>
-        {hasMore && (
-          <Divider>
-            <span className="cursor-pointer" onClick={loaderMoreChunks}>
-              {t('Load_more')}
-            </span>
-          </Divider>
-        )}
-      </div>
+      </Spin>
+      <Pagination
+        className="mx-2 my-4 float-right right-6 bottom-4"
+        defaultCurrent={1}
+        defaultPageSize={DEDAULT_PAGE_SIZE}
+        total={total}
+        showTotal={(total) => `Total ${total} items`}
+        onChange={loaderMore}
+      />
     </div>
   );
 }

web/pages/knowledge/index.tsx

@@ -2,21 +2,29 @@ import React, { useState, useEffect } from 'react';
 import { PlusOutlined } from '@ant-design/icons';
 import { Button, Modal, Steps } from 'antd';
 import SpaceCard from '@/components/knowledge/space-card';
-import { ISpace, StepChangeParams } from '@/types/knowledge';
+import { File, ISpace, StepChangeParams } from '@/types/knowledge';
 import { apiInterceptors, getSpaceList } from '@/client/api';
 import { useTranslation } from 'react-i18next';
 import DocUploadForm from '@/components/knowledge/doc-upload-form';
 import SpaceForm from '@/components/knowledge/space-form';
 import DocTypeForm from '@/components/knowledge/doc-type-form';
+import Segmentation from '@/components/knowledge/segmentation';
+import classNames from 'classnames';
 
 const Knowledge = () => {
   const [spaceList, setSpaceList] = useState<Array<ISpace> | null>([]);
   const [isAddShow, setIsAddShow] = useState<boolean>(false);
   const [activeStep, setActiveStep] = useState<number>(0);
   const [spaceName, setSpaceName] = useState<string>('');
+  const [files, setFiles] = useState<Array<File>>([]);
   const [docType, setDocType] = useState<string>('');
   const { t } = useTranslation();
-  const addKnowledgeSteps = [{ title: t('Knowledge_Space_Config') }, { title: t('Choose_a_Datasource_type') }, { title: t('Setup_the_Datasource') }];
+  const addKnowledgeSteps = [
+    { title: t('Knowledge_Space_Config') },
+    { title: t('Choose_a_Datasource_type') },
+    { title: t('Upload') },
+    { title: t('Segmentation') },
+  ];
 
   async function getSpaces() {
     const [_, data] = await apiInterceptors(getSpaceList());
@@ -26,18 +34,20 @@ const Knowledge = () => {
     getSpaces();
   }, []);
 
-  const handleStepChange = ({ label, spaceName, docType }: StepChangeParams) => {
+  const handleStepChange = ({ label, spaceName, docType, files }: StepChangeParams) => {
     if (label === 'finish') {
       setIsAddShow(false);
       getSpaces();
       setSpaceName('');
       setDocType('');
+      getSpaces();
     } else if (label === 'forward') {
       activeStep === 0 && getSpaces();
       setActiveStep((step) => step + 1);
     } else {
       setActiveStep((step) => step - 1);
     }
+    files && setFiles(files);
     spaceName && setSpaceName(spaceName);
     docType && setDocType(docType);
   };
@@ -78,13 +88,20 @@ const Knowledge = () => {
         width={1000}
         afterClose={() => {
           setActiveStep(0);
+          getSpaces();
         }}
         footer={null}
       >
         <Steps current={activeStep} items={addKnowledgeSteps} />
         {activeStep === 0 && <SpaceForm handleStepChange={handleStepChange} />}
         {activeStep === 1 && <DocTypeForm handleStepChange={handleStepChange} />}
-        {activeStep === 2 && <DocUploadForm spaceName={spaceName} docType={docType} handleStepChange={handleStepChange} />}
+        <DocUploadForm
+          className={classNames({ hidden: activeStep !== 2 })}
+          spaceName={spaceName}
+          docType={docType}
+          handleStepChange={handleStepChange}
+        />
+        {activeStep === 3 && <Segmentation spaceName={spaceName} docType={docType} uploadFiles={files} handleStepChange={handleStepChange} />}
       </Modal>
     </div>
   );