feat: file details ai (#7251)

Co-authored-by: 杨国璇 <ygx@Hello-word.local>

frontend/src/assets/icons/ai.svg

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 21.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="图层_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 32 32" style="enable-background:new 0 0 32 32;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#999999;}
+</style>
+<title>ai</title>
+<g id="ai">
+	<g id="ai-assistant">
+		<path id="形状" class="st0" d="M16.1,0C24.9,0,32,6.8,32,15s-7.1,15-15.9,15c-0.1,0-0.3,0-0.6,0c-0.4,0-3.7,0.2-4.9,0.6
+			c-1,0.2-2.8,0.9-3.9,1.2C6.6,31.9,6.2,32,6,32c-0.5,0-0.9-0.4-1-1c-0.2-1.1-0.2-3,0.4-4.7C2.1,23.5,0,19.6,0,15
+			C0.1,6.8,7.2,0,16.1,0z M16,3C8.8,3,3,8.6,3,15.4c0,3.5,1.7,6.6,4.5,9.1l1.2,1.1c-0.4,0.8-0.7,1.3-0.8,1.7
+			c-0.2,0.5-0.3,1.3-0.2,1.7c0.8-0.4,2-1,2.9-1.2c1.7-0.4,4.2-0.6,5.4-0.6c5.7,0,13-5,13-11.8S23.2,3,16,3z M13.1,6.9
+			C10.3,6.9,8,9.1,8,11.8v10c0,0.1,0.1,0.3,0.3,0.3h2.8V11.9c0-1,0.9-1.9,1.9-1.9s1.9,0.9,1.9,1.9v2.3H12c-0.1,0-0.3,0-0.4,0.1
+			c-0.1,0.1-0.1,0.3-0.1,0.4v1.9c0,0.4,0.3,0.6,0.6,0.6H15v4.9h2.8c0.1,0,0.3-0.1,0.3-0.3v-10C18.1,9.1,15.9,6.9,13.1,6.9L13.1,6.9z
+			 M22.4,11h-1.8c-0.4,0-0.6,0.2-0.6,0.6v9.8c0,0.4,0.2,0.6,0.6,0.6h1.8c0.4,0,0.6-0.2,0.6-0.6v-9.8C23,11.2,22.8,11,22.4,11z
+			 M21.5,10c0.8,0,1.5-0.7,1.5-1.5S22.3,7,21.5,7S20,7.7,20,8.5S20.7,10,21.5,10z"/>
+	</g>
+</g>
+</svg>

frontend/src/components/dialog/image-dialog.js

@@ -2,13 +2,13 @@ import React, { useCallback } from 'react';
 import PropTypes from 'prop-types';
 import { gettext } from '../../utils/constants';
 import Lightbox from '@seafile/react-image-lightbox';
-import { useMetadataOperations } from '../../hooks/metadata-operation';
+import { useMetadataAIOperations } from '../../hooks/metadata-ai-operation';
 import { SYSTEM_FOLDERS } from '../../constants';
 
 import '@seafile/react-image-lightbox/style.css';
 
 const ImageDialog = ({ enableRotate: oldEnableRotate, imageItems, imageIndex, closeImagePopup, moveToPrevImage, moveToNextImage, onDeleteImage, onRotateImage }) => {
-  const { onOCR } = useMetadataOperations();
+  const { enableOCR, enableMetadata, canModify, onOCR: onOCRAPI, OCRSuccessCallBack } = useMetadataAIOperations();
 
   const downloadImage = useCallback((url) => {
     location.href = url;
@@ -33,6 +33,10 @@ const ImageDialog = ({ enableRotate: oldEnableRotate, imageItems, imageIndex, cl
   }
 
   const isSystemFolder = SYSTEM_FOLDERS.find(folderPath => mainImg.parentDir.startsWith(folderPath));
+  let onOCR = null;
+  if (enableOCR && enableMetadata && canModify && !isSystemFolder) {
+    onOCR = () => onOCRAPI({ parentDir: mainImg.parentDir, fileName: mainImg.name }, { success_callback: OCRSuccessCallBack });
+  }
 
   return (
     <Lightbox
@@ -57,7 +61,7 @@ const ImageDialog = ({ enableRotate: oldEnableRotate, imageItems, imageIndex, cl
       onViewOriginal={onViewOriginal}
       viewOriginalImageLabel={gettext('View original image')}
       onRotateImage={(onRotateImage && enableRotate) ? (angle) => onRotateImage(imageIndex, angle) : null}
-      onOCR={onOCR && !isSystemFolder ? () => onOCR(mainImg.parentDir, mainImg.name) : null}
+      onOCR={onOCR}
       OCRLabel={gettext('OCR')}
     />
   );

frontend/src/components/dirent-detail/dirent-details/index.js

@@ -10,7 +10,7 @@ import DirDetails from './dir-details';
 import FileDetails from './file-details';
 import ObjectUtils from '../../../metadata/utils/object-utils';
 import { MetadataDetailsProvider } from '../../../metadata/hooks';
-import Settings from '../../../metadata/components/metadata-details/settings';
+import { Settings, AI } from '../../../metadata/components/metadata-details';
 import { getDirentPath } from './utils';
 
 import './index.css';
@@ -129,6 +129,7 @@ class DirentDetails extends React.Component {
       >
         <Detail>
           <Header title={dirent?.name || ''} icon={Utils.getDirentIcon(dirent, true)} onClose={this.props.onClose} >
+            <AI />
             <Settings />
           </Header>
           <Body>

frontend/src/components/dirent-detail/embedded-file-details/index.js

@@ -8,7 +8,7 @@ import { Header, Body } from '../detail';
 import FileDetails from './file-details';
 import { MetadataContext } from '../../../metadata';
 import { MetadataDetailsProvider } from '../../../metadata/hooks';
-import Settings from '../../../metadata/components/metadata-details/settings';
+import { AI, Settings } from '../../../metadata/components/metadata-details';
 
 import './index.css';
 
@@ -54,6 +54,7 @@ const EmbeddedFileDetails = ({ repoID, repoInfo, dirent, path, onClose, width =
         style={{ width }}
       >
         <Header title={dirent?.name || ''} icon={Utils.getDirentIcon(dirent, true)} onClose={onClose} component={headerComponent} >
+          <AI />
           <Settings />
         </Header>
         <Body>

frontend/src/hooks/metadata-ai-operation.js

@@ -0,0 +1,109 @@
+import React, { useContext, useCallback, useMemo } from 'react';
+import metadataAPI from '../metadata/api';
+import { Utils } from '../utils/utils';
+import toaster from '../components/toast';
+import { PRIVATE_COLUMN_KEY, EVENT_BUS_TYPE } from '../metadata/constants';
+import { gettext, lang } from '../utils/constants';
+
+// This hook provides content related to metadata ai operation
+const MetadataAIOperationsContext = React.createContext(null);
+
+export const MetadataAIOperationsProvider = ({
+  repoID,
+  enableMetadata = false,
+  enableOCR = false,
+  enableTags = false,
+  tagsLang,
+  repoInfo,
+  children
+}) => {
+  const permission = useMemo(() => repoInfo.permission !== 'admin' && repoInfo.permission !== 'rw' ? 'r' : 'rw', [repoInfo]);
+  const canModify = useMemo(() => permission === 'rw', [permission]);
+
+  const OCRSuccessCallBack = useCallback(({ parentDir, fileName, ocrResult } = {}) => {
+    toaster.success(gettext('Successfully OCR'));
+    if (!ocrResult) return;
+    const update = { [PRIVATE_COLUMN_KEY.OCR]: ocrResult };
+    metadataAPI.modifyRecord(repoID, { parentDir, fileName }, update).then(res => {
+      const eventBus = window?.sfMetadataContext?.eventBus;
+      eventBus && eventBus.dispatch(EVENT_BUS_TYPE.LOCAL_RECORD_CHANGED, { parentDir, fileName }, update);
+    });
+  }, [repoID]);
+
+  const onOCR = useCallback(({ parentDir, fileName }, { success_callback, fail_callback } = {}) => {
+    const filePath = Utils.joinPath(parentDir, fileName);
+    metadataAPI.ocr(repoID, filePath).then(res => {
+      const ocrResult = res.data.ocr_result;
+      const validResult = Array.isArray(ocrResult) && ocrResult.length > 0 ? JSON.stringify(ocrResult) : null;
+      success_callback && success_callback({ parentDir, fileName, ocrResult: validResult });
+    }).catch(error => {
+      const errorMessage = gettext('OCR failed');
+      toaster.danger(errorMessage);
+      fail_callback && fail_callback();
+    });
+  }, [repoID]);
+
+  const generateDescription = useCallback(({ parentDir, fileName }, { success_callback, fail_callback } = {}) => {
+    const filePath = Utils.joinPath(parentDir, fileName);
+    const isImage = Utils.imageCheck(fileName);
+    let APIName = '';
+    if (Utils.isDescriptionSupportedFile(fileName)) {
+      APIName = isImage ? 'imageCaption' : 'generateDescription';
+    }
+    if (!APIName) return;
+
+    metadataAPI[APIName](repoID, filePath, lang).then(res => {
+      const description = res?.data?.summary || res.data.desc || '';
+      success_callback && success_callback({ parentDir, fileName, description });
+    }).catch(error => {
+      const errorMessage = isImage ? gettext('Failed to generate image description') : gettext('Failed to generate description');
+      toaster.danger(errorMessage);
+      fail_callback && fail_callback();
+    });
+  }, [repoID]);
+
+  const extractFilesDetails = useCallback((objIds, { success_callback, fail_callback } = {}) => {
+    metadataAPI.extractFileDetails(repoID, objIds).then(res => {
+      const details = res?.data?.details || [];
+      success_callback && success_callback({ details });
+    }).catch(error => {
+      const errorMessage = gettext('Failed to extract file details');
+      toaster.danger(errorMessage);
+      fail_callback && fail_callback();
+    });
+  }, [repoID]);
+
+  const extractFileDetails = useCallback((objId, { success_callback, fail_callback } = {}) => {
+    extractFilesDetails([objId], {
+      success_callback: ({ details }) => {
+        success_callback && success_callback({ detail: details[0] });
+      },
+      fail_callback
+    });
+  }, [extractFilesDetails]);
+
+  return (
+    <MetadataAIOperationsContext.Provider value={{
+      enableMetadata,
+      enableOCR,
+      enableTags,
+      tagsLang,
+      canModify,
+      onOCR,
+      OCRSuccessCallBack,
+      generateDescription,
+      extractFilesDetails,
+      extractFileDetails,
+    }}>
+      {children}
+    </MetadataAIOperationsContext.Provider>
+  );
+};
+
+export const useMetadataAIOperations = () => {
+  const context = useContext(MetadataAIOperationsContext);
+  if (!context) {
+    throw new Error('\'MetadataAIOperationsContext\' is null');
+  }
+  return context;
+};

frontend/src/hooks/metadata-operation.js

@@ -1,54 +0,0 @@
-import React, { useContext, useCallback, useMemo } from 'react';
-import metadataAPI from '../metadata/api';
-import { Utils } from '../utils/utils';
-import toaster from '../components/toast';
-import { PRIVATE_COLUMN_KEY, EVENT_BUS_TYPE } from '../metadata/constants';
-import { gettext } from '../utils/constants';
-
-// This hook provides content related to metadata record operation
-const MetadataOperationsContext = React.createContext(null);
-
-export const MetadataOperationsProvider = ({ repoID, enableMetadata, enableOCR, repoInfo, children }) => {
-  const permission = useMemo(() => repoInfo.permission !== 'admin' && repoInfo.permission !== 'rw' ? 'r' : 'rw', [repoInfo]);
-  const canModify = useMemo(() => permission === 'rw', [permission]);
-
-  const onOCR = useCallback((parentDir, fileName) => {
-    const filePath = Utils.joinPath(parentDir, fileName);
-    metadataAPI.ocr(repoID, filePath).then(res => {
-      const ocrResult = res.data.ocr_result;
-      const validResult = Array.isArray(ocrResult) && ocrResult.length > 0 ? JSON.stringify(ocrResult) : null;
-      toaster.success(gettext('Successfully OCR'));
-      if (validResult) {
-        const update = { [PRIVATE_COLUMN_KEY.OCR]: validResult };
-        metadataAPI.modifyRecord(repoID, { parentDir, fileName }, update).then(res => {
-          const eventBus = window?.sfMetadataContext?.eventBus;
-          if (eventBus) {
-            eventBus.dispatch(EVENT_BUS_TYPE.LOCAL_RECORD_CHANGED, { parentDir, fileName }, update);
-          }
-        });
-      }
-    }).catch(error => {
-      const errorMessage = Utils.getErrorMsg(error);
-      toaster.danger(errorMessage);
-    });
-  }, [repoID]);
-
-  let value = {};
-  if (canModify && enableMetadata && enableOCR) {
-    value['onOCR'] = onOCR;
-  }
-
-  return (
-    <MetadataOperationsContext.Provider value={value}>
-      {children}
-    </MetadataOperationsContext.Provider>
-  );
-};
-
-export const useMetadataOperations = () => {
-  const context = useContext(MetadataOperationsContext);
-  if (!context) {
-    throw new Error('\'MetadataOperationsContext\' is null');
-  }
-  return context;
-};

frontend/src/hooks/metadata-status.js

@@ -2,7 +2,7 @@ import React, { useContext, useEffect, useCallback, useState, useMemo } from 're
 import metadataAPI from '../metadata/api';
 import { Utils } from '../utils/utils';
 import toaster from '../components/toast';
-import { MetadataOperationsProvider } from './metadata-operation';
+import { MetadataAIOperationsProvider } from './metadata-ai-operation';
 
 // This hook provides content related to seahub interaction, such as whether to enable extended attributes
 const MetadataStatusContext = React.createContext(null);
@@ -119,14 +119,16 @@ export const MetadataStatusProvider = ({ repoID, currentRepoInfo, hideMetadataVi
       }}
     >
       {!isLoading && (
-        <MetadataOperationsProvider
+        <MetadataAIOperationsProvider
           repoID={repoID}
           enableMetadata={enableMetadata}
           enableOCR={enableOCR}
+          enableTags={enableTags}
+          tagsLang={tagsLang}
           repoInfo={currentRepoInfo}
         >
           {children}
-        </MetadataOperationsProvider>
+        </MetadataAIOperationsProvider>
       )}
     </MetadataStatusContext.Provider>
   );

frontend/src/metadata/components/detail-editor/long-text-editor/index.js

@@ -1,4 +1,4 @@
-import React, { useCallback, useState } from 'react';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
 import PropTypes from 'prop-types';
 import { LongTextFormatter } from '@seafile/sf-metadata-ui-component';
 import Editor from '../../cell-editors/long-text-editor';
@@ -10,6 +10,8 @@ const LongTextEditor = ({ field, value: oldValue, onChange }) => {
   const [value, setValue] = useState(oldValue);
   const [showEditor, setShowEditor] = useState(false);
 
+  const valueRef = useRef(null);
+
   const openEditor = useCallback(() => {
     setShowEditor(true);
   }, []);
@@ -23,6 +25,13 @@ const LongTextEditor = ({ field, value: oldValue, onChange }) => {
     setShowEditor(false);
   }, []);
 
+  useEffect(() => {
+    if (showEditor) return;
+    if (valueRef.current === oldValue) return;
+    setValue(oldValue);
+    valueRef.current = oldValue;
+  }, [showEditor, oldValue]);
+
   const isEmpty = !value || !value.trim();
 
   return (

frontend/src/metadata/components/metadata-details/ai/index.css

@@ -0,0 +1,3 @@
+.sf-metadata-ai-dropdown-menu .dropdown-menu {
+  left: -8px !important;
+}

frontend/src/metadata/components/metadata-details/ai/index.js

@@ -0,0 +1,180 @@
+import React, { useCallback, useMemo, useState } from 'react';
+import { Dropdown, DropdownToggle, DropdownMenu, DropdownItem } from 'reactstrap';
+import { ModalPortal } from '@seafile/sf-metadata-ui-component';
+import Icon from '../../../../components/icon';
+import { useMetadataDetails } from '../../../hooks';
+import { useMetadataStatus } from '../../../../hooks';
+import { gettext } from '../../../../utils/constants';
+import { Utils } from '../../../../utils/utils';
+import { getFileNameFromRecord, getFileObjIdFromRecord, getParentDirFromRecord, getRecordIdFromRecord } from '../../../utils/cell';
+import { getColumnByKey } from '../../../utils/column';
+import { PRIVATE_COLUMN_KEY } from '../constants';
+import { useMetadataAIOperations } from '../../../../hooks/metadata-ai-operation';
+import FileTagsDialog from '../../dialog/file-tags-dialog';
+import { checkIsDir } from '../../../utils/row';
+
+import './index.css';
+
+const OPERATION = {
+  GENERATE_DESCRIPTION: 'generate-description',
+  OCR: 'ocr',
+  FILE_TAGS: 'file-tags',
+  FILE_DETAIL: 'file-detail',
+};
+
+const AI = () => {
+  const [isMenuShow, setMenuShow] = useState(false);
+  const [isFileTagsDialogShow, setFileTagsDialogShow] = useState(false);
+
+  const { enableMetadata, enableTags, enableOCR } = useMetadataStatus();
+  const { canModifyRecord, columns, record, onChange, onLocalRecordChange, updateFileTags } = useMetadataDetails();
+  const { onOCR, generateDescription, extractFileDetails } = useMetadataAIOperations();
+
+  const options = useMemo(() => {
+    if (!canModifyRecord) return [];
+    if (!record) return [];
+    if (checkIsDir(record)) return [];
+    const descriptionColumn = getColumnByKey(columns, PRIVATE_COLUMN_KEY.FILE_DESCRIPTION);
+    const fileName = getFileNameFromRecord(record);
+    const isImage = Utils.imageCheck(fileName);
+    const isVideo = Utils.videoCheck(fileName);
+    const isDescribableDoc = Utils.isDescriptionSupportedFile(fileName);
+    let list = [];
+
+    if (descriptionColumn && isDescribableDoc) {
+      list.push({
+        value: OPERATION.GENERATE_DESCRIPTION,
+        label: isImage ? gettext('Generate image description') : gettext('Generate description'),
+        record
+      });
+    }
+
+    if (enableOCR && isImage) {
+      list.push({ value: OPERATION.OCR, label: gettext('OCR'), record });
+    }
+
+    if (isImage || isVideo) {
+      list.push({ value: OPERATION.FILE_DETAIL, label: gettext('Extract file detail'), record });
+    }
+
+    if (enableTags && isDescribableDoc && !isVideo) {
+      list.push({ value: OPERATION.FILE_TAGS, label: gettext('Generate file tags'), record });
+    }
+    return list;
+  }, [enableOCR, enableTags, canModifyRecord, columns, record]);
+
+  const onToggle = useCallback((event) => {
+    event && event.preventDefault();
+    event && event.stopPropagation();
+    setMenuShow(!isMenuShow);
+  }, [isMenuShow]);
+
+  const toggleFileTagsDialog = useCallback(() => {
+    setFileTagsDialogShow(!isFileTagsDialogShow);
+  }, [isFileTagsDialogShow]);
+
+  const handelOperation = useCallback((op) => {
+    const { value: opType, record } = op;
+    const recordId = getRecordIdFromRecord(record);
+    const parentDir = getParentDirFromRecord(record);
+    const fileName = getFileNameFromRecord(record);
+    const objId = getFileObjIdFromRecord(record);
+
+    switch (opType) {
+      case OPERATION.GENERATE_DESCRIPTION: {
+        generateDescription({ parentDir, fileName }, {
+          success_callback: ({ description }) => {
+            if (!description) return;
+            onChange && onChange(PRIVATE_COLUMN_KEY.FILE_DESCRIPTION, description);
+          },
+        });
+        break;
+      }
+      case OPERATION.OCR: {
+        onOCR({ parentDir, fileName }, {
+          success_callback: ({ ocrResult }) => {
+            if (!ocrResult) return;
+            onChange && onChange(PRIVATE_COLUMN_KEY.OCR, JSON.stringify(ocrResult));
+          },
+        });
+        break;
+      }
+      case OPERATION.FILE_TAGS: {
+        setFileTagsDialogShow(true);
+        break;
+      }
+      case OPERATION.FILE_DETAIL: {
+        extractFileDetails(objId, {
+          success_callback: ({ detail }) => {
+            if (!detail) return;
+            const captureColumn = getColumnByKey(columns, PRIVATE_COLUMN_KEY.CAPTURE_TIME);
+            if (captureColumn) {
+              const value = detail[PRIVATE_COLUMN_KEY.CAPTURE_TIME];
+              value && onChange && onChange(PRIVATE_COLUMN_KEY.CAPTURE_TIME, value);
+            }
+            const fileDetails = detail[PRIVATE_COLUMN_KEY.FILE_DETAILS];
+            const location = detail[PRIVATE_COLUMN_KEY.LOCATION];
+            let update = {};
+            if (fileDetails) {
+              update[PRIVATE_COLUMN_KEY.FILE_DETAILS] = fileDetails;
+            }
+            if (location) {
+              update[PRIVATE_COLUMN_KEY.LOCATION] = location;
+            }
+            Object.keys(update).length > 0 && onLocalRecordChange(recordId, update);
+          },
+        });
+        break;
+      }
+      default: {
+        setMenuShow(false);
+        break;
+      }
+    }
+  }, [columns, generateDescription, onOCR, extractFileDetails, onChange, onLocalRecordChange]);
+
+  const renderDropdown = useCallback(() => {
+    if (!enableMetadata) return null;
+    if (!canModifyRecord) return null;
+    if (!record) return null;
+    if (options.length === 0) return null;
+
+    return (
+      <Dropdown className="sf-metadata-dropdown-menu" isOpen={isMenuShow} toggle={onToggle}>
+        <DropdownToggle
+          tag="span"
+          role="button"
+          data-toggle="dropdown"
+          aria-expanded={isMenuShow}
+          title={gettext('AI')}
+          aria-label={gettext('AI')}
+          tabIndex={0}
+        >
+          <div className="detail-control mr-2">
+            <Icon symbol="ai" className="detail-control-close" />
+          </div>
+        </DropdownToggle>
+        {isMenuShow && (
+          <ModalPortal>
+            <div className="sf-metadata-ai-dropdown-menu large">
+              <DropdownMenu right={true}>
+                {options.map(op => (<DropdownItem key={op.value} onClick={() => handelOperation(op)}>{op.label}</DropdownItem>))}
+              </DropdownMenu>
+            </div>
+          </ModalPortal>
+        )}
+      </Dropdown>
+    );
+  }, [isMenuShow, enableMetadata, canModifyRecord, record, options, onToggle, handelOperation]);
+
+  return (
+    <>
+      {renderDropdown()}
+      {isFileTagsDialogShow && (
+        <FileTagsDialog record={record} onToggle={toggleFileTagsDialog} onSubmit={updateFileTags} />
+      )}
+    </>
+  );
+};
+
+export default AI;

frontend/src/metadata/components/metadata-details/index.js

@@ -9,6 +9,8 @@ import { PRIVATE_COLUMN_KEY } from '../../constants';
 import Location from './location';
 import { useMetadataDetails } from '../../hooks';
 import { checkIsDir } from '../../utils/row';
+import AI from './ai';
+import Settings from './settings';
 
 import './index.css';
 
@@ -64,3 +66,7 @@ const MetadataDetails = () => {
 };
 
 export default MetadataDetails;
+export {
+  AI,
+  Settings,
+};

frontend/src/metadata/components/metadata-details/settings/index.js

@@ -10,7 +10,7 @@ const Settings = () => {
   const [isShowSetter, setShowSetter] = useState(false);
 
   const { enableMetadata } = useMetadataStatus();
-  const { modifyColumnOrder, modifyHiddenColumns, columns, canModifyDetails } = useMetadataDetails();
+  const { modifyColumnOrder, modifyHiddenColumns, record, columns, canModifyDetails } = useMetadataDetails();
   const hiddenColumns = useMemo(() => columns.filter(c => !c.shown).map(c => c.key), [columns]);
 
   const onSetterToggle = useCallback(() => {
@@ -20,6 +20,7 @@ const Settings = () => {
 
   if (!enableMetadata) return null;
   if (!canModifyDetails) return null;
+  if (!record) return null;
 
   return (
     <>

frontend/src/metadata/hooks/metadata-details.js

@@ -49,7 +49,7 @@ export const MetadataDetailsProvider = ({ repoID, repoInfo, path, dirent, dirent
     return [...exitColumnsOrder, ...newColumns];
   }, [originColumns, detailsSettings]);
 
-  const localRecordChanged = useCallback((recordId, updates) => {
+  const onLocalRecordChange = useCallback((recordId, updates) => {
     if (getRecordIdFromRecord(record) !== recordId) return;
     const newRecord = { ...record, ...updates };
     setRecord(newRecord);
@@ -180,11 +180,11 @@ export const MetadataDetailsProvider = ({ repoID, repoInfo, path, dirent, dirent
   useEffect(() => {
     const eventBus = window?.sfMetadataContext?.eventBus;
     if (!eventBus) return;
-    const unsubscribeLocalRecordChanged = eventBus.subscribe(EVENT_BUS_TYPE.LOCAL_RECORD_DETAIL_CHANGED, localRecordChanged);
+    const unsubscribeLocalRecordChanged = eventBus.subscribe(EVENT_BUS_TYPE.LOCAL_RECORD_DETAIL_CHANGED, onLocalRecordChange);
     return () => {
       unsubscribeLocalRecordChanged();
     };
-  }, [localRecordChanged]);
+  }, [onLocalRecordChange]);
 
   return (
     <MetadataDetailsContext.Provider
@@ -195,6 +195,7 @@ export const MetadataDetailsProvider = ({ repoID, repoInfo, path, dirent, dirent
         record,
         columns,
         onChange,
+        onLocalRecordChange,
         modifyColumnData,
         updateFileTags,
         modifyHiddenColumns,

frontend/src/metadata/views/table/context-menu/index.js

@@ -16,6 +16,7 @@ import { openInNewTab, openParentFolder } from '../../../utils/file';
 import DeleteFolderDialog from '../../../../components/dialog/delete-folder-dialog';
 import MoveDirent from '../../../../components/dialog/move-dirent-dialog';
 import { Dirent } from '../../../../models';
+import { useMetadataAIOperations } from '../../../../hooks/metadata-ai-operation';
 
 const OPERATION = {
   CLEAR_SELECTED: 'clear-selected',
@@ -24,7 +25,7 @@ const OPERATION = {
   OPEN_IN_NEW_TAB: 'open-new-tab',
   GENERATE_DESCRIPTION: 'generate-description',
   OCR: 'ocr',
-  IMAGE_CAPTION: 'image-caption',
+  IMAGE_DESCRIPTION: 'image-description',
   FILE_TAGS: 'file-tags',
   DELETE_RECORD: 'delete-record',
   DELETE_RECORDS: 'delete-records',
@@ -49,6 +50,7 @@ const ContextMenu = ({
 
   const { metadata } = useMetadataView();
   const { enableOCR } = useMetadataStatus();
+  const { onOCR, generateDescription, extractFilesDetails } = useMetadataAIOperations();
 
   const repoID = window.sfMetadataStore.repoId;
 
@@ -56,7 +58,7 @@ const ContextMenu = ({
     return window.sfMetadataContext.canModifyRow(row);
   };
 
-  const checkIsDescribableDoc = useCallback((record) => {
+  const checkIsDescribableFile = useCallback((record) => {
     const fileName = getFileNameFromRecord(record);
     return checkCanModifyRow(record) && Utils.isDescriptionSupportedFile(fileName);
   }, []);
@@ -142,6 +144,10 @@ const ContextMenu = ({
         list.push({ value: OPERATION.DELETE_RECORDS, label: gettext('Delete'), records: ableDeleteRecords });
       }
       const imageOrVideoRecords = records.filter(record => {
+        const isFolder = checkIsDir(record);
+        if (isFolder) return false;
+        const canModifyRow = checkCanModifyRow(record);
+        if (!canModifyRow) return false;
         const fileName = getFileNameFromRecord(record);
         return Utils.imageCheck(fileName) || Utils.videoCheck(fileName);
       });
@@ -165,24 +171,29 @@ const ContextMenu = ({
     list.push({ value: OPERATION.OPEN_PARENT_FOLDER, label: gettext('Open parent folder'), record });
     const fileName = getFileNameFromRecord(record);
 
-    if (descriptionColumn) {
+    if (!isFolder && canModifyRow) {
-      if (checkIsDescribableDoc(record)) {
+      const isDescribableFile = checkIsDescribableFile(record);
-        list.push({ value: OPERATION.GENERATE_DESCRIPTION, label: gettext('Generate description'), record });
+      const isImage = Utils.imageCheck(fileName);
-      } else if (canModifyRow && Utils.imageCheck(fileName)) {
+      const isVideo = Utils.videoCheck(fileName);
-        list.push({ value: OPERATION.IMAGE_CAPTION, label: gettext('Generate image description'), record });
+      if (descriptionColumn && isDescribableFile) {
+        list.push({
+          value: OPERATION.GENERATE_DESCRIPTION,
+          label: Utils.imageCheck(fileName) ? gettext('Generate image description') : gettext('Generate description'),
+          record
+        });
       }
-    }
 
-    if (enableOCR && canModifyRow && Utils.imageCheck(fileName)) {
+      if (enableOCR && isImage) {
-      list.push({ value: OPERATION.OCR, label: gettext('OCR'), record });
+        list.push({ value: OPERATION.OCR, label: gettext('OCR'), record });
-    }
+      }
 
-    if (canModifyRow && (Utils.imageCheck(fileName) || Utils.videoCheck(fileName))) {
+      if (isImage || isVideo) {
-      list.push({ value: OPERATION.FILE_DETAIL, label: gettext('Extract file detail'), record: record });
+        list.push({ value: OPERATION.FILE_DETAIL, label: gettext('Extract file detail'), record: record });
-    }
+      }
 
-    if (tagsColumn && canModifyRow && (Utils.imageCheck(fileName) || checkIsDescribableDoc(record))) {
+      if (tagsColumn && isDescribableFile && !isVideo) {
-      list.push({ value: OPERATION.FILE_TAGS, label: gettext('Generate file tags'), record: record });
+        list.push({ value: OPERATION.FILE_TAGS, label: gettext('Generate file tags'), record: record });
+      }
     }
 
     // handle delete folder/file
@@ -199,7 +210,7 @@ const ContextMenu = ({
     }
 
     return list;
-  }, [visible, isGroupView, selectedPosition, recordMetrics, selectedRange, metadata, recordGetterByIndex, checkIsDescribableDoc, enableOCR, getAbleDeleteRecords]);
+  }, [visible, isGroupView, selectedPosition, recordMetrics, selectedRange, metadata, recordGetterByIndex, checkIsDescribableFile, enableOCR, getAbleDeleteRecords]);
 
   const handleHide = useCallback((event) => {
     if (!menuRef.current && visible) {
@@ -212,83 +223,46 @@ const ContextMenu = ({
     }
   }, [menuRef, visible]);
 
-  const generateDescription = useCallback((record) => {
+  const handelGenerateDescription = useCallback((record) => {
-    const descriptionColumnKey = PRIVATE_COLUMN_KEY.FILE_DESCRIPTION;
+    if (!checkCanModifyRow(record)) return;
-    let path = '';
+    const parentDir = getParentDirFromRecord(record);
-    let idOldRecordData = {};
-    let idOriginalOldRecordData = {};
     const fileName = getFileNameFromRecord(record);
-    if (Utils.isDescriptionSupportedFile(fileName) && checkCanModifyRow(record)) {
+    if (!fileName || !parentDir) return;
-      const parentDir = getParentDirFromRecord(record);
+    const checkIsDescribableFile = Utils.isDescriptionSupportedFile(fileName);
-      path = Utils.joinPath(parentDir, fileName);
+    if (!checkIsDescribableFile) return;
-      idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [descriptionColumnKey]: record[descriptionColumnKey] };
-      idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [descriptionColumnKey]: record[descriptionColumnKey] };
-    }
-    if (path === '') return;
-    window.sfMetadataContext.generateDescription(path).then(res => {
-      const description = res.data.summary;
-      const updateRecordId = record[PRIVATE_COLUMN_KEY.ID];
-      const recordIds = [updateRecordId];
-      let idRecordUpdates = {};
-      let idOriginalRecordUpdates = {};
-      idRecordUpdates[updateRecordId] = { [descriptionColumnKey]: description };
-      idOriginalRecordUpdates[updateRecordId] = { [descriptionColumnKey]: description };
-      updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
-    }).catch(error => {
-      const errorMessage = gettext('Failed to generate description');
-      toaster.danger(errorMessage);
-    });
-  }, [updateRecords]);
 
-  const imageCaption = useCallback((record) => {
+    const descriptionColumnKey = PRIVATE_COLUMN_KEY.FILE_DESCRIPTION;
-    const summaryColumnKey = PRIVATE_COLUMN_KEY.FILE_DESCRIPTION;
+    let idOldRecordData = { [record[PRIVATE_COLUMN_KEY.ID]]: { [descriptionColumnKey]: record[descriptionColumnKey] } };
-    let path = '';
+    let idOriginalOldRecordData = { [record[PRIVATE_COLUMN_KEY.ID]]: { [descriptionColumnKey]: record[descriptionColumnKey] } };
-    let idOldRecordData = {};
+    generateDescription({ parentDir, fileName }, {
-    let idOriginalOldRecordData = {};
+      success_callback: ({ description }) => {
-    const fileName = getFileNameFromRecord(record);
+        const updateRecordId = record[PRIVATE_COLUMN_KEY.ID];
-    if (Utils.imageCheck(fileName) && checkCanModifyRow(record)) {
+        const recordIds = [updateRecordId];
-      const parentDir = getParentDirFromRecord(record);
+        let idRecordUpdates = {};
-      path = Utils.joinPath(parentDir, fileName);
+        let idOriginalRecordUpdates = {};
-      idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
+        idRecordUpdates[updateRecordId] = { [descriptionColumnKey]: description };
-      idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
+        idOriginalRecordUpdates[updateRecordId] = { [descriptionColumnKey]: description };
-    }
+        updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
-    if (path === '') return;
+      }
-    window.sfMetadataContext.imageCaption(path).then(res => {
-      const desc = res.data.desc;
-      const updateRecordId = record[PRIVATE_COLUMN_KEY.ID];
-      const recordIds = [updateRecordId];
-      let idRecordUpdates = {};
-      let idOriginalRecordUpdates = {};
-      idRecordUpdates[updateRecordId] = { [summaryColumnKey]: desc };
-      idOriginalRecordUpdates[updateRecordId] = { [summaryColumnKey]: desc };
-      updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
-    }).catch(error => {
-      const errorMessage = gettext('Failed to generate image description');
-      toaster.danger(errorMessage);
     });
-  }, [updateRecords]);
+  }, [updateRecords, generateDescription]);
 
   const toggleFileTagsRecord = useCallback((record = null) => {
     setFileTagsRecord(record);
   }, []);
 
   const ocr = useCallback((record) => {
-    const ocrResultColumnKey = PRIVATE_COLUMN_KEY.OCR;
+    if (!checkCanModifyRow(record)) return;
-    let path = '';
+    const parentDir = getParentDirFromRecord(record);
-    let idOldRecordData = {};
-    let idOriginalOldRecordData = {};
     const fileName = getFileNameFromRecord(record);
-    if (Utils.imageCheck(fileName) && checkCanModifyRow(record)) {
+    if (!Utils.imageCheck(fileName)) return;
-      const parentDir = getParentDirFromRecord(record);
+
-      path = Utils.joinPath(parentDir, fileName);
+    const ocrResultColumnKey = PRIVATE_COLUMN_KEY.OCR;
-      idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [ocrResultColumnKey]: record[ocrResultColumnKey] };
+    let idOldRecordData = { [record[PRIVATE_COLUMN_KEY.ID]]: { [ocrResultColumnKey]: record[ocrResultColumnKey] } };
-      idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [ocrResultColumnKey]: record[ocrResultColumnKey] };
+    let idOriginalOldRecordData = { [record[PRIVATE_COLUMN_KEY.ID]]: { [ocrResultColumnKey]: record[ocrResultColumnKey] } };
-    }
+    onOCR({ parentDir, fileName }, {
-    if (path === '') return;
+      success_callback: ({ ocrResult }) => {
-    window.sfMetadataContext.ocr(path).then(res => {
+        if (!ocrResult) return;
-      const ocrResult = res.data.ocr_result;
-      const validResult = Array.isArray(ocrResult) && ocrResult.length > 0 ? JSON.stringify(ocrResult) : null;
-      if (validResult) {
         const updateRecordId = record[PRIVATE_COLUMN_KEY.ID];
         const recordIds = [updateRecordId];
         let idRecordUpdates = {};
@@ -296,12 +270,9 @@ const ContextMenu = ({
         idRecordUpdates[updateRecordId] = { [ocrResultColumnKey]: ocrResult ? JSON.stringify(ocrResult) : null };
         idOriginalRecordUpdates[updateRecordId] = { [ocrResultColumnKey]: ocrResult ? JSON.stringify(ocrResult) : null };
         updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
-      }
+      },
-    }).catch(error => {
-      const errorMessage = gettext('OCR failed');
-      toaster.danger(errorMessage);
     });
-  }, [updateRecords]);
+  }, [updateRecords, onOCR]);
 
   const updateFileDetails = useCallback((records) => {
     const recordObjIds = records.map(record => getFileObjIdFromRecord(record));
@@ -311,10 +282,10 @@ const ContextMenu = ({
     }
 
     const recordIds = records.map(record => getRecordIdFromRecord(record));
-    window.sfMetadataContext.extractFileDetails(recordObjIds).then(res => {
+    extractFilesDetails(recordObjIds, {
-      const captureColumn = getColumnByKey(metadata.columns, PRIVATE_COLUMN_KEY.CAPTURE_TIME);
+      success_callback: ({ details }) => {
-
+        const captureColumn = getColumnByKey(metadata.columns, PRIVATE_COLUMN_KEY.CAPTURE_TIME);
-      if (captureColumn) {
+        if (!captureColumn) return;
         let idOldRecordData = {};
         let idOriginalOldRecordData = {};
         const captureColumnKey = PRIVATE_COLUMN_KEY.CAPTURE_TIME;
@@ -324,18 +295,15 @@ const ContextMenu = ({
         });
         let idRecordUpdates = {};
         let idOriginalRecordUpdates = {};
-        res.data.details.forEach(detail => {
+        details.forEach(detail => {
           const updateRecordId = detail[PRIVATE_COLUMN_KEY.ID];
           idRecordUpdates[updateRecordId] = { [captureColumnKey]: detail[captureColumnKey] };
           idOriginalRecordUpdates[updateRecordId] = { [captureColumnKey]: detail[captureColumnKey] };
         });
         updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
       }
-    }).catch(error => {
-      const errorMessage = gettext('Failed to extract file details');
-      toaster.danger(errorMessage);
     });
-  }, [metadata, updateRecords]);
+  }, [metadata, extractFilesDetails, updateRecords]);
 
   const handleOptionClick = useCallback((event, option) => {
     event.stopPropagation();
@@ -363,13 +331,7 @@ const ContextMenu = ({
       case OPERATION.GENERATE_DESCRIPTION: {
         const { record } = option;
         if (!record) break;
-        generateDescription(record);
+        handelGenerateDescription(record);
-        break;
-      }
-      case OPERATION.IMAGE_CAPTION: {
-        const { record } = option;
-        if (!record) break;
-        imageCaption(record);
         break;
       }
       case OPERATION.FILE_TAGS: {
@@ -433,7 +395,7 @@ const ContextMenu = ({
       }
     }
     setVisible(false);
-  }, [repoID, onCopySelected, onClearSelected, generateDescription, imageCaption, ocr, deleteRecords, toggleDeleteFolderDialog, selectNone, updateFileDetails, toggleFileTagsRecord, toggleMoveDialog]);
+  }, [repoID, onCopySelected, onClearSelected, handelGenerateDescription, ocr, deleteRecords, toggleDeleteFolderDialog, selectNone, updateFileDetails, toggleFileTagsRecord, toggleMoveDialog]);
 
   const getMenuPosition = useCallback((x = 0, y = 0) => {
     let menuStyles = {

frontend/src/pages/sdoc/sdoc-editor/index.css

@@ -15,13 +15,15 @@
   background-color: inherit;
 }
 
-.sdoc-content-right-panel .detail-header .seafile-multicolor-icon-set-up {
+.sdoc-content-right-panel .detail-header .seafile-multicolor-icon-set-up,
+.sdoc-content-right-panel .detail-header .seafile-multicolor-icon-ai {
   fill: #999;
   font-weight: 700;
   font-size: 16px;
 }
 
-.sdoc-content-right-panel .detail-header .detail-control:hover .seafile-multicolor-icon-set-up {
+.sdoc-content-right-panel .detail-header .detail-control:hover .seafile-multicolor-icon-set-up,
+.sdoc-content-right-panel .detail-header .detail-control:hover .seafile-multicolor-icon-ai {
   fill: #5a5a5a;
 }
 

frontend/src/shared-dir-view.js

@@ -20,7 +20,7 @@ import CopyMoveDirentProgressDialog from './components/dialog/copy-move-dirent-p
 import RepoInfoBar from './components/repo-info-bar';
 import RepoTag from './models/repo-tag';
 import { GRID_MODE, LIST_MODE } from './components/dir-view-mode/constants';
-import { MetadataOperationsProvider } from './hooks/metadata-operation';
+import { MetadataAIOperationsProvider } from './hooks/metadata-ai-operation';
 
 import './css/shared-dir-view.css';
 import './css/grid-view.css';
@@ -441,7 +441,7 @@ class SharedDirView extends React.Component {
     const isDesktop = Utils.isDesktop();
     const modeBaseClass = 'btn btn-secondary btn-icon sf-view-mode-btn';
     return (
-      <MetadataOperationsProvider repoID={repoID} enableMetadata={false} enableOCR={false} repoInfo={{ permission: 'r' }} >
+      <MetadataAIOperationsProvider repoID={repoID} enableMetadata={false} enableOCR={false} repoInfo={{ permission: 'r' }} >
         <div className="h-100 d-flex flex-column">
           <div className="top-header d-flex justify-content-between">
             <a href={siteRoot}>
@@ -581,7 +581,7 @@ class SharedDirView extends React.Component {
           />
         </ModalPortal>
         }
-      </MetadataOperationsProvider>
+      </MetadataAIOperationsProvider>
     );
   }
 }

frontend/src/utils/utils.js

@@ -1055,7 +1055,12 @@ export const Utils = {
   },
 
   isDescriptionSupportedFile: function (filePath) {
-    return Utils.isSdocFile(filePath) || Utils.isMarkdownFile(filePath) || Utils.pdfCheck(filePath) || Utils.isDocxFile(filePath) || Utils.isPptxFile(filePath);
+    return Utils.isSdocFile(filePath) ||
+      Utils.isMarkdownFile(filePath) ||
+      Utils.pdfCheck(filePath) ||
+      Utils.isDocxFile(filePath) ||
+      Utils.isPptxFile(filePath) ||
+      Utils.imageCheck(filePath);
   },
 
   isFileMetadata: function (type) {