Merge pull request #7755 from haiwen/tags-migrate

Tags migrate

frontend/src/components/dialog/lib-settings.js

@@ -14,6 +14,7 @@ import {
 } from '../../metadata';
 import SeahubModalHeader from '@/components/common/seahub-modal-header';
 import { useMetadataStatus } from '../../hooks';
+import Loading from '../../components/loading';
 
 import '../../css/lib-settings.css';
 
@@ -22,12 +23,12 @@ const { enableSeafileAI, enableSeafileOCR } = window.app.config;
 const propTypes = {
   toggleDialog: PropTypes.func.isRequired,
   repoID: PropTypes.string.isRequired,
-  currentRepoInfo: PropTypes.object.isRequired
+  currentRepoInfo: PropTypes.object.isRequired,
 };
 
-const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip }) => {
+const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip, onMigrateSuccess }) => {
   const [activeTab, setActiveTab] = useState(tab || TAB.HISTORY_SETTING);
-
+  const [isMigrating, setIsMigrating] = useState(false);
   const toggleTab = useCallback((tab) => {
     setActiveTab(tab);
   }, []);
@@ -46,9 +47,38 @@ const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMig
   const enableAutoDelSetting = is_admin && enableRepoAutoDel;
   const enableExtendedPropertiesSetting = !encrypted && is_admin && enableMetadataManagement;
 
+  const handleMigrateStart = useCallback(() => {
+    setIsMigrating(true);
+  }, []);
+
+  const handleMigrateEnd = useCallback(() => {
+    setIsMigrating(false);
+    onMigrateSuccess && onMigrateSuccess();
+  }, [onMigrateSuccess]);
+
+  const handleMigrateError = useCallback(() => {
+    setIsMigrating(false);
+  }, []);
+
   return (
     <div>
       <Modal isOpen={true} className="lib-settings-dialog" toggle={toggleDialog}>
+        {isMigrating && (
+          <div style={{
+            position: 'absolute',
+            top: 0,
+            left: 0,
+            right: 0,
+            bottom: 0,
+            backgroundColor: 'rgba(255, 255, 255, 0.8)',
+            display: 'flex',
+            justifyContent: 'center',
+            alignItems: 'center',
+            zIndex: 1050
+          }}>
+            <Loading />
+          </div>
+        )}
         <SeahubModalHeader toggle={toggleDialog}>
           {gettext('Settings')}
         </SeahubModalHeader>
@@ -202,6 +232,9 @@ const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMig
                     toggleDialog={toggleDialog}
                     enableMetadata={enableMetadata}
                     showMigrateTip={showMigrateTip}
+                    onMigrateSuccess={handleMigrateEnd}
+                    onMigrateError={handleMigrateError}
+                    onMigrateStart={handleMigrateStart}
                   />
                 </TabPane>
               )}

frontend/src/components/dir-view-mode/dir-others/index.js

@@ -1,4 +1,4 @@
-import React, { useState, useEffect } from 'react';
+import React, { useState, useEffect, useCallback } from 'react';
 import PropTypes from 'prop-types';
 import { gettext } from '../../../utils/constants';
 import { Utils } from '../../../utils/utils';
@@ -15,6 +15,7 @@ import './index.css';
 const DirOthers = ({ userPerm, repoID, currentRepoInfo }) => {
 
   const showSettings = currentRepoInfo.is_admin; // repo owner, department admin, shared with 'Admin' permission
+  const repoName = currentRepoInfo.repo_name;
   let [isSettingsDialogOpen, setSettingsDialogOpen] = useState(false);
   let [activeTab, setActiveTab] = useState(TAB.HISTORY_SETTING);
   let [showMigrateTip, setShowMigrateTip] = useState(false);
@@ -23,6 +24,12 @@ const DirOthers = ({ userPerm, repoID, currentRepoInfo }) => {
     setSettingsDialogOpen(!isSettingsDialogOpen);
   };
 
+  const handleMigrateSuccess = useCallback(() => {
+    setShowMigrateTip(false);
+    const serviceUrl = window.app.config.serviceURL;
+    window.location.href = serviceUrl + '/library/' + repoID + '/' + repoName + '/?tag=__all_tags';
+  }, [repoID, repoName]);
+
   useEffect(() => {
     const unsubscribeUnselectFiles = eventBus.subscribe(EVENT_BUS_TYPE.OPEN_LIBRARY_SETTINGS_TAGS, () => {
       setSettingsDialogOpen(true);
@@ -79,6 +86,7 @@ const DirOthers = ({ userPerm, repoID, currentRepoInfo }) => {
           toggleDialog={toggleSettingsDialog}
           tab={activeTab}
           showMigrateTip={showMigrateTip}
+          onMigrateSuccess={handleMigrateSuccess}
         />
       )}
       {isRepoHistoryDialogOpen && (

frontend/src/components/repo-info-bar-migrate.js

@@ -7,22 +7,19 @@ import { EVENT_BUS_TYPE } from '../components/common/event-bus-type';
 
 const RepoInfoBarMigrate = () => {
 
-  const { enableMetadata } = useMetadataStatus();
-
+  const { enableMetadataManagement } = useMetadataStatus();
   const openMigrate = () => {
     eventBus.dispatch(EVENT_BUS_TYPE.OPEN_TREE_PANEL, () => eventBus.dispatch(EVENT_BUS_TYPE.OPEN_LIBRARY_SETTINGS_TAGS));
   };
 
   return (
     <div className="repo-info-bar-migrate mt-2">
-      {enableMetadata ?
-        (
+      {enableMetadataManagement ? (
         <>
           {gettext('Tips: There are tags of old version. Please migrate tags to new version.')}
           <Button color="link" size="sm" tag="a" onClick={openMigrate}>{gettext('Migrate')}</Button>
         </>
-        ) :
-        (
+      ) : (
         <>{gettext('Tips: These are tags of old version. The feature is deprecated and can no longer be used.')}</>
       )
       }

frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js

@@ -19,12 +19,11 @@ const langOptions = [
   }
 ];
 
-const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip }) => {
+const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip, onMigrateSuccess, onMigrateStart, onMigrateError }) => {
   const [value, setValue] = useState(oldValue);
   const [lang, setLang] = useState(oldLang);
   const [submitting, setSubmitting] = useState(false);
   const [showTurnOffConfirmDialog, setShowTurnOffConfirmDialog] = useState(false);
-
   const onToggle = useCallback(() => {
     toggle();
   }, [toggle]);
@@ -46,8 +45,16 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg
   }, [lang, repoID, submit, toggle, value]);
 
   const migrateTag = useCallback(() => {
-    // TODO backend migrate old tags
-  }, []);
+    onMigrateStart && onMigrateStart();
+    tagsAPI.migrateTags(repoID).then(res => {
+      toaster.success(gettext('Tags migrated successfully'));
+      onMigrateSuccess && onMigrateSuccess();
+    }).catch(error => {
+      const errorMsg = Utils.getErrorMsg(error);
+      toaster.danger(errorMsg);
+      onMigrateError && onMigrateError();
+    });
+  }, [repoID, onMigrateSuccess, onMigrateStart, onMigrateError]);
 
   const turnOffConfirmToggle = useCallback(() => {
     setShowTurnOffConfirmDialog(!showTurnOffConfirmDialog);
@@ -110,7 +117,12 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg
             {showMigrateTip &&
               <FormGroup className="mt-6">
                 <p>{gettext('This library contains tags of old version. Do you like to migrate the tags to new version?')}</p>
-                <Button color="primary" onClick={migrateTag}>{gettext('Migrate old version tags')}</Button>
+                <Button
+                  color="primary"
+                  onClick={migrateTag}
+                >
+                  {gettext('Migrate old version tags')}
+                </Button>
               </FormGroup>
             }
           </ModalBody>
@@ -136,6 +148,9 @@ MetadataTagsStatusDialog.propTypes = {
   submit: PropTypes.func.isRequired,
   enableMetadata: PropTypes.bool.isRequired,
   showMigrateTip: PropTypes.bool,
+  onMigrateSuccess: PropTypes.func,
+  onMigrateError: PropTypes.func,
+  onMigrateStart: PropTypes.func,
 };
 
 export default MetadataTagsStatusDialog;

frontend/src/pages/lib-content-view/lib-content-view.js

@@ -2347,6 +2347,7 @@ class LibContentView extends React.Component {
     if (!currentDirent && currentMode !== METADATA_MODE && currentMode !== TAGS_MODE) {
       detailPath = Utils.getDirName(this.state.path);
     }
+
     const detailDirent = currentDirent || currentNode?.object || null;
     return (
       <DndProvider backend={HTML5Backend}>

frontend/src/tag/api.js

@@ -138,6 +138,11 @@ class TagsManagerAPI {
     return this.req.post(url, params);
   };
 
+  migrateTags = (repoID) => {
+    const url = this.server + '/api/v2.1/repos/' + repoID + '/metadata/migrate-tags/';
+    return this.req.post(url);
+  };
+
 }
 
 const tagsAPI = new TagsManagerAPI();

seahub/repo_metadata/apis.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import posixpath
 from datetime import datetime
 
 from rest_framework.authentication import SessionAuthentication
@@ -22,6 +23,8 @@ from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_met
 from seahub.utils.repo import is_repo_admin
 from seaserv import seafile_api
 from seahub.repo_metadata.constants import FACE_RECOGNITION_VIEW_ID, METADATA_RECORD_UPDATE_LIMIT
+from seahub.file_tags.models import FileTags
+from seahub.repo_tags.models import RepoTags
 
 
 logger = logging.getLogger(__name__)
@@ -2780,3 +2783,186 @@ class PeopleCoverPhoto(APIView):
             return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
 
         return Response({'success': True})
+
+
+class MetadataMigrateTags(APIView):
+    authentication_classes = (TokenAuthentication, SessionAuthentication)
+    permission_classes = (IsAuthenticated,)
+    throttle_classes = (UserRateThrottle,)
+
+    def _create_metadata_tags(self, repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE):
+        sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}'
+        existing_tags_result = metadata_server_api.query_rows(sql)
+        existing_tag_records = existing_tags_result.get('results', [])
+        existing_tag_map = {
+            tag_dict.get(TAGS_TABLE.columns.name.name, '') :
+            tag_dict.get(TAGS_TABLE.columns.id.name, '')
+            for tag_dict in existing_tag_records
+        }
+        
+        tags_to_create = [] # [{name:'', color:''}, ...]
+        old_tag_name_to_metadata_tag_id = {} # {old_tag_name:id,} Existing tags
+
+        for repo_tag in repo_tags:
+            tag_name = repo_tag.name
+            tag_color = repo_tag.color
+            
+            if tag_name in existing_tag_map:
+                # Tag already exists, no need to create it
+                new_tag_id = existing_tag_map.get(tag_name)
+                old_tag_name_to_metadata_tag_id[tag_name] = new_tag_id
+            else:
+                # Tag needs to be created
+                tags_to_create.append({
+                    TAGS_TABLE.columns.name.name: tag_name,
+                    TAGS_TABLE.columns.color.name: tag_color
+                })
+        
+        if tags_to_create:
+            response = metadata_server_api.insert_rows(tags_table_id, tags_to_create)
+            new_tag_ids = response.get('row_ids', [])
+            for idx, tag in enumerate(tags_to_create):
+                new_tag_id = new_tag_ids[idx]
+                tag_name = tag.get(TAGS_TABLE.columns.name.name)
+                old_tag_name_to_metadata_tag_id[tag_name] = new_tag_id
+        return old_tag_name_to_metadata_tag_id
+    
+    def _get_old_tags_info(self, tagged_files):
+        old_tag_name_to_file_paths = {} # {old_tag_name: {file_path,....}}
+        file_paths_set = set() # Used for querying metadata later
+        for tagged_file in tagged_files:
+            old_tag_name = tagged_file.repo_tag.name
+            parent_path = tagged_file.file_uuid.parent_path
+            filename = tagged_file.file_uuid.filename
+            file_path = posixpath.join(parent_path, filename)
+            
+            if old_tag_name not in old_tag_name_to_file_paths:
+                old_tag_name_to_file_paths[old_tag_name] = set()
+            
+            old_tag_name_to_file_paths[old_tag_name].add(file_path)
+            file_paths_set.add(file_path)
+        return old_tag_name_to_file_paths, file_paths_set
+
+    def _get_metadata_records(self, metadata_server_api, file_paths_set, METADATA_TABLE):
+        if not file_paths_set:
+            return []
+        dir_paths = []
+        filenames = []
+        for file_path in file_paths_set:
+            parent_dir = os.path.dirname(file_path)
+            filename = os.path.basename(file_path)
+            dir_paths.append(parent_dir)
+            filenames.append(filename)
+        
+        batch_size = 100
+        all_metadata_records = []
+        
+        for i in range(0, len(dir_paths), batch_size):
+            batch_dir_paths = dir_paths[i:i+batch_size]
+            batch_filenames = filenames[i:i+batch_size]
+            
+            where_conditions = []
+            parameters = []
+            for j in range(len(batch_dir_paths)):
+                where_conditions.append(f"(`{METADATA_TABLE.columns.parent_dir.name}` = ? AND `{METADATA_TABLE.columns.file_name.name}` = ?)")
+                parameters.append(batch_dir_paths[j])
+                parameters.append(batch_filenames[j])
+                
+            where_clause = " OR ".join(where_conditions)
+            sql = f'''
+                SELECT `{METADATA_TABLE.columns.id.name}`, 
+                `{METADATA_TABLE.columns.file_name.name}`, 
+                `{METADATA_TABLE.columns.parent_dir.name}` 
+                FROM `{METADATA_TABLE.name}` 
+                WHERE `{METADATA_TABLE.columns.is_dir.name}` = FALSE
+                AND ({where_clause})
+                '''
+                
+            query_result = metadata_server_api.query_rows(sql, parameters)
+            batch_metadata_records = query_result.get('results', [])
+            all_metadata_records.extend(batch_metadata_records)
+
+        return all_metadata_records
+    
+    def _handle_tags_link(self, metadata_records, metadata_tag_id_to_file_paths, METADATA_TABLE):
+        file_path_to_record_id = {}  # {file_path: record_id}
+        for record in metadata_records:
+            parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name)
+            file_name = record.get(METADATA_TABLE.columns.file_name.name)
+            record_id = record.get(METADATA_TABLE.columns.id.name)
+            
+            file_path = posixpath.join(parent_dir, file_name)
+            file_path_to_record_id[file_path] = record_id
+        # create record id to tag id mapping
+        record_to_tags_map = {}  # {record_id: [tag_id1, tag_id2]}
+        for tag_id, file_paths in metadata_tag_id_to_file_paths.items():
+            for file_path in file_paths:
+                record_id = file_path_to_record_id.get(file_path)
+                if not record_id:
+                    continue
+                    
+                if record_id not in record_to_tags_map:
+                    record_to_tags_map[record_id] = []
+                    
+                record_to_tags_map[record_id].append(tag_id)
+        
+        return record_to_tags_map
+    
+    def post(self, request, repo_id):
+        metadata = RepoMetadata.objects.filter(repo_id=repo_id).first()
+        if not metadata or not metadata.enabled or not metadata.tags_enabled:
+            error_msg = f'The tags is disabled for repo {repo_id}.'
+            return api_error(status.HTTP_404_NOT_FOUND, error_msg)
+
+        repo = seafile_api.get_repo(repo_id)
+        if not repo:
+            error_msg = 'Library %s not found.' % repo_id
+            return api_error(status.HTTP_404_NOT_FOUND, error_msg)
+
+        if not is_repo_admin(request.user.username, repo_id):
+            error_msg = 'Permission denied.'
+            return api_error(status.HTTP_403_FORBIDDEN, error_msg)
+
+        from seafevents.repo_metadata.constants import TAGS_TABLE, METADATA_TABLE
+        metadata_server_api = MetadataServerAPI(repo_id, request.user.username)
+        tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name)
+        if not tags_table:
+            return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found')
+        tags_table_id = tags_table['id']
+        
+        # create new tags
+        repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id)
+        if not repo_tags:
+            return Response({'success': True})
+        metadata_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE)
+
+        tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id)
+        if not tagged_files:
+            repo_tags.delete()
+            return Response({'success': True})
+        old_tag_name_to_file_paths, file_paths_set = self._get_old_tags_info(tagged_files)
+        
+        metadata_tag_id_to_file_paths = {}  # {tag_id: file_paths}
+        for tag_name, tag_id in metadata_tags.items():
+            if tag_name not in old_tag_name_to_file_paths:
+                continue
+            file_paths = old_tag_name_to_file_paths[tag_name]
+            metadata_tag_id_to_file_paths[tag_id] = file_paths
+        
+        try:
+            # query records
+            metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE)
+            record_to_tags_map = self._handle_tags_link(metadata_records, metadata_tag_id_to_file_paths, METADATA_TABLE)
+            metadata_server_api.insert_link(
+                TAGS_TABLE.file_link_id, 
+                METADATA_TABLE.id, 
+                record_to_tags_map
+            )
+            # clear old tag data
+            tagged_files.delete()
+            repo_tags.delete()
+        except Exception as e:
+            logger.error(e)
+            error_msg = 'Internal Server Error'
+            return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
+        return Response({'success': True})

seahub/repo_metadata/urls.py

@@ -3,7 +3,7 @@ from .apis import MetadataRecognizeFaces, MetadataRecords, MetadataManage, Metad
     MetadataFolders, MetadataViews, MetadataViewsMoveView, MetadataViewsDetailView, MetadataViewsDuplicateView, FacesRecords, \
     FaceRecognitionManage, FacesRecord, MetadataExtractFileDetails, PeoplePhotos, MetadataTagsStatusManage, MetadataTags, \
     MetadataTagsLinks, MetadataFileTags, MetadataTagFiles, MetadataMergeTags, MetadataTagsFiles, MetadataDetailsSettingsView, \
-    MetadataOCRManageView, PeopleCoverPhoto
+    MetadataOCRManageView, PeopleCoverPhoto, MetadataMigrateTags
 
 urlpatterns = [
     re_path(r'^$', MetadataManage.as_view(), name='api-v2.1-metadata'),
@@ -43,4 +43,5 @@ urlpatterns = [
     re_path(r'^tag-files/(?P<tag_id>.+)/$', MetadataTagFiles.as_view(), name='api-v2.1-metadata-tag-files'),
     re_path(r'^merge-tags/$', MetadataMergeTags.as_view(), name='api-v2.1-metadata-merge-tags'),
     re_path(r'^tags-files/$', MetadataTagsFiles.as_view(), name='api-v2.1-metadata-tags-files'),
+    re_path(r'^migrate-tags/$', MetadataMigrateTags.as_view(), name='api-v2.1-metadata-migrate-tags'),
 ]

seahub/repo_metadata/utils.py

@@ -73,7 +73,6 @@ def gen_unique_id(id_set, length=4):
             return _id
         _id = generator_base64_code(length)
 
-
 def get_face_columns():
     from seafevents.repo_metadata.constants import FACES_TABLE
     columns = [