From b8fe71d5a2812dec3d0889eec49777ef4409c9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Tue, 15 Apr 2025 09:48:44 +0800 Subject: [PATCH 01/15] tags migrate handle exist tag name --- .../src/components/dialog/lib-settings.js | 8 +- .../dir-view-mode/dir-column-nav/index.js | 7 +- .../dir-view-mode/dir-column-view.js | 1 + .../dir-view-mode/dir-others/index.js | 14 +- .../src/components/repo-info-bar-migrate.js | 6 +- .../metadata-tags-status-dialog/index.js | 25 ++- .../lib-content-view/lib-content-view.js | 1 + frontend/src/tag/api.js | 8 + seahub/repo_metadata/apis.py | 177 +++++++++++++++++- seahub/repo_metadata/metadata_server_api.py | 20 ++ seahub/repo_metadata/urls.py | 3 +- seahub/repo_metadata/utils.py | 5 + 12 files changed, 257 insertions(+), 18 deletions(-) diff --git a/frontend/src/components/dialog/lib-settings.js b/frontend/src/components/dialog/lib-settings.js index 1078cc0d3b..3f6eddac1d 100644 --- a/frontend/src/components/dialog/lib-settings.js +++ b/frontend/src/components/dialog/lib-settings.js @@ -22,12 +22,12 @@ const { enableSeafileAI, enableSeafileOCR } = window.app.config; const propTypes = { toggleDialog: PropTypes.func.isRequired, repoID: PropTypes.string.isRequired, - currentRepoInfo: PropTypes.object.isRequired + currentRepoInfo: PropTypes.object.isRequired, + usedRepoTags: PropTypes.array, }; -const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip }) => { +const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip, usedRepoTags, onMigrateSuccess }) => { const [activeTab, setActiveTab] = useState(tab || TAB.HISTORY_SETTING); - const toggleTab = useCallback((tab) => { setActiveTab(tab); }, []); @@ -202,6 +202,8 @@ const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMig toggleDialog={toggleDialog} enableMetadata={enableMetadata} showMigrateTip={showMigrateTip} + usedRepoTags={usedRepoTags} + onMigrateSuccess={onMigrateSuccess} /> )} diff --git a/frontend/src/components/dir-view-mode/dir-column-nav/index.js b/frontend/src/components/dir-view-mode/dir-column-nav/index.js index 9afedb4252..9cb8222a42 100644 --- a/frontend/src/components/dir-view-mode/dir-column-nav/index.js +++ b/frontend/src/components/dir-view-mode/dir-column-nav/index.js @@ -33,6 +33,7 @@ const propTypes = { getMenuContainerSize: PropTypes.func, updateDirent: PropTypes.func, updateTreeNode: PropTypes.func, + usedRepoTags: PropTypes.array, }; class DirColumnNav extends React.Component { @@ -43,7 +44,7 @@ class DirColumnNav extends React.Component { render() { const { - isTreeDataLoading, userPerm, treeData, repoID, currentPath, currentRepoInfo, navRate = 0.25 + isTreeDataLoading, userPerm, treeData, repoID, currentPath, currentRepoInfo, navRate = 0.25, usedRepoTags } = this.props; const flex = navRate ? '0 0 ' + navRate * 100 + '%' : '0 0 25%'; const select = this.props.inResizing ? 'none' : ''; @@ -74,9 +75,9 @@ class DirColumnNav extends React.Component { updateDirent={this.props.updateDirent} updateTreeNode={this.props.updateTreeNode} /> - + - + )} diff --git a/frontend/src/components/dir-view-mode/dir-column-view.js b/frontend/src/components/dir-view-mode/dir-column-view.js index aef5a78696..aaef405051 100644 --- a/frontend/src/components/dir-view-mode/dir-column-view.js +++ b/frontend/src/components/dir-view-mode/dir-column-view.js @@ -185,6 +185,7 @@ class DirColumnView extends React.Component { direntList={this.props.direntList} updateDirent={this.props.updateDirent} updateTreeNode={this.props.updateTreeNode} + usedRepoTags={this.props.usedRepoTags} /> { +const DirOthers = ({ userPerm, repoID, currentRepoInfo, usedRepoTags }) => { const showSettings = currentRepoInfo.is_admin; // repo owner, department admin, shared with 'Admin' permission + const repoName = currentRepoInfo.repo_name; let [isSettingsDialogOpen, setSettingsDialogOpen] = useState(false); let [activeTab, setActiveTab] = useState(TAB.HISTORY_SETTING); let [showMigrateTip, setShowMigrateTip] = useState(false); @@ -23,6 +24,12 @@ const DirOthers = ({ userPerm, repoID, currentRepoInfo }) => { setSettingsDialogOpen(!isSettingsDialogOpen); }; + const handleMigrateSuccess = useCallback(() => { + setShowMigrateTip(false); + const serviceUrl = window.app.config.serviceURL; + window.location.href = serviceUrl + '/library/' + repoID + '/' + repoName + '/?tag=__all_tags'; + }, [repoID, repoName]); + useEffect(() => { const unsubscribeUnselectFiles = eventBus.subscribe(EVENT_BUS_TYPE.OPEN_LIBRARY_SETTINGS_TAGS, () => { setSettingsDialogOpen(true); @@ -79,6 +86,8 @@ const DirOthers = ({ userPerm, repoID, currentRepoInfo }) => { toggleDialog={toggleSettingsDialog} tab={activeTab} showMigrateTip={showMigrateTip} + usedRepoTags={usedRepoTags} + onMigrateSuccess={handleMigrateSuccess} /> )} {isRepoHistoryDialogOpen && ( @@ -97,6 +106,7 @@ DirOthers.propTypes = { userPerm: PropTypes.string, repoID: PropTypes.string, currentRepoInfo: PropTypes.object.isRequired, + usedRepoTags: PropTypes.array, }; export default DirOthers; diff --git a/frontend/src/components/repo-info-bar-migrate.js b/frontend/src/components/repo-info-bar-migrate.js index 8a47b4df86..bc2bd3f4bc 100644 --- a/frontend/src/components/repo-info-bar-migrate.js +++ b/frontend/src/components/repo-info-bar-migrate.js @@ -7,15 +7,15 @@ import { EVENT_BUS_TYPE } from '../components/common/event-bus-type'; const RepoInfoBarMigrate = () => { - const { enableMetadata } = useMetadataStatus(); - + const { enableMetadata, detailsSettings } = useMetadataStatus(); + const tagsMigrated = detailsSettings?.tags_migrated; const openMigrate = () => { eventBus.dispatch(EVENT_BUS_TYPE.OPEN_TREE_PANEL, () => eventBus.dispatch(EVENT_BUS_TYPE.OPEN_LIBRARY_SETTINGS_TAGS)); }; return (
- {enableMetadata ? + {enableMetadata && !tagsMigrated ? ( <> {gettext('Tips: There are tags of old version. Please migrate tags to new version.')} diff --git a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js index 1c058fc9b3..af02775952 100644 --- a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js +++ b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js @@ -19,10 +19,11 @@ const langOptions = [ } ]; -const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip }) => { +const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip, usedRepoTags, onMigrateSuccess }) => { const [value, setValue] = useState(oldValue); const [lang, setLang] = useState(oldLang); const [submitting, setSubmitting] = useState(false); + const [migrated, setMigrated] = useState(false); const [showTurnOffConfirmDialog, setShowTurnOffConfirmDialog] = useState(false); const onToggle = useCallback(() => { @@ -46,8 +47,16 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg }, [lang, repoID, submit, toggle, value]); const migrateTag = useCallback(() => { - // TODO backend migrate old tags - }, []); + tagsAPI.migrateTags(repoID, usedRepoTags).then(res => { + setMigrated(true); + toaster.success(gettext('Tags migrated successfully')); + onMigrateSuccess && onMigrateSuccess(); + }).catch(error => { + const errorMsg = Utils.getErrorMsg(error); + toaster.danger(errorMsg); + setMigrated(false); + }); + }, [repoID, usedRepoTags, onMigrateSuccess]); const turnOffConfirmToggle = useCallback(() => { setShowTurnOffConfirmDialog(!showTurnOffConfirmDialog); @@ -110,7 +119,13 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg {showMigrateTip &&

{gettext('This library contains tags of old version. Do you like to migrate the tags to new version?')}

- + } @@ -136,6 +151,8 @@ MetadataTagsStatusDialog.propTypes = { submit: PropTypes.func.isRequired, enableMetadata: PropTypes.bool.isRequired, showMigrateTip: PropTypes.bool, + repoTags: PropTypes.array, + onMigrateSuccess: PropTypes.func, }; export default MetadataTagsStatusDialog; diff --git a/frontend/src/pages/lib-content-view/lib-content-view.js b/frontend/src/pages/lib-content-view/lib-content-view.js index 39087421b2..3d0c133498 100644 --- a/frontend/src/pages/lib-content-view/lib-content-view.js +++ b/frontend/src/pages/lib-content-view/lib-content-view.js @@ -2350,6 +2350,7 @@ class LibContentView extends React.Component { if (!currentDirent && currentMode !== METADATA_MODE && currentMode !== TAGS_MODE) { detailPath = Utils.getDirName(this.state.path); } + const detailDirent = currentDirent || currentNode?.object || null; return ( diff --git a/frontend/src/tag/api.js b/frontend/src/tag/api.js index 1652c05d39..8e29f701cb 100644 --- a/frontend/src/tag/api.js +++ b/frontend/src/tag/api.js @@ -138,6 +138,14 @@ class TagsManagerAPI { return this.req.post(url, params); }; + migrateTags = (repoID, usedRepoTags) => { + const url = this.server + '/api/v2.1/repos/' + repoID + '/metadata/migrate-tags/'; + const params = { + tag_ids: usedRepoTags.map(tag => tag.id), + }; + return this.req.post(url, params); + }; + } const tagsAPI = new TagsManagerAPI(); diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index b14fdcb40a..18a73fa77a 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -1,6 +1,7 @@ import json import logging import os +import posixpath from datetime import datetime from rest_framework.authentication import SessionAuthentication @@ -17,11 +18,12 @@ from seahub.repo_metadata.utils import add_init_metadata_task, recognize_faces, get_unmodifiable_columns, can_read_metadata, init_faces, \ extract_file_details, get_table_by_name, remove_faces_table, FACES_SAVE_PATH, \ init_tags, init_tag_self_link_columns, remove_tags_table, add_init_face_recognition_task, init_ocr, \ - remove_ocr_column, get_update_record, update_people_cover_photo -from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_metadata_view_records + remove_ocr_column, get_update_record, update_people_cover_photo, gen_unique_tag_name +from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_metadata_view_records, list_eligible_metadata_records from seahub.utils.repo import is_repo_admin from seaserv import seafile_api from seahub.repo_metadata.constants import FACE_RECOGNITION_VIEW_ID, METADATA_RECORD_UPDATE_LIMIT +from seahub.file_tags.models import FileTags logger = logging.getLogger(__name__) @@ -2780,3 +2782,174 @@ class PeopleCoverPhoto(APIView): return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) return Response({'success': True}) + + +class MetadataMigrateTags(APIView): + authentication_classes = (TokenAuthentication, SessionAuthentication) + permission_classes = (IsAuthenticated,) + throttle_classes = (UserRateThrottle,) + + def post(self, request, repo_id): + tag_ids = request.data.get('tag_ids') + if not tag_ids: + error_msg = 'tag_ids invalid' + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + metadata = RepoMetadata.objects.filter(repo_id=repo_id).first() + if not metadata or not metadata.enabled or not metadata.tags_enabled: + error_msg = f'The tags is disabled for repo {repo_id}.' + return api_error(status.HTTP_404_NOT_FOUND, error_msg) + + # resource check + repo = seafile_api.get_repo(repo_id) + if not repo: + error_msg = 'Library %s not found.' % repo_id + return api_error(status.HTTP_404_NOT_FOUND, error_msg) + + if not is_repo_admin(request.user.username, repo_id): + error_msg = 'Permission denied.' + return api_error(status.HTTP_403_FORBIDDEN, error_msg) + + # get all records + from seafevents.repo_metadata.constants import METADATA_TABLE + from seafevents.repo_metadata.constants import TAGS_TABLE + + try: + basic_filters = [{'column_key': METADATA_TABLE.columns.is_dir.name, 'filter_predicate': 'is', 'filter_term': 'file'}] + view = { + 'filters': [], + 'basic_filters': basic_filters + } + filter_columns = [METADATA_TABLE.columns.id.name, METADATA_TABLE.columns.file_name.name, METADATA_TABLE.columns.parent_dir.name] + results = list_eligible_metadata_records(repo_id, request.user.username, view, filter_columns) + records = results.get('results') + except Exception as e: + logger.exception(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + + # 每个record对应的文件路径 + records_dict = {} # {file_path: record_id} + for record in records: + file_path = posixpath.join(record.get(METADATA_TABLE.columns.parent_dir.name), record.get(METADATA_TABLE.columns.file_name.name)) + records_dict[file_path] = record.get(METADATA_TABLE.columns.id.name) + + # list old tag info + old_repo_tag_info = {} # {repo_tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} + try: + tagged_file_objs = FileTags.objects.filter( + repo_tag__id__in=tag_ids).select_related('repo_tag', 'file_uuid') + for tagged_file_obj in tagged_file_objs: + repo_tag_id = tagged_file_obj.repo_tag.id + parent_path = tagged_file_obj.file_uuid.parent_path + filename = tagged_file_obj.file_uuid.filename + file_path = posixpath.join(parent_path, filename) + if repo_tag_id not in old_repo_tag_info: + old_repo_tag_info[repo_tag_id] = { + 'name': tagged_file_obj.repo_tag.name, + 'color': tagged_file_obj.repo_tag.color, + 'file_paths': [] + } + old_repo_tag_info[repo_tag_id]['file_paths'].append(file_path) + except Exception as err: + logger.error(err) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + print(old_repo_tag_info, '---old_repo_tag_info') + + # file_path to record_id + repo_tag_id_map = {} # {repo_tag_id: [record_id1, record_id2]} + tags_data = [] # [{name: '', color: ''}] + for repo_tag_id, tag_data in old_repo_tag_info.items(): + repo_tag_id_map[repo_tag_id] = [] + for file_path in tag_data['file_paths']: + record_id = records_dict.get(file_path, '') + if record_id: + repo_tag_id_map[repo_tag_id].append(record_id) + tags_data.append({ + TAGS_TABLE.columns.name.name: tag_data['name'], + TAGS_TABLE.columns.color.name: tag_data['color'], + }) + + # {"tags_data":[{"_tag_color":"#46A1FD","_tag_name":"va"}]} + metadata_server_api = MetadataServerAPI(repo_id, request.user.username) + try: + tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) + except Exception as e: + logger.error(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + + if not tags_table: + return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') + + tags_table_id = tags_table['id'] + exist_tags = [] + new_tags = [] + sql = f'SELECT `{TAGS_TABLE.columns.name.name}` FROM {TAGS_TABLE.name}' + + try: + exist_rows = metadata_server_api.query_rows(sql) + exist_tags = exist_rows.get('results', []) + if exist_tags: + exist_tags = [tag_data.get(TAGS_TABLE.columns.name.name, '') for tag_data in exist_tags] + except Exception as e: + logger.exception(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + # handle exist tags + if exist_tags: + for index, tag_data in enumerate(tags_data): + tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') + if tag_name not in exist_tags: + new_tags.append(tag_data) + else: + unique_tag_name = gen_unique_tag_name(tag_name, exist_tags) + new_tags.append({ + TAGS_TABLE.columns.color.name: tag_data.get(TAGS_TABLE.columns.color.name, ''), + TAGS_TABLE.columns.name.name: unique_tag_name + }) + else: + new_tags = tags_data + + try: + resp = metadata_server_api.insert_rows(tags_table_id, new_tags) + row_ids = resp.get('row_ids', []) + except Exception as e: + logger.error(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + # record_id : tags + # handle file_tags_data + new_tags_info_array = [] + for index, old_tag_id in enumerate(repo_tag_id_map): + new_tag_id = row_ids[index] + record_ids = repo_tag_id_map[old_tag_id] + new_tags_info_array.append({ + 'record_ids': record_ids, + 'tag_id': new_tag_id + }) + record_id_tag_id_map = {} + for tag_info in new_tags_info_array: + record_ids = tag_info.get('record_ids', []) + tag_id = tag_info.get('tag_id', '') + for record_id in record_ids: + if record_id not in record_id_tag_id_map: + record_id_tag_id_map[record_id] = [] + record_id_tag_id_map[record_id].append(tag_id) + + try: + metadata_server_api.insert_link(TAGS_TABLE.file_link_id, METADATA_TABLE.id, record_id_tag_id_map) + record = RepoMetadata.objects.filter(repo_id=repo_id).first() + if not record.details_settings: + details_settings = {} + else: + details_settings = json.loads(record.details_settings) + details_settings['tags_migrated'] = True + record.details_settings = json.dumps(details_settings) + + record.save() + except Exception as e: + logger.error(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + return Response({'success': True}) diff --git a/seahub/repo_metadata/metadata_server_api.py b/seahub/repo_metadata/metadata_server_api.py index 12738dd6dd..0cc5be911b 100644 --- a/seahub/repo_metadata/metadata_server_api.py +++ b/seahub/repo_metadata/metadata_server_api.py @@ -78,6 +78,26 @@ def list_metadata_view_records(repo_id, user, view, tags_enabled, start=0, limit return response_results +def list_eligible_metadata_records(repo_id, user, view, filter_columns): + from seafevents.repo_metadata.constants import METADATA_TABLE + from seafevents.repo_metadata.utils import gen_view_data_sql + metadata_server_api = MetadataServerAPI(repo_id, user) + columns = metadata_server_api.list_columns(METADATA_TABLE.id).get('columns') + tags_data = {'metadata': [], 'results': []} + + sql = gen_view_data_sql(METADATA_TABLE, columns, view, 0, 0, {'tags_data': tags_data, 'username': user}) + query_fields_str = '' + for column in columns: + column_name = column.get('name') + if column_name in filter_columns: + column_name_str = '`%s`, ' % column_name + query_fields_str += column_name_str + query_fields_str = query_fields_str.strip(', ') + sql = sql.replace('*', query_fields_str) + response_results = metadata_server_api.query_rows(sql, []) + return response_results + + def parse_response(response): if response.status_code >= 300 or response.status_code < 200: raise ConnectionError(response.status_code, response.text) diff --git a/seahub/repo_metadata/urls.py b/seahub/repo_metadata/urls.py index 3c59b414ee..6ffde33f25 100644 --- a/seahub/repo_metadata/urls.py +++ b/seahub/repo_metadata/urls.py @@ -3,7 +3,7 @@ from .apis import MetadataRecognizeFaces, MetadataRecords, MetadataManage, Metad MetadataFolders, MetadataViews, MetadataViewsMoveView, MetadataViewsDetailView, MetadataViewsDuplicateView, FacesRecords, \ FaceRecognitionManage, FacesRecord, MetadataExtractFileDetails, PeoplePhotos, MetadataTagsStatusManage, MetadataTags, \ MetadataTagsLinks, MetadataFileTags, MetadataTagFiles, MetadataMergeTags, MetadataTagsFiles, MetadataDetailsSettingsView, \ - MetadataOCRManageView, PeopleCoverPhoto + MetadataOCRManageView, PeopleCoverPhoto, MetadataMigrateTags urlpatterns = [ re_path(r'^$', MetadataManage.as_view(), name='api-v2.1-metadata'), @@ -43,4 +43,5 @@ urlpatterns = [ re_path(r'^tag-files/(?P.+)/$', MetadataTagFiles.as_view(), name='api-v2.1-metadata-tag-files'), re_path(r'^merge-tags/$', MetadataMergeTags.as_view(), name='api-v2.1-metadata-merge-tags'), re_path(r'^tags-files/$', MetadataTagsFiles.as_view(), name='api-v2.1-metadata-tags-files'), + re_path(r'^migrate-tags/$', MetadataMigrateTags.as_view(), name='api-v2.1-metadata-migrate-tags'), ] diff --git a/seahub/repo_metadata/utils.py b/seahub/repo_metadata/utils.py index 845548c40e..6156e25d20 100644 --- a/seahub/repo_metadata/utils.py +++ b/seahub/repo_metadata/utils.py @@ -73,6 +73,11 @@ def gen_unique_id(id_set, length=4): return _id _id = generator_base64_code(length) +def gen_unique_tag_name(tag_name, exist_tags, counter=1): + new_name = f'{tag_name}({counter})' + if new_name not in exist_tags: + return new_name + return gen_unique_tag_name(tag_name, exist_tags, counter + 1) def get_face_columns(): from seafevents.repo_metadata.constants import FACES_TABLE From 1118d408b43344a885612ee34daf9818d6be094c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Tue, 22 Apr 2025 09:48:29 +0800 Subject: [PATCH 02/15] optimize code --- seahub/repo_metadata/apis.py | 234 +++++++++++--------- seahub/repo_metadata/metadata_server_api.py | 20 -- 2 files changed, 127 insertions(+), 127 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 18a73fa77a..9f400943b6 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -19,7 +19,7 @@ from seahub.repo_metadata.utils import add_init_metadata_task, recognize_faces, extract_file_details, get_table_by_name, remove_faces_table, FACES_SAVE_PATH, \ init_tags, init_tag_self_link_columns, remove_tags_table, add_init_face_recognition_task, init_ocr, \ remove_ocr_column, get_update_record, update_people_cover_photo, gen_unique_tag_name -from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_metadata_view_records, list_eligible_metadata_records +from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_metadata_view_records from seahub.utils.repo import is_repo_admin from seaserv import seafile_api from seahub.repo_metadata.constants import FACE_RECOGNITION_VIEW_ID, METADATA_RECORD_UPDATE_LIMIT @@ -2794,12 +2794,12 @@ class MetadataMigrateTags(APIView): if not tag_ids: error_msg = 'tag_ids invalid' return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + metadata = RepoMetadata.objects.filter(repo_id=repo_id).first() if not metadata or not metadata.enabled or not metadata.tags_enabled: error_msg = f'The tags is disabled for repo {repo_id}.' return api_error(status.HTTP_404_NOT_FOUND, error_msg) - # resource check repo = seafile_api.get_repo(repo_id) if not repo: error_msg = 'Library %s not found.' % repo_id @@ -2808,137 +2808,158 @@ class MetadataMigrateTags(APIView): if not is_repo_admin(request.user.username, repo_id): error_msg = 'Permission denied.' return api_error(status.HTTP_403_FORBIDDEN, error_msg) - - # get all records - from seafevents.repo_metadata.constants import METADATA_TABLE - from seafevents.repo_metadata.constants import TAGS_TABLE - - try: - basic_filters = [{'column_key': METADATA_TABLE.columns.is_dir.name, 'filter_predicate': 'is', 'filter_term': 'file'}] - view = { - 'filters': [], - 'basic_filters': basic_filters - } - filter_columns = [METADATA_TABLE.columns.id.name, METADATA_TABLE.columns.file_name.name, METADATA_TABLE.columns.parent_dir.name] - results = list_eligible_metadata_records(repo_id, request.user.username, view, filter_columns) - records = results.get('results') - except Exception as e: - logger.exception(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - - # 每个record对应的文件路径 - records_dict = {} # {file_path: record_id} - for record in records: - file_path = posixpath.join(record.get(METADATA_TABLE.columns.parent_dir.name), record.get(METADATA_TABLE.columns.file_name.name)) - records_dict[file_path] = record.get(METADATA_TABLE.columns.id.name) # list old tag info - old_repo_tag_info = {} # {repo_tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} + source_tags_info = {} # {tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} + file_paths_set = set() # Used for querying metadata later try: - tagged_file_objs = FileTags.objects.filter( + tagged_files = FileTags.objects.filter( repo_tag__id__in=tag_ids).select_related('repo_tag', 'file_uuid') - for tagged_file_obj in tagged_file_objs: - repo_tag_id = tagged_file_obj.repo_tag.id - parent_path = tagged_file_obj.file_uuid.parent_path - filename = tagged_file_obj.file_uuid.filename + + for tagged_file in tagged_files: + tag_id = tagged_file.repo_tag.id + parent_path = tagged_file.file_uuid.parent_path + filename = tagged_file.file_uuid.filename file_path = posixpath.join(parent_path, filename) - if repo_tag_id not in old_repo_tag_info: - old_repo_tag_info[repo_tag_id] = { - 'name': tagged_file_obj.repo_tag.name, - 'color': tagged_file_obj.repo_tag.color, + + if tag_id not in source_tags_info: + source_tags_info[tag_id] = { + 'name': tagged_file.repo_tag.name, + 'color': tagged_file.repo_tag.color, 'file_paths': [] } - old_repo_tag_info[repo_tag_id]['file_paths'].append(file_path) + + source_tags_info[tag_id]['file_paths'].append(file_path) + file_paths_set.add(file_path) except Exception as err: logger.error(err) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - print(old_repo_tag_info, '---old_repo_tag_info') - # file_path to record_id - repo_tag_id_map = {} # {repo_tag_id: [record_id1, record_id2]} - tags_data = [] # [{name: '', color: ''}] - for repo_tag_id, tag_data in old_repo_tag_info.items(): - repo_tag_id_map[repo_tag_id] = [] - for file_path in tag_data['file_paths']: - record_id = records_dict.get(file_path, '') - if record_id: - repo_tag_id_map[repo_tag_id].append(record_id) - tags_data.append({ - TAGS_TABLE.columns.name.name: tag_data['name'], - TAGS_TABLE.columns.color.name: tag_data['color'], - }) - - # {"tags_data":[{"_tag_color":"#46A1FD","_tag_name":"va"}]} + # query records + from seafevents.repo_metadata.constants import METADATA_TABLE + from seafevents.repo_metadata.constants import TAGS_TABLE metadata_server_api = MetadataServerAPI(repo_id, request.user.username) try: - tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) - except Exception as e: - logger.error(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + dir_paths = [os.path.dirname(path) for path in file_paths_set] + filenames = [os.path.basename(path) for path in file_paths_set] + + filenames_str = ', '.join([f'"{name}"' for name in filenames]) + dir_paths_str = ', '.join([f'"{path}"' for path in dir_paths]) - if not tags_table: - return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') - - tags_table_id = tags_table['id'] - exist_tags = [] - new_tags = [] - sql = f'SELECT `{TAGS_TABLE.columns.name.name}` FROM {TAGS_TABLE.name}' - - try: - exist_rows = metadata_server_api.query_rows(sql) - exist_tags = exist_rows.get('results', []) - if exist_tags: - exist_tags = [tag_data.get(TAGS_TABLE.columns.name.name, '') for tag_data in exist_tags] + sql = f''' + SELECT `{METADATA_TABLE.columns.id.name}`, + `{METADATA_TABLE.columns.file_name.name}`, + `{METADATA_TABLE.columns.parent_dir.name}` + FROM `{METADATA_TABLE.name}` + WHERE `{METADATA_TABLE.columns.is_dir.name}` = False + AND `{METADATA_TABLE.columns.file_name.name}` IN ({filenames_str}) + AND `{METADATA_TABLE.columns.parent_dir.name}` IN ({dir_paths_str}) + ''' + + query_result = metadata_server_api.query_rows(sql, []) + metadata_records = query_result.get('results', []) except Exception as e: logger.exception(e) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - # handle exist tags - if exist_tags: - for index, tag_data in enumerate(tags_data): - tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') - if tag_name not in exist_tags: - new_tags.append(tag_data) - else: - unique_tag_name = gen_unique_tag_name(tag_name, exist_tags) - new_tags.append({ - TAGS_TABLE.columns.color.name: tag_data.get(TAGS_TABLE.columns.color.name, ''), - TAGS_TABLE.columns.name.name: unique_tag_name - }) - else: - new_tags = tags_data + + + file_to_record_map = {} # {file_path: record_id} + for record in metadata_records: + parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name) + file_name = record.get(METADATA_TABLE.columns.file_name.name) + record_id = record.get(METADATA_TABLE.columns.id.name) + + file_path = posixpath.join(parent_dir, file_name) + file_to_record_map[file_path] = record_id + + tags_data = [] # [{name: '', color: ''}] + for tag_id, tag_info in source_tags_info.items(): + tags_data.append({ + TAGS_TABLE.columns.name.name: tag_info['name'], + TAGS_TABLE.columns.color.name: tag_info['color'], + }) try: - resp = metadata_server_api.insert_rows(tags_table_id, new_tags) - row_ids = resp.get('row_ids', []) + tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) + if not tags_table: + return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') + + sql = f'SELECT `{TAGS_TABLE.columns.name.name}` FROM {TAGS_TABLE.name}' + existing_tags_result = metadata_server_api.query_rows(sql) + existing_tag_records = existing_tags_result.get('results', []) + + existing_tag_names = [] + if existing_tag_records: + existing_tag_names = [ + tag_dict.get(TAGS_TABLE.columns.name.name, '') + for tag_dict in existing_tag_records + ] except Exception as e: logger.error(e) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - # record_id : tags - # handle file_tags_data - new_tags_info_array = [] - for index, old_tag_id in enumerate(repo_tag_id_map): - new_tag_id = row_ids[index] - record_ids = repo_tag_id_map[old_tag_id] - new_tags_info_array.append({ - 'record_ids': record_ids, - 'tag_id': new_tag_id - }) - record_id_tag_id_map = {} - for tag_info in new_tags_info_array: - record_ids = tag_info.get('record_ids', []) - tag_id = tag_info.get('tag_id', '') - for record_id in record_ids: - if record_id not in record_id_tag_id_map: - record_id_tag_id_map[record_id] = [] - record_id_tag_id_map[record_id].append(tag_id) + + # handle tag name conflict + tags_table_id = tags_table['id'] + tags_to_create = [] + + if existing_tag_names: + for idx, tag_data in enumerate(tags_data): + tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') + tag_color = tag_data.get(TAGS_TABLE.columns.color.name, '') + + if tag_name not in existing_tag_names: + tags_to_create.append(tag_data) + else: + unique_name = gen_unique_tag_name(tag_name, existing_tag_names) + + tags_to_create.append({ + TAGS_TABLE.columns.color.name: tag_color, + TAGS_TABLE.columns.name.name: unique_name + }) + else: + tags_to_create = tags_data + + try: + response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) + new_tag_ids = response.get('row_ids', []) + except Exception as e: + logger.error(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + + # create new old tag id mapping + destination_tags_info = {} # new tag info + for idx, old_tag_id in enumerate(source_tags_info): + new_tag_id = new_tag_ids[idx] + + file_paths = source_tags_info[old_tag_id]['file_paths'] + destination_tags_info[new_tag_id] = { + 'file_paths': file_paths + } + + # create record id to tag id mapping + record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} + + for tag_id, tag_info in destination_tags_info.items(): + for file_path in tag_info['file_paths']: + record_id = file_to_record_map.get(file_path) + if not record_id: + continue + + if record_id not in record_to_tags_map: + record_to_tags_map[record_id] = [] + + record_to_tags_map[record_id].append(tag_id) try: - metadata_server_api.insert_link(TAGS_TABLE.file_link_id, METADATA_TABLE.id, record_id_tag_id_map) + metadata_server_api.insert_link( + TAGS_TABLE.file_link_id, + METADATA_TABLE.id, + record_to_tags_map + ) record = RepoMetadata.objects.filter(repo_id=repo_id).first() if not record.details_settings: details_settings = {} @@ -2946,7 +2967,6 @@ class MetadataMigrateTags(APIView): details_settings = json.loads(record.details_settings) details_settings['tags_migrated'] = True record.details_settings = json.dumps(details_settings) - record.save() except Exception as e: logger.error(e) diff --git a/seahub/repo_metadata/metadata_server_api.py b/seahub/repo_metadata/metadata_server_api.py index 0cc5be911b..12738dd6dd 100644 --- a/seahub/repo_metadata/metadata_server_api.py +++ b/seahub/repo_metadata/metadata_server_api.py @@ -78,26 +78,6 @@ def list_metadata_view_records(repo_id, user, view, tags_enabled, start=0, limit return response_results -def list_eligible_metadata_records(repo_id, user, view, filter_columns): - from seafevents.repo_metadata.constants import METADATA_TABLE - from seafevents.repo_metadata.utils import gen_view_data_sql - metadata_server_api = MetadataServerAPI(repo_id, user) - columns = metadata_server_api.list_columns(METADATA_TABLE.id).get('columns') - tags_data = {'metadata': [], 'results': []} - - sql = gen_view_data_sql(METADATA_TABLE, columns, view, 0, 0, {'tags_data': tags_data, 'username': user}) - query_fields_str = '' - for column in columns: - column_name = column.get('name') - if column_name in filter_columns: - column_name_str = '`%s`, ' % column_name - query_fields_str += column_name_str - query_fields_str = query_fields_str.strip(', ') - sql = sql.replace('*', query_fields_str) - response_results = metadata_server_api.query_rows(sql, []) - return response_results - - def parse_response(response): if response.status_code >= 300 or response.status_code < 200: raise ConnectionError(response.status_code, response.text) From 5860161fe14a404249550a36cc89f972948066fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Tue, 22 Apr 2025 10:00:49 +0800 Subject: [PATCH 03/15] update --- seahub/repo_metadata/apis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 9f400943b6..6c5cbdbcbd 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2960,6 +2960,7 @@ class MetadataMigrateTags(APIView): METADATA_TABLE.id, record_to_tags_map ) + # Record that the old version tags have been migrated record = RepoMetadata.objects.filter(repo_id=repo_id).first() if not record.details_settings: details_settings = {} From 1b02d7b4915e1f9fe55c1dd32bbe6d7bbc78befd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Thu, 24 Apr 2025 11:15:37 +0800 Subject: [PATCH 04/15] update ui --- .../src/components/dialog/lib-settings.js | 35 ++++++++++++++++++- .../src/components/repo-info-bar-migrate.js | 11 +++--- .../metadata-tags-status-dialog/index.js | 15 ++++---- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/dialog/lib-settings.js b/frontend/src/components/dialog/lib-settings.js index 3f6eddac1d..229db467bc 100644 --- a/frontend/src/components/dialog/lib-settings.js +++ b/frontend/src/components/dialog/lib-settings.js @@ -14,6 +14,7 @@ import { } from '../../metadata'; import SeahubModalHeader from '@/components/common/seahub-modal-header'; import { useMetadataStatus } from '../../hooks'; +import Loading from '../../components/loading'; import '../../css/lib-settings.css'; @@ -28,6 +29,7 @@ const propTypes = { const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip, usedRepoTags, onMigrateSuccess }) => { const [activeTab, setActiveTab] = useState(tab || TAB.HISTORY_SETTING); + const [isMigrating, setIsMigrating] = useState(false); const toggleTab = useCallback((tab) => { setActiveTab(tab); }, []); @@ -46,9 +48,38 @@ const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMig const enableAutoDelSetting = is_admin && enableRepoAutoDel; const enableExtendedPropertiesSetting = !encrypted && is_admin && enableMetadataManagement; + const handleMigrateStart = useCallback(() => { + setIsMigrating(true); + }, []); + + const handleMigrateEnd = useCallback(() => { + setIsMigrating(false); + onMigrateSuccess && onMigrateSuccess(); + }, [onMigrateSuccess]); + + const handleMigrateError = useCallback(() => { + setIsMigrating(false); + }, []); + return (
+ {isMigrating && ( +
+ +
+ )} {gettext('Settings')} @@ -203,7 +234,9 @@ const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMig enableMetadata={enableMetadata} showMigrateTip={showMigrateTip} usedRepoTags={usedRepoTags} - onMigrateSuccess={onMigrateSuccess} + onMigrateSuccess={handleMigrateEnd} + onMigrateError={handleMigrateError} + onMigrateStart={handleMigrateStart} /> )} diff --git a/frontend/src/components/repo-info-bar-migrate.js b/frontend/src/components/repo-info-bar-migrate.js index bc2bd3f4bc..f5e72d259a 100644 --- a/frontend/src/components/repo-info-bar-migrate.js +++ b/frontend/src/components/repo-info-bar-migrate.js @@ -7,7 +7,7 @@ import { EVENT_BUS_TYPE } from '../components/common/event-bus-type'; const RepoInfoBarMigrate = () => { - const { enableMetadata, detailsSettings } = useMetadataStatus(); + const { enableMetadataManagement, detailsSettings } = useMetadataStatus(); const tagsMigrated = detailsSettings?.tags_migrated; const openMigrate = () => { eventBus.dispatch(EVENT_BUS_TYPE.OPEN_TREE_PANEL, () => eventBus.dispatch(EVENT_BUS_TYPE.OPEN_LIBRARY_SETTINGS_TAGS)); @@ -15,17 +15,16 @@ const RepoInfoBarMigrate = () => { return (
- {enableMetadata && !tagsMigrated ? - ( + {!tagsMigrated && ( + enableMetadataManagement ? ( <> {gettext('Tips: There are tags of old version. Please migrate tags to new version.')} - ) : - ( + ) : ( <>{gettext('Tips: These are tags of old version. The feature is deprecated and can no longer be used.')} ) - } + )}
); }; diff --git a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js index af02775952..4346339e45 100644 --- a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js +++ b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js @@ -19,13 +19,11 @@ const langOptions = [ } ]; -const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip, usedRepoTags, onMigrateSuccess }) => { +const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip, usedRepoTags, onMigrateSuccess, onMigrateStart, onMigrateError }) => { const [value, setValue] = useState(oldValue); const [lang, setLang] = useState(oldLang); const [submitting, setSubmitting] = useState(false); - const [migrated, setMigrated] = useState(false); const [showTurnOffConfirmDialog, setShowTurnOffConfirmDialog] = useState(false); - const onToggle = useCallback(() => { toggle(); }, [toggle]); @@ -47,16 +45,16 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg }, [lang, repoID, submit, toggle, value]); const migrateTag = useCallback(() => { + onMigrateStart && onMigrateStart(); tagsAPI.migrateTags(repoID, usedRepoTags).then(res => { - setMigrated(true); toaster.success(gettext('Tags migrated successfully')); onMigrateSuccess && onMigrateSuccess(); }).catch(error => { const errorMsg = Utils.getErrorMsg(error); toaster.danger(errorMsg); - setMigrated(false); + onMigrateError && onMigrateError(); }); - }, [repoID, usedRepoTags, onMigrateSuccess]); + }, [repoID, usedRepoTags, onMigrateSuccess, onMigrateStart, onMigrateError]); const turnOffConfirmToggle = useCallback(() => { setShowTurnOffConfirmDialog(!showTurnOffConfirmDialog); @@ -122,9 +120,8 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg } @@ -153,6 +150,8 @@ MetadataTagsStatusDialog.propTypes = { showMigrateTip: PropTypes.bool, repoTags: PropTypes.array, onMigrateSuccess: PropTypes.func, + onMigrateError: PropTypes.func, + onMigrateStart: PropTypes.func, }; export default MetadataTagsStatusDialog; From ce08321429f7b737184cefd0f2895b95cee2e7f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Wed, 7 May 2025 10:45:34 +0800 Subject: [PATCH 05/15] update design --- .../src/components/dialog/lib-settings.js | 4 +- .../dir-view-mode/dir-column-nav/index.js | 7 +- .../dir-view-mode/dir-column-view.js | 1 - .../dir-view-mode/dir-others/index.js | 4 +- .../metadata-tags-status-dialog/index.js | 6 +- frontend/src/tag/api.js | 7 +- seahub/repo_metadata/apis.py | 107 ++++++++++-------- 7 files changed, 69 insertions(+), 67 deletions(-) diff --git a/frontend/src/components/dialog/lib-settings.js b/frontend/src/components/dialog/lib-settings.js index 229db467bc..158bea5bad 100644 --- a/frontend/src/components/dialog/lib-settings.js +++ b/frontend/src/components/dialog/lib-settings.js @@ -24,10 +24,9 @@ const propTypes = { toggleDialog: PropTypes.func.isRequired, repoID: PropTypes.string.isRequired, currentRepoInfo: PropTypes.object.isRequired, - usedRepoTags: PropTypes.array, }; -const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip, usedRepoTags, onMigrateSuccess }) => { +const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMigrateTip, onMigrateSuccess }) => { const [activeTab, setActiveTab] = useState(tab || TAB.HISTORY_SETTING); const [isMigrating, setIsMigrating] = useState(false); const toggleTab = useCallback((tab) => { @@ -233,7 +232,6 @@ const LibSettingsDialog = ({ repoID, currentRepoInfo, toggleDialog, tab, showMig toggleDialog={toggleDialog} enableMetadata={enableMetadata} showMigrateTip={showMigrateTip} - usedRepoTags={usedRepoTags} onMigrateSuccess={handleMigrateEnd} onMigrateError={handleMigrateError} onMigrateStart={handleMigrateStart} diff --git a/frontend/src/components/dir-view-mode/dir-column-nav/index.js b/frontend/src/components/dir-view-mode/dir-column-nav/index.js index 9cb8222a42..9afedb4252 100644 --- a/frontend/src/components/dir-view-mode/dir-column-nav/index.js +++ b/frontend/src/components/dir-view-mode/dir-column-nav/index.js @@ -33,7 +33,6 @@ const propTypes = { getMenuContainerSize: PropTypes.func, updateDirent: PropTypes.func, updateTreeNode: PropTypes.func, - usedRepoTags: PropTypes.array, }; class DirColumnNav extends React.Component { @@ -44,7 +43,7 @@ class DirColumnNav extends React.Component { render() { const { - isTreeDataLoading, userPerm, treeData, repoID, currentPath, currentRepoInfo, navRate = 0.25, usedRepoTags + isTreeDataLoading, userPerm, treeData, repoID, currentPath, currentRepoInfo, navRate = 0.25 } = this.props; const flex = navRate ? '0 0 ' + navRate * 100 + '%' : '0 0 25%'; const select = this.props.inResizing ? 'none' : ''; @@ -75,9 +74,9 @@ class DirColumnNav extends React.Component { updateDirent={this.props.updateDirent} updateTreeNode={this.props.updateTreeNode} /> - + - + )}
diff --git a/frontend/src/components/dir-view-mode/dir-column-view.js b/frontend/src/components/dir-view-mode/dir-column-view.js index aaef405051..aef5a78696 100644 --- a/frontend/src/components/dir-view-mode/dir-column-view.js +++ b/frontend/src/components/dir-view-mode/dir-column-view.js @@ -185,7 +185,6 @@ class DirColumnView extends React.Component { direntList={this.props.direntList} updateDirent={this.props.updateDirent} updateTreeNode={this.props.updateTreeNode} - usedRepoTags={this.props.usedRepoTags} /> { +const DirOthers = ({ userPerm, repoID, currentRepoInfo }) => { const showSettings = currentRepoInfo.is_admin; // repo owner, department admin, shared with 'Admin' permission const repoName = currentRepoInfo.repo_name; @@ -86,7 +86,6 @@ const DirOthers = ({ userPerm, repoID, currentRepoInfo, usedRepoTags }) => { toggleDialog={toggleSettingsDialog} tab={activeTab} showMigrateTip={showMigrateTip} - usedRepoTags={usedRepoTags} onMigrateSuccess={handleMigrateSuccess} /> )} @@ -106,7 +105,6 @@ DirOthers.propTypes = { userPerm: PropTypes.string, repoID: PropTypes.string, currentRepoInfo: PropTypes.object.isRequired, - usedRepoTags: PropTypes.array, }; export default DirOthers; diff --git a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js index 4346339e45..d6f5dfefdf 100644 --- a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js +++ b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js @@ -19,7 +19,7 @@ const langOptions = [ } ]; -const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip, usedRepoTags, onMigrateSuccess, onMigrateStart, onMigrateError }) => { +const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, toggleDialog: toggle, submit, enableMetadata, showMigrateTip, onMigrateSuccess, onMigrateStart, onMigrateError }) => { const [value, setValue] = useState(oldValue); const [lang, setLang] = useState(oldLang); const [submitting, setSubmitting] = useState(false); @@ -46,7 +46,7 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg const migrateTag = useCallback(() => { onMigrateStart && onMigrateStart(); - tagsAPI.migrateTags(repoID, usedRepoTags).then(res => { + tagsAPI.migrateTags(repoID).then(res => { toaster.success(gettext('Tags migrated successfully')); onMigrateSuccess && onMigrateSuccess(); }).catch(error => { @@ -54,7 +54,7 @@ const MetadataTagsStatusDialog = ({ value: oldValue, lang: oldLang, repoID, togg toaster.danger(errorMsg); onMigrateError && onMigrateError(); }); - }, [repoID, usedRepoTags, onMigrateSuccess, onMigrateStart, onMigrateError]); + }, [repoID, onMigrateSuccess, onMigrateStart, onMigrateError]); const turnOffConfirmToggle = useCallback(() => { setShowTurnOffConfirmDialog(!showTurnOffConfirmDialog); diff --git a/frontend/src/tag/api.js b/frontend/src/tag/api.js index 8e29f701cb..b55cbc4938 100644 --- a/frontend/src/tag/api.js +++ b/frontend/src/tag/api.js @@ -138,12 +138,9 @@ class TagsManagerAPI { return this.req.post(url, params); }; - migrateTags = (repoID, usedRepoTags) => { + migrateTags = (repoID) => { const url = this.server + '/api/v2.1/repos/' + repoID + '/metadata/migrate-tags/'; - const params = { - tag_ids: usedRepoTags.map(tag => tag.id), - }; - return this.req.post(url, params); + return this.req.post(url); }; } diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 6c5cbdbcbd..94c832c01f 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -24,6 +24,7 @@ from seahub.utils.repo import is_repo_admin from seaserv import seafile_api from seahub.repo_metadata.constants import FACE_RECOGNITION_VIEW_ID, METADATA_RECORD_UPDATE_LIMIT from seahub.file_tags.models import FileTags +from seahub.repo_tags.models import RepoTags logger = logging.getLogger(__name__) @@ -2789,12 +2790,10 @@ class MetadataMigrateTags(APIView): permission_classes = (IsAuthenticated,) throttle_classes = (UserRateThrottle,) + def _list_old_tags(self): + pass + def post(self, request, repo_id): - tag_ids = request.data.get('tag_ids') - if not tag_ids: - error_msg = 'tag_ids invalid' - return api_error(status.HTTP_400_BAD_REQUEST, error_msg) - metadata = RepoMetadata.objects.filter(repo_id=repo_id).first() if not metadata or not metadata.enabled or not metadata.tags_enabled: error_msg = f'The tags is disabled for repo {repo_id}.' @@ -2813,9 +2812,7 @@ class MetadataMigrateTags(APIView): source_tags_info = {} # {tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} file_paths_set = set() # Used for querying metadata later try: - tagged_files = FileTags.objects.filter( - repo_tag__id__in=tag_ids).select_related('repo_tag', 'file_uuid') - + tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) for tagged_file in tagged_files: tag_id = tagged_file.repo_tag.id parent_path = tagged_file.file_uuid.parent_path @@ -2877,6 +2874,7 @@ class MetadataMigrateTags(APIView): tags_data = [] # [{name: '', color: ''}] for tag_id, tag_info in source_tags_info.items(): tags_data.append({ + TAGS_TABLE.columns.id.name: tag_id, TAGS_TABLE.columns.name.name: tag_info['name'], TAGS_TABLE.columns.color.name: tag_info['color'], }) @@ -2886,16 +2884,16 @@ class MetadataMigrateTags(APIView): if not tags_table: return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') - sql = f'SELECT `{TAGS_TABLE.columns.name.name}` FROM {TAGS_TABLE.name}' + sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}' existing_tags_result = metadata_server_api.query_rows(sql) existing_tag_records = existing_tags_result.get('results', []) - - existing_tag_names = [] + existing_tag_map = {} if existing_tag_records: - existing_tag_names = [ - tag_dict.get(TAGS_TABLE.columns.name.name, '') + existing_tag_map = { + tag_dict.get(TAGS_TABLE.columns.name.name, '') : + tag_dict.get(TAGS_TABLE.columns.id.name, '') for tag_dict in existing_tag_records - ] + } except Exception as e: logger.error(e) error_msg = 'Internal Server Error' @@ -2904,45 +2902,58 @@ class MetadataMigrateTags(APIView): # handle tag name conflict tags_table_id = tags_table['id'] tags_to_create = [] - - if existing_tag_names: + tags_to_create_info = [] + tags_not_to_create = [] + if existing_tag_map: for idx, tag_data in enumerate(tags_data): tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') - tag_color = tag_data.get(TAGS_TABLE.columns.color.name, '') - - if tag_name not in existing_tag_names: - tags_to_create.append(tag_data) - else: - unique_name = gen_unique_tag_name(tag_name, existing_tag_names) - + tag_color = tag_data.get(TAGS_TABLE.columns.color.name) + if tag_name not in existing_tag_map: tags_to_create.append({ - TAGS_TABLE.columns.color.name: tag_color, - TAGS_TABLE.columns.name.name: unique_name + TAGS_TABLE.columns.name.name: tag_name, + TAGS_TABLE.columns.color.name: tag_color + }) + tags_to_create_info.append(tag_data) + else: + tags_not_to_create.append({ + 'old_tag_id': tag_data.get(TAGS_TABLE.columns.id.name), + TAGS_TABLE.columns.id.name: existing_tag_map.get(tag_name), + TAGS_TABLE.columns.name.name: tag_name }) else: - tags_to_create = tags_data - + for tag_data in tags_data: + tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') + tag_color = tag_data.get(TAGS_TABLE.columns.color.name) + tags_to_create.append({ + TAGS_TABLE.columns.name.name: tag_name, + TAGS_TABLE.columns.color.name: tag_color + }) + tags_to_create_info = tags_data try: - response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) - new_tag_ids = response.get('row_ids', []) + destination_tags_info = {} # new tag info + if tags_to_create: + response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) + new_tag_ids = response.get('row_ids', []) + # create new old tag id mapping + for idx, tag in enumerate(tags_to_create): + new_tag_id = new_tag_ids[idx] + file_paths = source_tags_info[tags_to_create_info[idx].get(TAGS_TABLE.columns.id.name)].get('file_paths') + destination_tags_info[new_tag_id] = { + 'file_paths': file_paths + } + if tags_not_to_create: + for tag in tags_not_to_create: + file_paths = source_tags_info[tag['old_tag_id']].get('file_paths') + destination_tags_info[tag.get(TAGS_TABLE.columns.id.name)] = { + 'file_paths': file_paths + } except Exception as e: logger.error(e) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - # create new old tag id mapping - destination_tags_info = {} # new tag info - for idx, old_tag_id in enumerate(source_tags_info): - new_tag_id = new_tag_ids[idx] - - file_paths = source_tags_info[old_tag_id]['file_paths'] - destination_tags_info[new_tag_id] = { - 'file_paths': file_paths - } - # create record id to tag id mapping record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} - for tag_id, tag_info in destination_tags_info.items(): for file_path in tag_info['file_paths']: record_id = file_to_record_map.get(file_path) @@ -2961,14 +2972,14 @@ class MetadataMigrateTags(APIView): record_to_tags_map ) # Record that the old version tags have been migrated - record = RepoMetadata.objects.filter(repo_id=repo_id).first() - if not record.details_settings: - details_settings = {} - else: - details_settings = json.loads(record.details_settings) - details_settings['tags_migrated'] = True - record.details_settings = json.dumps(details_settings) - record.save() + # record = RepoMetadata.objects.filter(repo_id=repo_id).first() + # if not record.details_settings: + # details_settings = {} + # else: + # details_settings = json.loads(record.details_settings) + # details_settings['tags_migrated'] = True + # record.details_settings = json.dumps(details_settings) + # record.save() except Exception as e: logger.error(e) error_msg = 'Internal Server Error' From 38d74dda2fe5013dc6b78bfe90a999a709b6e761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Wed, 7 May 2025 11:46:07 +0800 Subject: [PATCH 06/15] optimize sql --- seahub/repo_metadata/apis.py | 199 +++++++++++++++++++---------------- 1 file changed, 107 insertions(+), 92 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 94c832c01f..17303a26ba 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2789,79 +2789,30 @@ class MetadataMigrateTags(APIView): authentication_classes = (TokenAuthentication, SessionAuthentication) permission_classes = (IsAuthenticated,) throttle_classes = (UserRateThrottle,) - - def _list_old_tags(self): - pass - - def post(self, request, repo_id): - metadata = RepoMetadata.objects.filter(repo_id=repo_id).first() - if not metadata or not metadata.enabled or not metadata.tags_enabled: - error_msg = f'The tags is disabled for repo {repo_id}.' - return api_error(status.HTTP_404_NOT_FOUND, error_msg) - - repo = seafile_api.get_repo(repo_id) - if not repo: - error_msg = 'Library %s not found.' % repo_id - return api_error(status.HTTP_404_NOT_FOUND, error_msg) - - if not is_repo_admin(request.user.username, repo_id): - error_msg = 'Permission denied.' - return api_error(status.HTTP_403_FORBIDDEN, error_msg) - - # list old tag info + + # return old tag info + def _get_source_tags_info(self, repo_id): source_tags_info = {} # {tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} file_paths_set = set() # Used for querying metadata later - try: - tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) - for tagged_file in tagged_files: - tag_id = tagged_file.repo_tag.id - parent_path = tagged_file.file_uuid.parent_path - filename = tagged_file.file_uuid.filename - file_path = posixpath.join(parent_path, filename) - - if tag_id not in source_tags_info: - source_tags_info[tag_id] = { - 'name': tagged_file.repo_tag.name, - 'color': tagged_file.repo_tag.color, - 'file_paths': [] - } - - source_tags_info[tag_id]['file_paths'].append(file_path) - file_paths_set.add(file_path) - except Exception as err: - logger.error(err) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - - # query records - from seafevents.repo_metadata.constants import METADATA_TABLE - from seafevents.repo_metadata.constants import TAGS_TABLE - metadata_server_api = MetadataServerAPI(repo_id, request.user.username) - try: - dir_paths = [os.path.dirname(path) for path in file_paths_set] - filenames = [os.path.basename(path) for path in file_paths_set] + tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) + for tagged_file in tagged_files: + tag_id = tagged_file.repo_tag.id + parent_path = tagged_file.file_uuid.parent_path + filename = tagged_file.file_uuid.filename + file_path = posixpath.join(parent_path, filename) - filenames_str = ', '.join([f'"{name}"' for name in filenames]) - dir_paths_str = ', '.join([f'"{path}"' for path in dir_paths]) - - sql = f''' - SELECT `{METADATA_TABLE.columns.id.name}`, - `{METADATA_TABLE.columns.file_name.name}`, - `{METADATA_TABLE.columns.parent_dir.name}` - FROM `{METADATA_TABLE.name}` - WHERE `{METADATA_TABLE.columns.is_dir.name}` = False - AND `{METADATA_TABLE.columns.file_name.name}` IN ({filenames_str}) - AND `{METADATA_TABLE.columns.parent_dir.name}` IN ({dir_paths_str}) - ''' + if tag_id not in source_tags_info: + source_tags_info[tag_id] = { + 'name': tagged_file.repo_tag.name, + 'color': tagged_file.repo_tag.color, + 'file_paths': [] + } - query_result = metadata_server_api.query_rows(sql, []) - metadata_records = query_result.get('results', []) - except Exception as e: - logger.exception(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - - + source_tags_info[tag_id]['file_paths'].append(file_path) + file_paths_set.add(file_path) + return source_tags_info, file_paths_set, tagged_files + + def _prepare_tags_data_for_creation(self, metadata_records, source_tags_info, metadata_server_api, METADATA_TABLE, TAGS_TABLE): file_to_record_map = {} # {file_path: record_id} for record in metadata_records: parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name) @@ -2878,34 +2829,24 @@ class MetadataMigrateTags(APIView): TAGS_TABLE.columns.name.name: tag_info['name'], TAGS_TABLE.columns.color.name: tag_info['color'], }) + - try: - tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) - if not tags_table: - return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') - - sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}' - existing_tags_result = metadata_server_api.query_rows(sql) - existing_tag_records = existing_tags_result.get('results', []) - existing_tag_map = {} - if existing_tag_records: - existing_tag_map = { - tag_dict.get(TAGS_TABLE.columns.name.name, '') : - tag_dict.get(TAGS_TABLE.columns.id.name, '') - for tag_dict in existing_tag_records - } - except Exception as e: - logger.error(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - - # handle tag name conflict - tags_table_id = tags_table['id'] + sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}' + existing_tags_result = metadata_server_api.query_rows(sql) + existing_tag_records = existing_tags_result.get('results', []) + existing_tag_map = {} + if existing_tag_records: + existing_tag_map = { + tag_dict.get(TAGS_TABLE.columns.name.name, '') : + tag_dict.get(TAGS_TABLE.columns.id.name, '') + for tag_dict in existing_tag_records + } + tags_to_create = [] tags_to_create_info = [] tags_not_to_create = [] if existing_tag_map: - for idx, tag_data in enumerate(tags_data): + for tag_data in tags_data: tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') tag_color = tag_data.get(TAGS_TABLE.columns.color.name) if tag_name not in existing_tag_map: @@ -2929,6 +2870,78 @@ class MetadataMigrateTags(APIView): TAGS_TABLE.columns.color.name: tag_color }) tags_to_create_info = tags_data + + return tags_to_create, tags_to_create_info, tags_not_to_create, file_to_record_map + + def _get_metadata_records(self, metadata_server_api, file_paths_set, METADATA_TABLE): + if not file_paths_set: + return [] + + file_paths_str = ', '.join([f'"{path}"' for path in file_paths_set]) + print(file_paths_str) + sql = f''' + SELECT `{METADATA_TABLE.columns.id.name}`, + `{METADATA_TABLE.columns.file_name.name}`, + `{METADATA_TABLE.columns.parent_dir.name}` + FROM `{METADATA_TABLE.name}` + WHERE `{METADATA_TABLE.columns.is_dir.name}` = FALSE + AND CONCAT(`{METADATA_TABLE.columns.parent_dir.name}`, `/`, `{METADATA_TABLE.columns.file_name.name}`) + IN ({file_paths_str}) + ''' + + query_result = metadata_server_api.query_rows(sql, []) + metadata_records = query_result.get('results', []) + + return metadata_records + + def post(self, request, repo_id): + metadata = RepoMetadata.objects.filter(repo_id=repo_id).first() + if not metadata or not metadata.enabled or not metadata.tags_enabled: + error_msg = f'The tags is disabled for repo {repo_id}.' + return api_error(status.HTTP_404_NOT_FOUND, error_msg) + + repo = seafile_api.get_repo(repo_id) + if not repo: + error_msg = 'Library %s not found.' % repo_id + return api_error(status.HTTP_404_NOT_FOUND, error_msg) + + if not is_repo_admin(request.user.username, repo_id): + error_msg = 'Permission denied.' + return api_error(status.HTTP_403_FORBIDDEN, error_msg) + + try: + source_tags_info, file_paths_set, tagged_files = self._get_source_tags_info(repo_id) + except Exception as err: + logger.error(err) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + + from seafevents.repo_metadata.constants import METADATA_TABLE + from seafevents.repo_metadata.constants import TAGS_TABLE + metadata_server_api = MetadataServerAPI(repo_id, request.user.username) + try: + # query records + metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE) + except Exception as e: + logger.exception(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + + + try: + # preare tags data + tags_to_create, tags_to_create_info, tags_not_to_create, file_to_record_map = \ + self._prepare_tags_data_for_creation(metadata_records, source_tags_info, metadata_server_api, METADATA_TABLE, TAGS_TABLE) + except Exception as e: + logger.error(e) + error_msg = 'Internal Server Error' + return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + + tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) + if not tags_table: + return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') + + tags_table_id = tags_table['id'] try: destination_tags_info = {} # new tag info if tags_to_create: @@ -2980,6 +2993,8 @@ class MetadataMigrateTags(APIView): # details_settings['tags_migrated'] = True # record.details_settings = json.dumps(details_settings) # record.save() + # tagged_files.delete() + # RepoTags.objects.get_all_by_repo_id(repo_id).delete() except Exception as e: logger.error(e) error_msg = 'Internal Server Error' From f132411ee17e53a36b0cd702c0e6467274bbe23d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Wed, 7 May 2025 12:02:03 +0800 Subject: [PATCH 07/15] update sql --- seahub/repo_metadata/apis.py | 44 ++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 17303a26ba..16dda7531e 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2876,20 +2876,30 @@ class MetadataMigrateTags(APIView): def _get_metadata_records(self, metadata_server_api, file_paths_set, METADATA_TABLE): if not file_paths_set: return [] - - file_paths_str = ', '.join([f'"{path}"' for path in file_paths_set]) - print(file_paths_str) + dir_paths = [] + filenames = [] + for file_path in file_paths_set: + parent_dir = os.path.dirname(file_path) + filename = os.path.basename(file_path) + dir_paths.append(parent_dir) + filenames.append(filename) + + where_conditions = [] + parameters = [] + for i in range(len(dir_paths)): + where_conditions.append(f"(`{METADATA_TABLE.columns.parent_dir.name}` = ? AND `{METADATA_TABLE.columns.file_name.name}` = ?)") + parameters.append(dir_paths[i]) + parameters.append(filenames[i]) + where_clause = " OR ".join(where_conditions) sql = f''' SELECT `{METADATA_TABLE.columns.id.name}`, `{METADATA_TABLE.columns.file_name.name}`, `{METADATA_TABLE.columns.parent_dir.name}` FROM `{METADATA_TABLE.name}` WHERE `{METADATA_TABLE.columns.is_dir.name}` = FALSE - AND CONCAT(`{METADATA_TABLE.columns.parent_dir.name}`, `/`, `{METADATA_TABLE.columns.file_name.name}`) - IN ({file_paths_str}) + AND ({where_clause}) ''' - - query_result = metadata_server_api.query_rows(sql, []) + query_result = metadata_server_api.query_rows(sql, parameters) metadata_records = query_result.get('results', []) return metadata_records @@ -2985,16 +2995,16 @@ class MetadataMigrateTags(APIView): record_to_tags_map ) # Record that the old version tags have been migrated - # record = RepoMetadata.objects.filter(repo_id=repo_id).first() - # if not record.details_settings: - # details_settings = {} - # else: - # details_settings = json.loads(record.details_settings) - # details_settings['tags_migrated'] = True - # record.details_settings = json.dumps(details_settings) - # record.save() - # tagged_files.delete() - # RepoTags.objects.get_all_by_repo_id(repo_id).delete() + record = RepoMetadata.objects.filter(repo_id=repo_id).first() + if not record.details_settings: + details_settings = {} + else: + details_settings = json.loads(record.details_settings) + details_settings['tags_migrated'] = True + record.details_settings = json.dumps(details_settings) + record.save() + tagged_files.delete() + RepoTags.objects.get_all_by_repo_id(repo_id).delete() except Exception as e: logger.error(e) error_msg = 'Internal Server Error' From 3236b20d190df9b18c4ac7be1c24459039ebe0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Wed, 7 May 2025 12:05:53 +0800 Subject: [PATCH 08/15] update --- seahub/repo_metadata/apis.py | 2 +- seahub/repo_metadata/utils.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 16dda7531e..f8ae15c4d7 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -18,7 +18,7 @@ from seahub.repo_metadata.utils import add_init_metadata_task, recognize_faces, get_unmodifiable_columns, can_read_metadata, init_faces, \ extract_file_details, get_table_by_name, remove_faces_table, FACES_SAVE_PATH, \ init_tags, init_tag_self_link_columns, remove_tags_table, add_init_face_recognition_task, init_ocr, \ - remove_ocr_column, get_update_record, update_people_cover_photo, gen_unique_tag_name + remove_ocr_column, get_update_record, update_people_cover_photo from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_metadata_view_records from seahub.utils.repo import is_repo_admin from seaserv import seafile_api diff --git a/seahub/repo_metadata/utils.py b/seahub/repo_metadata/utils.py index 6156e25d20..aed44b8d66 100644 --- a/seahub/repo_metadata/utils.py +++ b/seahub/repo_metadata/utils.py @@ -73,12 +73,6 @@ def gen_unique_id(id_set, length=4): return _id _id = generator_base64_code(length) -def gen_unique_tag_name(tag_name, exist_tags, counter=1): - new_name = f'{tag_name}({counter})' - if new_name not in exist_tags: - return new_name - return gen_unique_tag_name(tag_name, exist_tags, counter + 1) - def get_face_columns(): from seafevents.repo_metadata.constants import FACES_TABLE columns = [ From d420105e5126d9532946ce5d238f016490beef93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Wed, 7 May 2025 16:38:06 +0800 Subject: [PATCH 09/15] update --- .../src/components/repo-info-bar-migrate.js | 22 +++++++++---------- .../metadata-tags-status-dialog/index.js | 1 - seahub/repo_metadata/apis.py | 22 +++++-------------- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/frontend/src/components/repo-info-bar-migrate.js b/frontend/src/components/repo-info-bar-migrate.js index f5e72d259a..f2ff57db7b 100644 --- a/frontend/src/components/repo-info-bar-migrate.js +++ b/frontend/src/components/repo-info-bar-migrate.js @@ -7,24 +7,22 @@ import { EVENT_BUS_TYPE } from '../components/common/event-bus-type'; const RepoInfoBarMigrate = () => { - const { enableMetadataManagement, detailsSettings } = useMetadataStatus(); - const tagsMigrated = detailsSettings?.tags_migrated; + const { enableMetadataManagement } = useMetadataStatus(); const openMigrate = () => { eventBus.dispatch(EVENT_BUS_TYPE.OPEN_TREE_PANEL, () => eventBus.dispatch(EVENT_BUS_TYPE.OPEN_LIBRARY_SETTINGS_TAGS)); }; return (
- {!tagsMigrated && ( - enableMetadataManagement ? ( - <> - {gettext('Tips: There are tags of old version. Please migrate tags to new version.')} - - - ) : ( - <>{gettext('Tips: These are tags of old version. The feature is deprecated and can no longer be used.')} - ) - )} + {enableMetadataManagement ? ( + <> + {gettext('Tips: There are tags of old version. Please migrate tags to new version.')} + + + ) : ( + <>{gettext('Tips: These are tags of old version. The feature is deprecated and can no longer be used.')} + ) + }
); }; diff --git a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js index d6f5dfefdf..5ce0bff961 100644 --- a/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js +++ b/frontend/src/metadata/components/dialog/metadata-tags-status-dialog/index.js @@ -148,7 +148,6 @@ MetadataTagsStatusDialog.propTypes = { submit: PropTypes.func.isRequired, enableMetadata: PropTypes.bool.isRequired, showMigrateTip: PropTypes.bool, - repoTags: PropTypes.array, onMigrateSuccess: PropTypes.func, onMigrateError: PropTypes.func, onMigrateStart: PropTypes.func, diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index f8ae15c4d7..092d843617 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2822,19 +2822,18 @@ class MetadataMigrateTags(APIView): file_path = posixpath.join(parent_dir, file_name) file_to_record_map[file_path] = record_id - tags_data = [] # [{name: '', color: ''}] + tags_data = [] # [{id: '', name: '', color: ''}] for tag_id, tag_info in source_tags_info.items(): tags_data.append({ TAGS_TABLE.columns.id.name: tag_id, TAGS_TABLE.columns.name.name: tag_info['name'], TAGS_TABLE.columns.color.name: tag_info['color'], }) - sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}' existing_tags_result = metadata_server_api.query_rows(sql) existing_tag_records = existing_tags_result.get('results', []) - existing_tag_map = {} + existing_tag_map = {} # {name:id, ...} if existing_tag_records: existing_tag_map = { tag_dict.get(TAGS_TABLE.columns.name.name, '') : @@ -2842,9 +2841,9 @@ class MetadataMigrateTags(APIView): for tag_dict in existing_tag_records } - tags_to_create = [] - tags_to_create_info = [] - tags_not_to_create = [] + tags_to_create = [] # [{name:'', color:''}] Tags that need to be created + tags_to_create_info = [] # Tags information that needs to be created + tags_not_to_create = [] # [{old_tag_id:'', id:'', name:''}] Existing tags if existing_tag_map: for tag_data in tags_data: tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') @@ -2937,7 +2936,6 @@ class MetadataMigrateTags(APIView): error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - try: # preare tags data tags_to_create, tags_to_create_info, tags_not_to_create, file_to_record_map = \ @@ -2994,15 +2992,7 @@ class MetadataMigrateTags(APIView): METADATA_TABLE.id, record_to_tags_map ) - # Record that the old version tags have been migrated - record = RepoMetadata.objects.filter(repo_id=repo_id).first() - if not record.details_settings: - details_settings = {} - else: - details_settings = json.loads(record.details_settings) - details_settings['tags_migrated'] = True - record.details_settings = json.dumps(details_settings) - record.save() + # clear old tag data tagged_files.delete() RepoTags.objects.get_all_by_repo_id(repo_id).delete() except Exception as e: From 859e06eeb5b4cb6d6aa9979f3bcb7ef03e102c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Wed, 7 May 2025 18:23:48 +0800 Subject: [PATCH 10/15] optimize code --- seahub/repo_metadata/apis.py | 166 ++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 82 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 092d843617..21596cfcd4 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2791,40 +2791,38 @@ class MetadataMigrateTags(APIView): throttle_classes = (UserRateThrottle,) # return old tag info - def _get_source_tags_info(self, repo_id): - source_tags_info = {} # {tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} + def _get_old_tags_info(self, repo_tags, tagged_files): + old_tags_info = {} # {tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} file_paths_set = set() # Used for querying metadata later - tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) for tagged_file in tagged_files: tag_id = tagged_file.repo_tag.id parent_path = tagged_file.file_uuid.parent_path filename = tagged_file.file_uuid.filename file_path = posixpath.join(parent_path, filename) - if tag_id not in source_tags_info: - source_tags_info[tag_id] = { + if tag_id not in old_tags_info: + old_tags_info[tag_id] = { 'name': tagged_file.repo_tag.name, 'color': tagged_file.repo_tag.color, 'file_paths': [] } - source_tags_info[tag_id]['file_paths'].append(file_path) + old_tags_info[tag_id]['file_paths'].append(file_path) file_paths_set.add(file_path) - return source_tags_info, file_paths_set, tagged_files + for repo_tag in repo_tags: + repo_tag_id = repo_tag.id + if repo_tag_id not in old_tags_info: + old_tags_info[repo_tag_id] = { + 'name': repo_tag.name, + 'color': repo_tag.color, + 'file_paths': [] + } + return old_tags_info, file_paths_set - def _prepare_tags_data_for_creation(self, metadata_records, source_tags_info, metadata_server_api, METADATA_TABLE, TAGS_TABLE): - file_to_record_map = {} # {file_path: record_id} - for record in metadata_records: - parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name) - file_name = record.get(METADATA_TABLE.columns.file_name.name) - record_id = record.get(METADATA_TABLE.columns.id.name) - - file_path = posixpath.join(parent_dir, file_name) - file_to_record_map[file_path] = record_id - - tags_data = [] # [{id: '', name: '', color: ''}] - for tag_id, tag_info in source_tags_info.items(): - tags_data.append({ + def _prepare_tags_data_for_creation(self, old_tags_info, metadata_server_api, TAGS_TABLE): + old_tags = [] # [{id: '', name: '', color: ''}] + for tag_id, tag_info in old_tags_info.items(): + old_tags.append({ TAGS_TABLE.columns.id.name: tag_id, TAGS_TABLE.columns.name.name: tag_info['name'], TAGS_TABLE.columns.color.name: tag_info['color'], @@ -2842,35 +2840,34 @@ class MetadataMigrateTags(APIView): } tags_to_create = [] # [{name:'', color:''}] Tags that need to be created - tags_to_create_info = [] # Tags information that needs to be created - tags_not_to_create = [] # [{old_tag_id:'', id:'', name:''}] Existing tags - if existing_tag_map: - for tag_data in tags_data: - tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') - tag_color = tag_data.get(TAGS_TABLE.columns.color.name) - if tag_name not in existing_tag_map: - tags_to_create.append({ - TAGS_TABLE.columns.name.name: tag_name, - TAGS_TABLE.columns.color.name: tag_color - }) - tags_to_create_info.append(tag_data) - else: - tags_not_to_create.append({ - 'old_tag_id': tag_data.get(TAGS_TABLE.columns.id.name), - TAGS_TABLE.columns.id.name: existing_tag_map.get(tag_name), - TAGS_TABLE.columns.name.name: tag_name - }) - else: - for tag_data in tags_data: - tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') - tag_color = tag_data.get(TAGS_TABLE.columns.color.name) + tags_details = [] # Tags information that needs to be created + exist_tags = [] # [{old_tag_id:'', id:'', name:''}] Existing tags + + for tag_data in old_tags: + tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') + tag_color = tag_data.get(TAGS_TABLE.columns.color.name) + tag_id = tag_data.get(TAGS_TABLE.columns.id.name) + + if tag_name in existing_tag_map: + # Tag already exists, no need to create it + exist_tags.append({ + 'old_tag_id': tag_id, + TAGS_TABLE.columns.id.name: existing_tag_map.get(tag_name), + TAGS_TABLE.columns.name.name: tag_name + }) + else: + # Tag needs to be created tags_to_create.append({ TAGS_TABLE.columns.name.name: tag_name, TAGS_TABLE.columns.color.name: tag_color }) - tags_to_create_info = tags_data + tags_details.append(tag_data) + + # If no existing tags, all tag data should be used for creation + if not existing_tag_map and not tags_details: + tags_details = old_tags - return tags_to_create, tags_to_create_info, tags_not_to_create, file_to_record_map + return tags_to_create, tags_details, exist_tags def _get_metadata_records(self, metadata_server_api, file_paths_set, METADATA_TABLE): if not file_paths_set: @@ -2903,6 +2900,31 @@ class MetadataMigrateTags(APIView): return metadata_records + def _handle_tags_link(self, metadata_records, destination_tags_info, METADATA_TABLE): + file_to_record_map = {} # {file_path: record_id} + for record in metadata_records: + parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name) + file_name = record.get(METADATA_TABLE.columns.file_name.name) + record_id = record.get(METADATA_TABLE.columns.id.name) + + file_path = posixpath.join(parent_dir, file_name) + file_to_record_map[file_path] = record_id + + # create record id to tag id mapping + record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} + for tag_id, tag_info in destination_tags_info.items(): + for file_path in tag_info['file_paths']: + record_id = file_to_record_map.get(file_path) + if not record_id: + continue + + if record_id not in record_to_tags_map: + record_to_tags_map[record_id] = [] + + record_to_tags_map[record_id].append(tag_id) + + return record_to_tags_map + def post(self, request, repo_id): metadata = RepoMetadata.objects.filter(repo_id=repo_id).first() if not metadata or not metadata.enabled or not metadata.tags_enabled: @@ -2919,7 +2941,9 @@ class MetadataMigrateTags(APIView): return api_error(status.HTTP_403_FORBIDDEN, error_msg) try: - source_tags_info, file_paths_set, tagged_files = self._get_source_tags_info(repo_id) + tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) + repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id) + old_tags_info, file_paths_set = self._get_old_tags_info(repo_tags, tagged_files) except Exception as err: logger.error(err) error_msg = 'Internal Server Error' @@ -2931,40 +2955,30 @@ class MetadataMigrateTags(APIView): try: # query records metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE) - except Exception as e: - logger.exception(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - try: - # preare tags data - tags_to_create, tags_to_create_info, tags_not_to_create, file_to_record_map = \ - self._prepare_tags_data_for_creation(metadata_records, source_tags_info, metadata_server_api, METADATA_TABLE, TAGS_TABLE) - except Exception as e: - logger.error(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + # Prepare the data required to create the tag + tags_to_create, tags_details, exist_tags = \ + self._prepare_tags_data_for_creation(old_tags_info, metadata_server_api, TAGS_TABLE) - tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) - if not tags_table: - return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') - - tags_table_id = tags_table['id'] - try: + # create new tags + tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) + if not tags_table: + return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') + tags_table_id = tags_table['id'] destination_tags_info = {} # new tag info if tags_to_create: response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) - new_tag_ids = response.get('row_ids', []) # create new old tag id mapping + new_tag_ids = response.get('row_ids', []) for idx, tag in enumerate(tags_to_create): new_tag_id = new_tag_ids[idx] - file_paths = source_tags_info[tags_to_create_info[idx].get(TAGS_TABLE.columns.id.name)].get('file_paths') + file_paths = old_tags_info[tags_details[idx].get(TAGS_TABLE.columns.id.name)].get('file_paths') destination_tags_info[new_tag_id] = { 'file_paths': file_paths } - if tags_not_to_create: - for tag in tags_not_to_create: - file_paths = source_tags_info[tag['old_tag_id']].get('file_paths') + if exist_tags: + for tag in exist_tags: + file_paths = old_tags_info[tag['old_tag_id']].get('file_paths') destination_tags_info[tag.get(TAGS_TABLE.columns.id.name)] = { 'file_paths': file_paths } @@ -2973,20 +2987,8 @@ class MetadataMigrateTags(APIView): error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - # create record id to tag id mapping - record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} - for tag_id, tag_info in destination_tags_info.items(): - for file_path in tag_info['file_paths']: - record_id = file_to_record_map.get(file_path) - if not record_id: - continue - - if record_id not in record_to_tags_map: - record_to_tags_map[record_id] = [] - - record_to_tags_map[record_id].append(tag_id) - try: + record_to_tags_map = self._handle_tags_link(metadata_records, destination_tags_info, METADATA_TABLE) metadata_server_api.insert_link( TAGS_TABLE.file_link_id, METADATA_TABLE.id, @@ -2994,7 +2996,7 @@ class MetadataMigrateTags(APIView): ) # clear old tag data tagged_files.delete() - RepoTags.objects.get_all_by_repo_id(repo_id).delete() + repo_tags.delete() except Exception as e: logger.error(e) error_msg = 'Internal Server Error' From e16bb8c13dcdd644902ae243286efdd8e4c4862f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Thu, 8 May 2025 16:29:38 +0800 Subject: [PATCH 11/15] update --- seahub/repo_metadata/apis.py | 154 +++++++++++++++-------------------- 1 file changed, 66 insertions(+), 88 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 21596cfcd4..252a0976f5 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2789,43 +2789,14 @@ class MetadataMigrateTags(APIView): authentication_classes = (TokenAuthentication, SessionAuthentication) permission_classes = (IsAuthenticated,) throttle_classes = (UserRateThrottle,) - - # return old tag info - def _get_old_tags_info(self, repo_tags, tagged_files): - old_tags_info = {} # {tag_id: {name: '', color: '', file_paths: [file_path1, file_path2]}} - file_paths_set = set() # Used for querying metadata later - for tagged_file in tagged_files: - tag_id = tagged_file.repo_tag.id - parent_path = tagged_file.file_uuid.parent_path - filename = tagged_file.file_uuid.filename - file_path = posixpath.join(parent_path, filename) - - if tag_id not in old_tags_info: - old_tags_info[tag_id] = { - 'name': tagged_file.repo_tag.name, - 'color': tagged_file.repo_tag.color, - 'file_paths': [] - } - - old_tags_info[tag_id]['file_paths'].append(file_path) - file_paths_set.add(file_path) - for repo_tag in repo_tags: - repo_tag_id = repo_tag.id - if repo_tag_id not in old_tags_info: - old_tags_info[repo_tag_id] = { - 'name': repo_tag.name, - 'color': repo_tag.color, - 'file_paths': [] - } - return old_tags_info, file_paths_set - - def _prepare_tags_data_for_creation(self, old_tags_info, metadata_server_api, TAGS_TABLE): + + def _create_metadata_tags(self, repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE): old_tags = [] # [{id: '', name: '', color: ''}] - for tag_id, tag_info in old_tags_info.items(): + for repo_tag in repo_tags: old_tags.append({ - TAGS_TABLE.columns.id.name: tag_id, - TAGS_TABLE.columns.name.name: tag_info['name'], - TAGS_TABLE.columns.color.name: tag_info['color'], + TAGS_TABLE.columns.id.name: repo_tag.id, + TAGS_TABLE.columns.name.name: repo_tag.name, + TAGS_TABLE.columns.color.name: repo_tag.color, }) sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}' @@ -2839,35 +2810,51 @@ class MetadataMigrateTags(APIView): for tag_dict in existing_tag_records } - tags_to_create = [] # [{name:'', color:''}] Tags that need to be created - tags_details = [] # Tags information that needs to be created - exist_tags = [] # [{old_tag_id:'', id:'', name:''}] Existing tags + tags_to_create = [] # [{name:'', color:''}, ...] + old_exist_tags = {} # {old_tag_name:id,} Existing tags for tag_data in old_tags: tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') tag_color = tag_data.get(TAGS_TABLE.columns.color.name) - tag_id = tag_data.get(TAGS_TABLE.columns.id.name) if tag_name in existing_tag_map: # Tag already exists, no need to create it - exist_tags.append({ - 'old_tag_id': tag_id, - TAGS_TABLE.columns.id.name: existing_tag_map.get(tag_name), - TAGS_TABLE.columns.name.name: tag_name - }) + new_tag_id = existing_tag_map.get(tag_name) + old_exist_tags[tag_name] = new_tag_id else: # Tag needs to be created tags_to_create.append({ TAGS_TABLE.columns.name.name: tag_name, TAGS_TABLE.columns.color.name: tag_color }) - tags_details.append(tag_data) - # If no existing tags, all tag data should be used for creation - if not existing_tag_map and not tags_details: - tags_details = old_tags + tags_created = {} # {name:id,...} + if tags_to_create: + response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) + new_tag_ids = response.get('row_ids', []) + + for idx, tag in enumerate(tags_to_create): + new_tag_id = new_tag_ids[idx] + tag_name = tag.get(TAGS_TABLE.columns.name.name) + tags_created[tag_name] = new_tag_id - return tags_to_create, tags_details, exist_tags + return tags_created, old_exist_tags + + def _get_old_tags_info(self, tagged_files): + old_tags_info = {} # {old_tag_name: {file_path,....}} + file_paths_set = set() # Used for querying metadata later + for tagged_file in tagged_files: + old_tag_name = tagged_file.repo_tag.name + parent_path = tagged_file.file_uuid.parent_path + filename = tagged_file.file_uuid.filename + file_path = posixpath.join(parent_path, filename) + + if old_tag_name not in old_tags_info: + old_tags_info[old_tag_name] = set() + + old_tags_info[old_tag_name].add(file_path) + file_paths_set.add(file_path) + return old_tags_info, file_paths_set def _get_metadata_records(self, metadata_server_api, file_paths_set, METADATA_TABLE): if not file_paths_set: @@ -2909,11 +2896,10 @@ class MetadataMigrateTags(APIView): file_path = posixpath.join(parent_dir, file_name) file_to_record_map[file_path] = record_id - # create record id to tag id mapping record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} - for tag_id, tag_info in destination_tags_info.items(): - for file_path in tag_info['file_paths']: + for tag_id, file_paths in destination_tags_info.items(): + for file_path in file_paths: record_id = file_to_record_map.get(file_path) if not record_id: continue @@ -2939,61 +2925,53 @@ class MetadataMigrateTags(APIView): if not is_repo_admin(request.user.username, repo_id): error_msg = 'Permission denied.' return api_error(status.HTTP_403_FORBIDDEN, error_msg) + + from seafevents.repo_metadata.constants import TAGS_TABLE + from seafevents.repo_metadata.constants import METADATA_TABLE + metadata_server_api = MetadataServerAPI(repo_id, request.user.username) + tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) + if not tags_table: + return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') + tags_table_id = tags_table['id'] try: - tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) + # create new tags repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id) - old_tags_info, file_paths_set = self._get_old_tags_info(repo_tags, tagged_files) + tags_created, old_exist_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) + + tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) + old_tags_info, file_paths_set = self._get_old_tags_info(tagged_files) except Exception as err: logger.error(err) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - - from seafevents.repo_metadata.constants import METADATA_TABLE - from seafevents.repo_metadata.constants import TAGS_TABLE - metadata_server_api = MetadataServerAPI(repo_id, request.user.username) + try: - # query records - metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE) - - # Prepare the data required to create the tag - tags_to_create, tags_details, exist_tags = \ - self._prepare_tags_data_for_creation(old_tags_info, metadata_server_api, TAGS_TABLE) - - # create new tags - tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) - if not tags_table: - return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') - tags_table_id = tags_table['id'] - destination_tags_info = {} # new tag info - if tags_to_create: - response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) - # create new old tag id mapping - new_tag_ids = response.get('row_ids', []) - for idx, tag in enumerate(tags_to_create): - new_tag_id = new_tag_ids[idx] - file_paths = old_tags_info[tags_details[idx].get(TAGS_TABLE.columns.id.name)].get('file_paths') - destination_tags_info[new_tag_id] = { - 'file_paths': file_paths - } - if exist_tags: - for tag in exist_tags: - file_paths = old_tags_info[tag['old_tag_id']].get('file_paths') - destination_tags_info[tag.get(TAGS_TABLE.columns.id.name)] = { - 'file_paths': file_paths - } + destination_tags_info = {} # {tag_id: file_paths} + for tag_name, tag_id in tags_created.items(): + if tag_name not in old_tags_info: + continue + file_paths = old_tags_info[tag_name] + destination_tags_info[tag_id] = file_paths + for tag_name, tag_id in old_exist_tags.items(): + if tag_name not in old_tags_info: + continue + file_paths = old_tags_info[tag_name] + destination_tags_info[tag_id] = file_paths except Exception as e: logger.error(e) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) - try: + # query records + metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE) record_to_tags_map = self._handle_tags_link(metadata_records, destination_tags_info, METADATA_TABLE) metadata_server_api.insert_link( TAGS_TABLE.file_link_id, METADATA_TABLE.id, record_to_tags_map ) + # clear old tag data tagged_files.delete() repo_tags.delete() From fe526651ab4d006b30849a71f07417c08832365b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Thu, 8 May 2025 18:35:35 +0800 Subject: [PATCH 12/15] update --- seahub/repo_metadata/apis.py | 55 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 252a0976f5..539bf058a5 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2791,36 +2791,26 @@ class MetadataMigrateTags(APIView): throttle_classes = (UserRateThrottle,) def _create_metadata_tags(self, repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE): - old_tags = [] # [{id: '', name: '', color: ''}] - for repo_tag in repo_tags: - old_tags.append({ - TAGS_TABLE.columns.id.name: repo_tag.id, - TAGS_TABLE.columns.name.name: repo_tag.name, - TAGS_TABLE.columns.color.name: repo_tag.color, - }) - sql = f'SELECT `{TAGS_TABLE.columns.name.name}`,`{TAGS_TABLE.columns.id.name}` FROM {TAGS_TABLE.name}' existing_tags_result = metadata_server_api.query_rows(sql) existing_tag_records = existing_tags_result.get('results', []) - existing_tag_map = {} # {name:id, ...} - if existing_tag_records: - existing_tag_map = { - tag_dict.get(TAGS_TABLE.columns.name.name, '') : - tag_dict.get(TAGS_TABLE.columns.id.name, '') - for tag_dict in existing_tag_records - } + existing_tag_map = { + tag_dict.get(TAGS_TABLE.columns.name.name, '') : + tag_dict.get(TAGS_TABLE.columns.id.name, '') + for tag_dict in existing_tag_records + } tags_to_create = [] # [{name:'', color:''}, ...] - old_exist_tags = {} # {old_tag_name:id,} Existing tags + old_tag_name_to_metadata_tag_id = {} # {old_tag_name:id,} Existing tags - for tag_data in old_tags: - tag_name = tag_data.get(TAGS_TABLE.columns.name.name, '') - tag_color = tag_data.get(TAGS_TABLE.columns.color.name) + for repo_tag in repo_tags: + tag_name = repo_tag.name + tag_color = repo_tag.color if tag_name in existing_tag_map: # Tag already exists, no need to create it new_tag_id = existing_tag_map.get(tag_name) - old_exist_tags[tag_name] = new_tag_id + old_tag_name_to_metadata_tag_id[tag_name] = new_tag_id else: # Tag needs to be created tags_to_create.append({ @@ -2828,17 +2818,16 @@ class MetadataMigrateTags(APIView): TAGS_TABLE.columns.color.name: tag_color }) - tags_created = {} # {name:id,...} + tags_created_name_to_metadata_tag_id = {} # {name:id,...} if tags_to_create: response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) new_tag_ids = response.get('row_ids', []) - for idx, tag in enumerate(tags_to_create): new_tag_id = new_tag_ids[idx] tag_name = tag.get(TAGS_TABLE.columns.name.name) - tags_created[tag_name] = new_tag_id - - return tags_created, old_exist_tags + tags_created_name_to_metadata_tag_id[tag_name] = new_tag_id + metadata_tags = {**tags_created_name_to_metadata_tag_id, **old_tag_name_to_metadata_tag_id} + return metadata_tags def _get_old_tags_info(self, tagged_files): old_tags_info = {} # {old_tag_name: {file_path,....}} @@ -2888,19 +2877,19 @@ class MetadataMigrateTags(APIView): return metadata_records def _handle_tags_link(self, metadata_records, destination_tags_info, METADATA_TABLE): - file_to_record_map = {} # {file_path: record_id} + file_path_to_record_id = {} # {file_path: record_id} for record in metadata_records: parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name) file_name = record.get(METADATA_TABLE.columns.file_name.name) record_id = record.get(METADATA_TABLE.columns.id.name) file_path = posixpath.join(parent_dir, file_name) - file_to_record_map[file_path] = record_id + file_path_to_record_id[file_path] = record_id # create record id to tag id mapping record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} for tag_id, file_paths in destination_tags_info.items(): for file_path in file_paths: - record_id = file_to_record_map.get(file_path) + record_id = file_path_to_record_id.get(file_path) if not record_id: continue @@ -2937,7 +2926,7 @@ class MetadataMigrateTags(APIView): try: # create new tags repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id) - tags_created, old_exist_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) + metadata_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) old_tags_info, file_paths_set = self._get_old_tags_info(tagged_files) @@ -2948,12 +2937,7 @@ class MetadataMigrateTags(APIView): try: destination_tags_info = {} # {tag_id: file_paths} - for tag_name, tag_id in tags_created.items(): - if tag_name not in old_tags_info: - continue - file_paths = old_tags_info[tag_name] - destination_tags_info[tag_id] = file_paths - for tag_name, tag_id in old_exist_tags.items(): + for tag_name, tag_id in metadata_tags.items(): if tag_name not in old_tags_info: continue file_paths = old_tags_info[tag_name] @@ -2971,7 +2955,6 @@ class MetadataMigrateTags(APIView): METADATA_TABLE.id, record_to_tags_map ) - # clear old tag data tagged_files.delete() repo_tags.delete() From 60d7d342848d26ec85f78780a632e5b4c8039e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Fri, 9 May 2025 11:19:20 +0800 Subject: [PATCH 13/15] optimize sql --- seahub/repo_metadata/apis.py | 81 ++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 539bf058a5..4dec766826 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2818,19 +2818,17 @@ class MetadataMigrateTags(APIView): TAGS_TABLE.columns.color.name: tag_color }) - tags_created_name_to_metadata_tag_id = {} # {name:id,...} if tags_to_create: response = metadata_server_api.insert_rows(tags_table_id, tags_to_create) new_tag_ids = response.get('row_ids', []) for idx, tag in enumerate(tags_to_create): new_tag_id = new_tag_ids[idx] tag_name = tag.get(TAGS_TABLE.columns.name.name) - tags_created_name_to_metadata_tag_id[tag_name] = new_tag_id - metadata_tags = {**tags_created_name_to_metadata_tag_id, **old_tag_name_to_metadata_tag_id} - return metadata_tags + old_tag_name_to_metadata_tag_id[tag_name] = new_tag_id + return old_tag_name_to_metadata_tag_id def _get_old_tags_info(self, tagged_files): - old_tags_info = {} # {old_tag_name: {file_path,....}} + old_tag_name_to_file_paths = {} # {old_tag_name: {file_path,....}} file_paths_set = set() # Used for querying metadata later for tagged_file in tagged_files: old_tag_name = tagged_file.repo_tag.name @@ -2838,12 +2836,12 @@ class MetadataMigrateTags(APIView): filename = tagged_file.file_uuid.filename file_path = posixpath.join(parent_path, filename) - if old_tag_name not in old_tags_info: - old_tags_info[old_tag_name] = set() + if old_tag_name not in old_tag_name_to_file_paths: + old_tag_name_to_file_paths[old_tag_name] = set() - old_tags_info[old_tag_name].add(file_path) + old_tag_name_to_file_paths[old_tag_name].add(file_path) file_paths_set.add(file_path) - return old_tags_info, file_paths_set + return old_tag_name_to_file_paths, file_paths_set def _get_metadata_records(self, metadata_server_api, file_paths_set, METADATA_TABLE): if not file_paths_set: @@ -2856,27 +2854,37 @@ class MetadataMigrateTags(APIView): dir_paths.append(parent_dir) filenames.append(filename) - where_conditions = [] - parameters = [] - for i in range(len(dir_paths)): - where_conditions.append(f"(`{METADATA_TABLE.columns.parent_dir.name}` = ? AND `{METADATA_TABLE.columns.file_name.name}` = ?)") - parameters.append(dir_paths[i]) - parameters.append(filenames[i]) - where_clause = " OR ".join(where_conditions) - sql = f''' - SELECT `{METADATA_TABLE.columns.id.name}`, - `{METADATA_TABLE.columns.file_name.name}`, - `{METADATA_TABLE.columns.parent_dir.name}` - FROM `{METADATA_TABLE.name}` - WHERE `{METADATA_TABLE.columns.is_dir.name}` = FALSE - AND ({where_clause}) - ''' - query_result = metadata_server_api.query_rows(sql, parameters) - metadata_records = query_result.get('results', []) + batch_size = 100 + all_metadata_records = [] + + for i in range(0, len(dir_paths), batch_size): + batch_dir_paths = dir_paths[i:i+batch_size] + batch_filenames = filenames[i:i+batch_size] + + where_conditions = [] + parameters = [] + for j in range(len(batch_dir_paths)): + where_conditions.append(f"(`{METADATA_TABLE.columns.parent_dir.name}` = ? AND `{METADATA_TABLE.columns.file_name.name}` = ?)") + parameters.append(batch_dir_paths[j]) + parameters.append(batch_filenames[j]) + + where_clause = " OR ".join(where_conditions) + sql = f''' + SELECT `{METADATA_TABLE.columns.id.name}`, + `{METADATA_TABLE.columns.file_name.name}`, + `{METADATA_TABLE.columns.parent_dir.name}` + FROM `{METADATA_TABLE.name}` + WHERE `{METADATA_TABLE.columns.is_dir.name}` = FALSE + AND ({where_clause}) + ''' + + query_result = metadata_server_api.query_rows(sql, parameters) + batch_metadata_records = query_result.get('results', []) + all_metadata_records.extend(batch_metadata_records) - return metadata_records + return all_metadata_records - def _handle_tags_link(self, metadata_records, destination_tags_info, METADATA_TABLE): + def _handle_tags_link(self, metadata_records, metadata_tag_id_to_file_paths, METADATA_TABLE): file_path_to_record_id = {} # {file_path: record_id} for record in metadata_records: parent_dir = record.get(METADATA_TABLE.columns.parent_dir.name) @@ -2887,7 +2895,7 @@ class MetadataMigrateTags(APIView): file_path_to_record_id[file_path] = record_id # create record id to tag id mapping record_to_tags_map = {} # {record_id: [tag_id1, tag_id2]} - for tag_id, file_paths in destination_tags_info.items(): + for tag_id, file_paths in metadata_tag_id_to_file_paths.items(): for file_path in file_paths: record_id = file_path_to_record_id.get(file_path) if not record_id: @@ -2915,8 +2923,7 @@ class MetadataMigrateTags(APIView): error_msg = 'Permission denied.' return api_error(status.HTTP_403_FORBIDDEN, error_msg) - from seafevents.repo_metadata.constants import TAGS_TABLE - from seafevents.repo_metadata.constants import METADATA_TABLE + from seafevents.repo_metadata.constants import TAGS_TABLE, METADATA_TABLE metadata_server_api = MetadataServerAPI(repo_id, request.user.username) tags_table = get_table_by_name(metadata_server_api, TAGS_TABLE.name) if not tags_table: @@ -2929,19 +2936,19 @@ class MetadataMigrateTags(APIView): metadata_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) - old_tags_info, file_paths_set = self._get_old_tags_info(tagged_files) + old_tag_name_to_file_paths, file_paths_set = self._get_old_tags_info(tagged_files) except Exception as err: logger.error(err) error_msg = 'Internal Server Error' return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) try: - destination_tags_info = {} # {tag_id: file_paths} + metadata_tag_id_to_file_paths = {} # {tag_id: file_paths} for tag_name, tag_id in metadata_tags.items(): - if tag_name not in old_tags_info: + if tag_name not in old_tag_name_to_file_paths: continue - file_paths = old_tags_info[tag_name] - destination_tags_info[tag_id] = file_paths + file_paths = old_tag_name_to_file_paths[tag_name] + metadata_tag_id_to_file_paths[tag_id] = file_paths except Exception as e: logger.error(e) error_msg = 'Internal Server Error' @@ -2949,7 +2956,7 @@ class MetadataMigrateTags(APIView): try: # query records metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE) - record_to_tags_map = self._handle_tags_link(metadata_records, destination_tags_info, METADATA_TABLE) + record_to_tags_map = self._handle_tags_link(metadata_records, metadata_tag_id_to_file_paths, METADATA_TABLE) metadata_server_api.insert_link( TAGS_TABLE.file_link_id, METADATA_TABLE.id, From d453bce98c47f2c4e5d4d139ec95cade14b1591d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Fri, 9 May 2025 13:42:35 +0800 Subject: [PATCH 14/15] optimize --- seahub/repo_metadata/apis.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 4dec766826..80890991b1 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2933,9 +2933,13 @@ class MetadataMigrateTags(APIView): try: # create new tags repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id) + if not repo_tags: + return Response({'success': True}) metadata_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) + if not tagged_files: + return Response({'success': True}) old_tag_name_to_file_paths, file_paths_set = self._get_old_tags_info(tagged_files) except Exception as err: logger.error(err) From 5f383805f5c63c3c4af60c315bfbac0ae0f78b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=B0=B8=E5=BC=BA?= <11704063+s-yongqiang@user.noreply.gitee.com> Date: Fri, 9 May 2025 13:48:54 +0800 Subject: [PATCH 15/15] update --- seahub/repo_metadata/apis.py | 42 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/seahub/repo_metadata/apis.py b/seahub/repo_metadata/apis.py index 80890991b1..3db63ef51f 100644 --- a/seahub/repo_metadata/apis.py +++ b/seahub/repo_metadata/apis.py @@ -2930,33 +2930,25 @@ class MetadataMigrateTags(APIView): return api_error(status.HTTP_404_NOT_FOUND, 'tags table not found') tags_table_id = tags_table['id'] - try: - # create new tags - repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id) - if not repo_tags: - return Response({'success': True}) - metadata_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) + # create new tags + repo_tags = RepoTags.objects.get_all_by_repo_id(repo_id) + if not repo_tags: + return Response({'success': True}) + metadata_tags = self._create_metadata_tags(repo_tags, tags_table_id, metadata_server_api, TAGS_TABLE) - tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) - if not tagged_files: - return Response({'success': True}) - old_tag_name_to_file_paths, file_paths_set = self._get_old_tags_info(tagged_files) - except Exception as err: - logger.error(err) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) + tagged_files = FileTags.objects.select_related('repo_tag').filter(repo_tag__repo_id=repo_id) + if not tagged_files: + repo_tags.delete() + return Response({'success': True}) + old_tag_name_to_file_paths, file_paths_set = self._get_old_tags_info(tagged_files) + + metadata_tag_id_to_file_paths = {} # {tag_id: file_paths} + for tag_name, tag_id in metadata_tags.items(): + if tag_name not in old_tag_name_to_file_paths: + continue + file_paths = old_tag_name_to_file_paths[tag_name] + metadata_tag_id_to_file_paths[tag_id] = file_paths - try: - metadata_tag_id_to_file_paths = {} # {tag_id: file_paths} - for tag_name, tag_id in metadata_tags.items(): - if tag_name not in old_tag_name_to_file_paths: - continue - file_paths = old_tag_name_to_file_paths[tag_name] - metadata_tag_id_to_file_paths[tag_id] = file_paths - except Exception as e: - logger.error(e) - error_msg = 'Internal Server Error' - return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg) try: # query records metadata_records = self._get_metadata_records(metadata_server_api, file_paths_set, METADATA_TABLE)