haiwen/seahub
1
0
Fork 0
mirror of https://github.com/haiwen/seahub.git synced 2025-08-01 23:38:37 +00:00
Code Issues Releases Wiki Activity

image_caption (#6657)

Browse Source 
* image_caption

* update

* update

* update

* update

* update

---------

Co-authored-by: zheng.shen <zheng.shen@seafile.com>
This commit is contained in:
shenzheng-1 2024-08-30 14:36:52 +08:00 committed by GitHub
parent 1e5b5565e8
commit d26c6d119c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 268 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
frontend/src/metadata/api.js
View File

@ -194,10 +194,21 @@ class MetadataManagerAPI {
};
// ai
generateSummary = (repoID, filePaths) => {
const url = this.server + '/api/v2.1/repos/' + repoID + '/metadata/ai/summarize-documents/';
generateSummary = (repoID, filePath) => {
const url = this.server + '/api/v2.1/ai/generate-summary/';
const params = {
file_paths_list: filePaths,
path: filePath,
repo_id: repoID,
};
return this.req.post(url, params);
};
imageCaption = (repoID, filePath, lang) => {
const url = this.server + '/api/v2.1/ai/image-caption/';
const params = {
path: filePath,
repo_id: repoID,
lang: lang,
};
return this.req.post(url, params);
};

103
frontend/src/metadata/metadata-view/components/context-menu/index.js
View File

@ -15,6 +15,7 @@ const OPERATION = {
OPEN_PARENT_FOLDER: 'open-parent-folder',
OPEN_IN_NEW_TAB: 'open-new-tab',
GENERATE_SUMMARY: 'generate-summary',
IMAGE_CAPTION: 'image-caption',
};
const ContextMenu = ({
@ -90,6 +91,8 @@ const ContextMenu = ({
const fileName = record[PRIVATE_COLUMN_KEY.FILE_NAME];
if (Utils.isSdocFile(fileName) && canModifyRow(record)) {
list.push({ value: OPERATION.GENERATE_SUMMARY, label: gettext('Generate summary') });
} else if (Utils.imageCheck(fileName) && canModifyRow(record)) {
list.push({ value: OPERATION.IMAGE_CAPTION, label: gettext('Generate image description') });
}
}
@ -134,63 +137,65 @@ const ContextMenu = ({
const generateSummary = useCallback(() => {
const canModifyRow = window.sfMetadataContext.canModifyRow;
const selectedRecords = Object.keys(recordMetrics.idSelectedRecordMap);
const summaryColumnKey = PRIVATE_COLUMN_KEY.FILE_SUMMARY;
let paths = [];
let path = '';
let idOldRecordData = {};
let idOriginalOldRecordData = {};
if (selectedRange) {
const { topLeft, bottomRight } = selectedRange;
for (let i = topLeft.rowIdx; i <= bottomRight.rowIdx; i++) {
const record = recordGetterByIndex({ isGroupView, groupRecordIndex: topLeft.groupRecordIndex, recordIndex: i });
if (!canModifyRow(record)) continue;
const fileName = record[PRIVATE_COLUMN_KEY.FILE_NAME];
if (!Utils.isSdocFile(fileName)) continue;
const parentDir = record[PRIVATE_COLUMN_KEY.PARENT_DIR];
paths.push(Utils.joinPath(parentDir, fileName));
idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
}
} else if (selectedRecords.length > 0) {
selectedRecords.forEach(recordId => {
const record = metadata.id_row_map[recordId];
const fileName = record[PRIVATE_COLUMN_KEY.FILE_NAME];
if (Utils.isSdocFile(fileName) && canModifyRow(record)) {
const parentDir = record[PRIVATE_COLUMN_KEY.PARENT_DIR];
paths.push(Utils.joinPath(parentDir, fileName));
idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
}
});
} else if (selectedPosition) {
const { groupRecordIndex, rowIdx } = selectedPosition;
const record = recordGetterByIndex({ isGroupView, groupRecordIndex, recordIndex: rowIdx });
const fileName = record[PRIVATE_COLUMN_KEY.FILE_NAME];
if (Utils.isSdocFile(fileName) && canModifyRow(record)) {
const parentDir = record[PRIVATE_COLUMN_KEY.PARENT_DIR];
paths.push(Utils.joinPath(parentDir, fileName));
idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
}
const { groupRecordIndex, rowIdx } = selectedPosition;
const record = recordGetterByIndex({ isGroupView, groupRecordIndex, recordIndex: rowIdx });
const fileName = record[PRIVATE_COLUMN_KEY.FILE_NAME];
if (Utils.isSdocFile(fileName) && canModifyRow(record)) {
const parentDir = record[PRIVATE_COLUMN_KEY.PARENT_DIR];
path = Utils.joinPath(parentDir, fileName);
idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
}
if (paths.length === 0) return;
window.sfMetadataContext.generateSummary(paths).then(res => {
const updatedRecords = res.data.rows;
let recordIds = [];
if (path === '') return;
window.sfMetadataContext.generateSummary(path).then(res => {
const summary = res.data.summary;
const updateRecordId = record[PRIVATE_COLUMN_KEY.ID];
const recordIds = [updateRecordId];
let idRecordUpdates = {};
let idOriginalRecordUpdates = {};
updatedRecords.forEach(updatedRecord => {
const { _id: updateRecordId, _summary } = updatedRecord;
recordIds.push(updateRecordId);
idRecordUpdates[updateRecordId] = { [summaryColumnKey]: _summary };
idOriginalRecordUpdates[updateRecordId] = { [summaryColumnKey]: _summary };
});
idRecordUpdates[updateRecordId] = { [summaryColumnKey]: summary };
idOriginalRecordUpdates[updateRecordId] = { [summaryColumnKey]: summary };
updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
}).catch(error => {
const errorMessage = gettext('Failed to generate summary');
toaster.danger(errorMessage);
});
}, [isGroupView, selectedRange, selectedPosition, recordMetrics, metadata, recordGetterByIndex, updateRecords]);
}, [isGroupView, selectedPosition, recordGetterByIndex, updateRecords]);
const imageCaption = useCallback(() => {
const canModifyRow = window.sfMetadataContext.canModifyRow;
const summaryColumnKey = PRIVATE_COLUMN_KEY.FILE_SUMMARY;
let path = '';
let idOldRecordData = {};
let idOriginalOldRecordData = {};
const { groupRecordIndex, rowIdx } = selectedPosition;
const record = recordGetterByIndex({ isGroupView, groupRecordIndex, recordIndex: rowIdx });
const fileName = record[PRIVATE_COLUMN_KEY.FILE_NAME];
if (Utils.imageCheck(fileName) && canModifyRow(record)) {
const parentDir = record[PRIVATE_COLUMN_KEY.PARENT_DIR];
path = Utils.joinPath(parentDir, fileName);
idOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
idOriginalOldRecordData[record[PRIVATE_COLUMN_KEY.ID]] = { [summaryColumnKey]: record[summaryColumnKey] };
}
if (path === '') return;
window.sfMetadataContext.imageCaption(path).then(res => {
const desc = res.data.desc;
const updateRecordId = record[PRIVATE_COLUMN_KEY.ID];
const recordIds = [updateRecordId];
let idRecordUpdates = {};
let idOriginalRecordUpdates = {};
idRecordUpdates[updateRecordId] = { [summaryColumnKey]: desc };
idOriginalRecordUpdates[updateRecordId] = { [summaryColumnKey]: desc };
updateRecords({ recordIds, idRecordUpdates, idOriginalRecordUpdates, idOldRecordData, idOriginalOldRecordData });
}).catch(error => {
const errorMessage = gettext('Failed to generate image description');
toaster.danger(errorMessage);
});
}, [isGroupView, selectedPosition, recordGetterByIndex, updateRecords]);
const handleOptionClick = useCallback((event, option) => {
event.stopPropagation();
@ -215,12 +220,16 @@ const ContextMenu = ({
generateSummary && generateSummary();
break;
}
case OPERATION.IMAGE_CAPTION: {
imageCaption && imageCaption();
break;
}
default: {
break;
}
}
setVisible(false);
}, [onOpenFileInNewTab, onOpenParentFolder, onCopySelected, onClearSelected, generateSummary]);
}, [onOpenFileInNewTab, onOpenParentFolder, onCopySelected, onClearSelected, generateSummary, imageCaption]);
const getMenuPosition = useCallback((x = 0, y = 0) => {
let menuStyles = {

16
frontend/src/metadata/metadata-view/context.js
View File

@ -2,12 +2,12 @@ import metadataAPI from '../api';
import { LocalStorage, PRIVATE_COLUMN_KEYS, EDITABLE_DATA_PRIVATE_COLUMN_KEYS,
EDITABLE_PRIVATE_COLUMN_KEYS, PREDEFINED_COLUMN_KEYS } from './_basic';
import EventBus from '../../components/common/event-bus';
import { username } from '../../utils/constants';
import { username, lang } from '../../utils/constants';
class Context {
constructor() {
this.settings = {};
this.settings = { lang };
this.metadataAPI = null;
this.localStorage = null;
this.eventBus = null;
@ -20,7 +20,7 @@ class Context {
if (this.hasInit) return;
// init settings
this.settings = settings || {};
this.settings = { ...this.settings, ...settings };
// init metadataAPI
const { repoInfo } = this.settings;
@ -192,9 +192,15 @@ class Context {
};
// ai
generateSummary = (filePaths) => {
generateSummary = (filePath) => {
const repoID = this.settings['repoID'];
return this.metadataAPI.generateSummary(repoID, filePaths);
return this.metadataAPI.generateSummary(repoID, filePath);
};
imageCaption = (filePath) => {
const repoID = this.settings['repoID'];
const lang = this.settings['lang'];
return this.metadataAPI.imageCaption(repoID, filePath, lang);
};
}

0
seahub/ai/__init__.py Normal file
View File

139
seahub/ai/apis.py Normal file
View File

@ -0,0 +1,139 @@
import logging
import os.path
from pysearpc import SearpcError
from seaserv import seafile_api
from rest_framework.authentication import SessionAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework import status
from rest_framework.views import APIView
from seahub.api2.utils import api_error
from seahub.api2.throttling import UserRateThrottle
from seahub.api2.authentication import TokenAuthentication
from seahub.utils import get_file_type_and_ext, IMAGE
from seahub.views import check_folder_permission
from seahub.ai.utils import image_caption, verify_ai_config, generate_summary
logger = logging.getLogger(__name__)
class ImageCaption(APIView):
authentication_classes = (TokenAuthentication, SessionAuthentication)
permission_classes = (IsAuthenticated,)
throttle_classes = (UserRateThrottle,)
def post(self, request):
if not verify_ai_config():
return api_error(status.HTTP_400_BAD_REQUEST, 'AI server not configured')
repo_id = request.data.get('repo_id')
path = request.data.get('path')
lang = request.data.get('lang')
if not repo_id:
return api_error(status.HTTP_400_BAD_REQUEST, 'repo_id invalid')
if not path:
return api_error(status.HTTP_400_BAD_REQUEST, 'path invalid')
if not lang:
return api_error(status.HTTP_400_BAD_REQUEST, 'lang invalid')
file_type, _ = get_file_type_and_ext(os.path.basename(path))
if file_type != IMAGE:
return api_error(status.HTTP_400_BAD_REQUEST, 'file type not image')
repo = seafile_api.get_repo(repo_id)
if not repo:
error_msg = 'Library %s not found.' % repo_id
return api_error(status.HTTP_404_NOT_FOUND, error_msg)
permission = check_folder_permission(request, repo_id, os.path.dirname(path))
if not permission:
error_msg = 'Permission denied.'
return api_error(status.HTTP_403_FORBIDDEN, error_msg)
try:
file_id = seafile_api.get_file_id_by_path(repo_id, path)
except SearpcError as e:
logger.error(e)
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error')
if not file_id:
return api_error(status.HTTP_404_NOT_FOUND, f"File {path} not found")
token = seafile_api.get_fileserver_access_token(repo_id, file_id, 'download', request.user.username, use_onetime=True)
if not token:
error_msg = 'Internal Server Error'
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
params = {
'path': path,
'download_token': token,
'lang': lang
}
try:
resp = image_caption(params)
resp_json = resp.json()
except Exception as e:
error_msg = 'Internal Server Error'
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
return Response(resp_json, resp.status_code)
class GenerateSummary(APIView):
authentication_classes = (TokenAuthentication, SessionAuthentication)
permission_classes = (IsAuthenticated,)
throttle_classes = (UserRateThrottle,)
def post(self, request):
if not verify_ai_config():
return api_error(status.HTTP_400_BAD_REQUEST, 'AI server not configured')
repo_id = request.data.get('repo_id')
path = request.data.get('path')
if not repo_id:
return api_error(status.HTTP_400_BAD_REQUEST, 'repo_id invalid')
if not path:
return api_error(status.HTTP_400_BAD_REQUEST, 'path invalid')
repo = seafile_api.get_repo(repo_id)
if not repo:
error_msg = 'Library %s not found.' % repo_id
return api_error(status.HTTP_404_NOT_FOUND, error_msg)
permission = check_folder_permission(request, repo_id, os.path.dirname(path))
if not permission:
error_msg = 'Permission denied.'
return api_error(status.HTTP_403_FORBIDDEN, error_msg)
try:
file_id = seafile_api.get_file_id_by_path(repo_id, path)
except SearpcError as e:
logger.error(e)
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error')
if not file_id:
return api_error(status.HTTP_404_NOT_FOUND, f"File {path} not found")
token = seafile_api.get_fileserver_access_token(repo_id, file_id, 'download', request.user.username, use_onetime=True)
if not token:
error_msg = 'Internal Server Error'
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
params = {
'path': path,
'download_token': token
}
try:
resp = generate_summary(params)
resp_json = resp.json()
except Exception as e:
error_msg = 'Internal Server Error'
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
return Response(resp_json, resp.status_code)

35
seahub/ai/utils.py Normal file
View File

@ -0,0 +1,35 @@
import logging
import requests
import jwt
import time
from urllib.parse import urljoin
from seahub.settings import SEAFILE_AI_SECRET_KEY, SEAFILE_AI_SERVER_URL
logger = logging.getLogger(__name__)
def gen_headers():
payload = {'exp': int(time.time()) + 300, }
token = jwt.encode(payload, SEAFILE_AI_SECRET_KEY, algorithm='HS256')
return {"Authorization": "Token %s" % token}
def verify_ai_config():
if not SEAFILE_AI_SERVER_URL or not SEAFILE_AI_SECRET_KEY:
return False
return True
def image_caption(params):
headers = gen_headers()
url = urljoin(SEAFILE_AI_SERVER_URL, '/api/v1/image-caption/')
resp = requests.post(url, json=params, headers=headers, timeout=30)
return resp
def generate_summary(params):
headers = gen_headers()
url = urljoin(SEAFILE_AI_SERVER_URL, '/api/v1/generate-summary')
resp = requests.post(url, json=params, headers=headers, timeout=30)
return resp

64
seahub/api2/endpoints/metadata_manage.py
View File

@ -6,17 +6,16 @@ from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework import status
from rest_framework.views import APIView
from seahub.api2.utils import api_error, to_python_boolean
from seahub.api2.utils import api_error
from seahub.api2.throttling import UserRateThrottle
from seahub.api2.authentication import TokenAuthentication
from seahub.repo_metadata.models import RepoMetadata, RepoMetadataViews
from seahub.views import check_folder_permission
from seahub.repo_metadata.utils import add_init_metadata_task, gen_unique_id, init_metadata, \
get_sys_columns, update_docs_summary, get_file_download_token
get_sys_columns
from seahub.repo_metadata.metadata_server_api import MetadataServerAPI, list_metadata_view_records
from seahub.utils.timeutils import datetime_to_isoformat_timestr
from seahub.utils.repo import is_repo_admin
from pysearpc import SearpcError
from seaserv import seafile_api
@ -828,62 +827,3 @@ class MetadataViewsMoveView(APIView):
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
return Response({'navigation': results['navigation']})
class MetadataSummarizeDocs(APIView):
authentication_classes = (TokenAuthentication, SessionAuthentication)
permission_classes = (IsAuthenticated,)
throttle_classes = (UserRateThrottle,)
def post(self, request, repo_id):
file_paths_list = request.data.get('file_paths_list', '')
if not file_paths_list or not isinstance(file_paths_list, list):
error_msg = 'file_paths_list should be a non-empty list..'
return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
record = RepoMetadata.objects.filter(repo_id=repo_id).first()
if not record or not record.enabled:
error_msg = f'The metadata module is disabled for repo {repo_id}.'
return api_error(status.HTTP_404_NOT_FOUND, error_msg)
repo = seafile_api.get_repo(repo_id)
if not repo:
error_msg = 'Library %s not found.' % repo_id
return api_error(status.HTTP_404_NOT_FOUND, error_msg)
permission = check_folder_permission(request, repo_id, '/')
if permission != 'rw':
error_msg = 'Permission denied.'
return api_error(status.HTTP_403_FORBIDDEN, error_msg)
files_info_list = []
for file_path in file_paths_list:
try:
file_id = seafile_api.get_file_id_by_path(repo_id, file_path)
except SearpcError as e:
logger.error(e)
return api_error(
status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error'
)
if not file_id:
return api_error(
status.HTTP_404_NOT_FOUND, f"File {file_path} not found"
)
if token := get_file_download_token(repo_id, file_id, request.user.username):
files_info_list.append(
{'file_path': file_path, 'download_token': token}
)
else:
error_msg = 'Internal Server Error'
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
try:
resp = update_docs_summary(repo_id, files_info_list)
resp_json = resp.json()
except Exception as e:
error_msg = 'Internal Server Error'
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, error_msg)
return Response(resp_json, resp.status_code)

15
seahub/repo_metadata/utils.py
View File

@ -5,10 +5,11 @@ import json
import random
from urllib.parse import urljoin
from seahub.settings import SECRET_KEY, SEAFEVENTS_SERVER_URL, SEAFILE_AI_SECRET_KEY, SEAFILE_AI_SERVER_URL
from seahub.settings import SECRET_KEY, SEAFEVENTS_SERVER_URL
from seaserv import seafile_api
def add_init_metadata_task(params):
payload = {'exp': int(time.time()) + 300, }
token = jwt.encode(payload, SECRET_KEY, algorithm='HS256')
@ -66,17 +67,5 @@ def init_metadata(metadata_server_api):
metadata_server_api.add_columns(METADATA_TABLE.id, sys_columns)
def update_docs_summary(repo_id, files_info_list):
payload = {'exp': int(time.time()) + 300, }
token = jwt.encode(payload, SEAFILE_AI_SECRET_KEY, algorithm='HS256')
headers = {"Authorization": "Token %s" % token}
url = urljoin(SEAFILE_AI_SERVER_URL, '/api/v1/update-docs-summary')
params = {
'repo_id': repo_id,
'files_info_list': files_info_list,
}
resp = requests.post(url, json=params, headers=headers)
return resp
def get_file_download_token(repo_id, file_id, username):
return seafile_api.get_fileserver_access_token(repo_id, file_id, 'download', username, use_onetime=True)

1
seahub/settings.py
View File

@ -257,6 +257,7 @@ INSTALLED_APPS = [
'seahub.profile',
'seahub.share',
'seahub.help',
'seahub.ai',
'seahub.thumbnail',
'seahub.password_session',
'seahub.admin_log',

10
seahub/urls.py
View File

@ -2,6 +2,7 @@
from django.urls import include, path, re_path
from django.views.generic import TemplateView
from seahub.ai.apis import ImageCaption, GenerateSummary
from seahub.api2.endpoints.share_link_auth import ShareLinkUserAuthView, ShareLinkEmailAuthView
from seahub.api2.endpoints.internal_api import InternalUserListView
from seahub.auth.views import multi_adfs_sso
@ -209,7 +210,7 @@ from seahub.api2.endpoints.wiki2 import Wikis2View, Wiki2View, Wiki2ConfigView,
Wiki2DuplicatePageView, WikiPageTrashView
from seahub.api2.endpoints.subscription import SubscriptionView, SubscriptionPlansView, SubscriptionLogsView
from seahub.api2.endpoints.metadata_manage import MetadataRecords, MetadataManage, MetadataColumns, MetadataRecordInfo, \
MetadataViews, MetadataViewsMoveView, MetadataViewsDetailView, MetadataSummarizeDocs, MetadataViewsDuplicateView
MetadataViews, MetadataViewsMoveView, MetadataViewsDetailView, MetadataViewsDuplicateView
from seahub.api2.endpoints.user_list import UserListView
@ -1038,6 +1039,11 @@ if settings.ENABLE_METADATA_MANAGEMENT:
re_path(r'^api/v2.1/repos/(?P<repo_id>[-0-9a-f]{36})/metadata/views/$', MetadataViews.as_view(), name='api-v2.1-metadata-views'),
re_path(r'^api/v2.1/repos/(?P<repo_id>[-0-9a-f]{36})/metadata/views/(?P<view_id>[-0-9a-zA-Z]{4})/$', MetadataViewsDetailView.as_view(), name='api-v2.1-metadata-views-detail'),
re_path(r'^api/v2.1/repos/(?P<repo_id>[-0-9a-f]{36})/metadata/move-views/$', MetadataViewsMoveView.as_view(), name='api-v2.1-metadata-views-move'),
re_path(r'^api/v2.1/repos/(?P<repo_id>[-0-9a-f]{36})/metadata/ai/summarize-documents/$', MetadataSummarizeDocs.as_view(), name='api-v2.1-metadata-summarize-documents'),
re_path(r'^api/v2.1/repos/(?P<repo_id>[-0-9a-f]{36})/metadata/duplicate-view/$', MetadataViewsDuplicateView.as_view(), name='api-v2.1-metadata-view-duplicate'),
]
# ai API
urlpatterns += [
re_path(r'^api/v2.1/ai/image-caption/$', ImageCaption.as_view(), name='api-v2.1-image-caption'),
re_path(r'^api/v2.1/ai/generate-summary/$', GenerateSummary.as_view(), name='api-v2.1-generate-summary'),
]