update export file access (#6112)

* update export file access

* optimize

* update

* update

* optimeze code

* optimize

* Update utils.py

* Update utils.py

* optimize code

* update

* update

* optimize code

---------

Co-authored-by: 孙永强 <11704063+s-yongqiang@user.noreply.gitee.com>
Co-authored-by: r350178982 <32759763+r350178982@users.noreply.github.com>

frontend/src/components/dialog/sysadmin-dialog/sysadmin-logs-export-excel-dialog.js

@@ -2,6 +2,9 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { Button, Modal, ModalHeader, ModalBody, ModalFooter, FormGroup, Label, Input, Alert } from 'reactstrap';
 import { gettext, siteRoot } from '../../../utils/constants';
+import { systemAdminAPI } from '../../../utils/system-admin-api';
+import toaster from '../../../components/toast';
+import { Utils } from '../../../utils/utils';
 import moment from 'moment';
 
 class LogsExportExcelDialog extends React.Component {
@@ -13,6 +16,7 @@ class LogsExportExcelDialog extends React.Component {
       startDateStr: '',
       endDateStr: '',
       errMsg: '',
+      taskId: '',
     };
   }
 
@@ -20,25 +24,65 @@ class LogsExportExcelDialog extends React.Component {
     if (!this.isValidDateStr()) {
       return;
     }
-    let { startDateStr, endDateStr } = this.state;
-    let url = siteRoot;
-
     switch (this.props.logType) {
       case 'login':
-        url += 'sys/loginadmin/export-excel/';
+        this.sysExportLogs('loginadmin');
         break;
       case 'fileAccess':
-        url += 'sys/log/fileaudit/export-excel/';
+        this.sysExportLogs('fileaudit');
         break;
       case 'fileUpdate':
-        url += 'sys/log/fileupdate/export-excel/';
+        this.sysExportLogs('fileupdate');
         break;
       case 'sharePermission':
-        url += 'sys/log/permaudit/export-excel/';
+        this.sysExportLogs('permaudit');
         break;
     }
-    location.href = url + '?start=' + startDateStr + '&end=' + endDateStr;
+  };
-    this.props.toggle();
+
+  sysExportLogs = (logType) =>{
+    let { startDateStr, endDateStr } = this.state;
+    let task_id = '';
+    systemAdminAPI.sysAdminExportLogsExcel(startDateStr, endDateStr, logType).then(res => {
+      task_id = res.data.task_id;
+      this.setState({
+        taskId: task_id
+      });
+      this.props.toggle();
+      return systemAdminAPI.queryAsyncOperationExportExcel(task_id);
+    }).then(res => {
+      if (res.data.is_finished === true){
+        location.href = siteRoot + 'sys/log/export-excel/?task_id=' + task_id + '&log_type=' + logType;
+      } else {
+        this.timer = setInterval(() => {
+          systemAdminAPI.queryAsyncOperationExportExcel(task_id).then(res => {
+            if (res.data.is_finished === true){
+              this.setState({isFinished: true});
+              clearInterval(this.timer);
+              location.href = siteRoot + 'sys/log/export-excel/?task_id=' + task_id + '&log_type=' + logType;
+            }
+          }).catch(err => {
+            if (this.state.isFinished === false){
+              clearInterval(this.timer);
+              toaster.danger(gettext('Failed to export. Please check whether the size of table attachments exceeds the limit.'));
+            }
+          });
+        }, 1000);
+      }
+    }).catch(error => {
+      this.props.toggle();
+      if (error.response && error.response.status === 500) {
+        const error_msg = error.response.data ? error.response.data['error_msg'] : null;
+        if (error_msg && error_msg !== 'Internal Server Error') {
+          toaster.danger(error_msg);
+        } else {
+          toaster.danger(gettext('Internal Server Error.'));
+        }
+      } else {
+        let errMessage = Utils.getErrorMsg(error);
+        toaster.danger(errMessage);
+      }
+    });
   };
 
   isValidDateStr = () => {

frontend/src/utils/system-admin-api.js

@@ -66,6 +66,21 @@ class SystemAdminAPI {
     return this.req.get(url, {params: params});
   }
 
+  sysAdminExportLogsExcel(start, end, logType) {
+    const url = this.server + '/api/v2.1/admin/logs/export-excel/';
+    const params = {
+      start: start,
+      end: end,
+      logType: logType
+    };
+    return this.req.get(url, { params: params });
+  }
+
+  queryAsyncOperationExportExcel(task_id) {
+    const url = this.server + '/api/v2.1/query-export-status/?task_id=' + task_id;
+    return this.req.get(url);
+  }
+
 }
 
 let systemAdminAPI = new SystemAdminAPI();

seahub/api2/endpoints/admin/logs_export.py

@@ -0,0 +1,102 @@
+import os
+import logging
+import time
+import json
+from shutil import rmtree
+from django.http import FileResponse
+from rest_framework.authentication import SessionAuthentication
+from rest_framework.permissions import IsAdminUser
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from rest_framework import status
+from urllib.parse import quote
+
+from seahub.api2.authentication import TokenAuthentication
+from seahub.api2.endpoints.utils import check_time_period_valid, export_logs_to_excel, event_export_status
+from seahub.api2.permissions import IsProVersion
+from seahub.api2.throttling import UserRateThrottle
+from seahub.api2.utils import api_error
+from seahub.auth.decorators import login_required
+from seahub.base.decorators import sys_staff_required
+
+
+class SysLogsExport(APIView):
+    authentication_classes = (TokenAuthentication, SessionAuthentication)
+    permission_classes = (IsAdminUser, IsProVersion)
+    throttle_classes = (UserRateThrottle,)
+
+    def get(self, request):
+        start = request.GET.get('start', None)
+        end = request.GET.get('end', None)
+        log_type = request.GET.get('logType', None)
+
+        if not check_time_period_valid(start, end):
+            error_msg = 'Failed to export excel, invalid start or end date.'
+            return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
+
+        task_id = export_logs_to_excel(start, end, log_type)
+        res_data = {'task_id': task_id}
+        return Response(res_data)
+
+
+class FileLogsExportStatus(APIView):
+    authentication_classes = (TokenAuthentication, SessionAuthentication)
+    permission_classes = (IsAdminUser, IsProVersion)
+    throttle_classes = (UserRateThrottle,)
+
+    def get(self, request):
+        """
+        Get task status by task id
+        """
+        task_id = request.GET.get('task_id', '')
+        if not task_id:
+            error_msg = 'task_id invalid.'
+            return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
+
+        resp = event_export_status(task_id)
+        if resp.status_code == 500:
+            logger.error('query export status error: %s, %s' % (task_id, resp.content))
+            return api_error(500, 'Internal Server Error')
+        if not resp.status_code == 200:
+            return api_error(resp.status_code, resp.content)
+
+        is_finished = json.loads(resp.content)['is_finished']
+
+        return Response({'is_finished': is_finished})
+
+
+@login_required
+@sys_staff_required
+def sys_log_export_excel(request):
+    task_id = request.GET.get('task_id', None)
+    log_type = request.GET.get('log_type', None)
+    
+    if not task_id:
+        error_msg = 'task_id invalid.'
+        return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
+
+    if log_type == 'loginadmin':
+        excel_name = 'login-logs.xlsx'
+    elif log_type == 'fileaudit':
+        excel_name = 'file-access-logs.xlsx'
+    elif log_type == 'fileupdate':
+        excel_name = 'file-update-logs.xlsx'
+    elif log_type == 'permaudit':
+        excel_name = 'perm-audit-logs.xlsx'
+    else:
+        error_msg = 'log_type invalid'
+        return api_error(status.HTTP_400_BAD_REQUEST, error_msg)
+
+    target_dir = os.path.join('/tmp/seafile_events/', task_id)
+    tmp_excel_path = os.path.join(target_dir, excel_name)
+    if not os.path.isfile(tmp_excel_path):
+        return api_error(status.HTTP_400_BAD_REQUEST, excel_name + ' not found.')
+
+    response = FileResponse(open(tmp_excel_path, 'rb'), content_type='application/ms-excel', as_attachment=True)
+    try:
+        rmtree(target_dir)
+    except OSError:
+        pass
+    response['Content-Disposition'] = 'attachment;filename*=UTF-8\'\'' + quote(excel_name)
+
+    return response

seahub/api2/endpoints/utils.py

@@ -4,6 +4,7 @@ import jwt
 import time
 import logging
 import requests
+import json
 import datetime
 import urllib.request
 import urllib.parse
@@ -18,7 +19,8 @@ from seahub.api2.utils import api_error
 from seahub.base.templatetags.seahub_tags import email2nickname, email2contact_email
 from seahub.utils import get_log_events_by_time, is_pro_version, is_org_context
 
-from seahub.settings import SEADOC_PRIVATE_KEY, FILE_CONVERTER_SERVER_URL
+from seahub.settings import SEADOC_PRIVATE_KEY, FILE_CONVERTER_SERVER_URL, SECRET_KEY, \
+                            SEAFEVENTS_SERVER_URL
 
 try:
     from seahub.settings import MULTI_TENANCY
@@ -282,3 +284,35 @@ def sdoc_export_to_docx(path, username, doc_uuid, download_token,
     resp = requests.post(url, json=params, headers=headers, timeout=30)
 
     return resp
+
+
+def format_date(start, end):
+    start_struct_time = datetime.datetime.strptime(start, "%Y-%m-%d")
+    start_timestamp = time.mktime(start_struct_time.timetuple())
+
+    end_struct_time = datetime.datetime.strptime(end, "%Y-%m-%d")
+    end_timestamp = time.mktime(end_struct_time.timetuple())
+    end_timestamp += 24 * 60 * 60
+    return start_timestamp, end_timestamp
+
+
+def export_logs_to_excel(start, end, log_type):
+    start_time, end_time = format_date(start, end)
+    payload = {'exp': int(time.time()) + 300, }
+    token = jwt.encode(payload, SECRET_KEY, algorithm='HS256')
+    headers = {"Authorization": "Token %s" % token}
+    url = urljoin(SEAFEVENTS_SERVER_URL, '/add-init-export-log-task')
+    params = {'start_time': start_time, 'end_time': end_time, 'log_type': log_type}
+    resp = requests.get(url, params=params, headers=headers)
+    return json.loads(resp.content)['task_id']
+
+
+def event_export_status(task_id):
+    payload = {'exp': int(time.time()) + 300, }
+    token = jwt.encode(payload, SECRET_KEY, algorithm='HS256')
+    headers = {"Authorization": "Token %s" % token}
+    url = urljoin(SEAFEVENTS_SERVER_URL, '/query-export-status')
+    params = {'task_id': task_id}
+    resp = requests.get(url, params=params, headers=headers)
+
+    return resp

seahub/sysadmin_extra/views.py

@@ -1,262 +0,0 @@
-# Copyright (c) 2012-2016 Seafile Ltd.
-import os
-import logging
-
-from django.shortcuts import render
-
-from django.utils.translation import gettext as _
-from django.contrib import messages
-from django.http import HttpResponse, HttpResponseRedirect, Http404
-from django.core.exceptions import ValidationError
-
-from seahub.api2.endpoints.utils import check_time_period_valid, \
-    get_log_events_by_type_and_time
-
-from seahub.base.decorators import sys_staff_required
-from seahub.auth.decorators import login_required
-from seahub.sysadmin_extra.models import UserLoginLog
-from seahub.utils import EVENTS_ENABLED, get_file_audit_events, \
-    get_file_update_events, get_perm_audit_events, \
-    is_pro_version, generate_file_audit_event_type
-from seahub.utils.timeutils import utc_to_local
-from seahub.utils.ms_excel import write_xls
-from seahub.settings import SITE_ROOT
-
-from seaserv import seafile_api, ccnet_api
-
-logger = logging.getLogger(__name__)
-
-@login_required
-@sys_staff_required
-def sys_login_admin_export_excel(request):
-    """ Export user login logs to excel.
-    """
-    next_page = request.headers.get('referer', None)
-    if not next_page:
-        next_page = SITE_ROOT
-
-    start = request.GET.get('start', None)
-    end = request.GET.get('end', None)
-
-    if not check_time_period_valid(start, end):
-        messages.error(request, _('Failed to export excel, invalid start or end date'))
-        return HttpResponseRedirect(next_page)
-
-    # Filtering a DateTimeField with dates won't include items on the last day,
-    # because the bounds are interpreted as '0am on the given date'.
-    end = end + ' 23:59:59'
-
-    try:
-        user_login_logs = UserLoginLog.objects.filter(login_date__range=(start, end))
-    except ValidationError as e:
-        logger.error(e)
-        messages.error(request, _('Failed to export excel, invalid start or end date'))
-        return HttpResponseRedirect(next_page)
-
-    logs = list(user_login_logs)
-    head = [_("Name"), _("IP"), _("Status"), _("Time")]
-    data_list = []
-    for log in logs:
-        login_time = log.login_date.strftime("%Y-%m-%d %H:%M:%S")
-        status = _('Success') if log.login_success else _('Failed')
-        row = [log.username, log.login_ip, status, login_time]
-        data_list.append(row)
-
-    wb = write_xls('login-logs', head, data_list)
-    if not wb:
-        messages.error(request, _('Failed to export excel'))
-        return HttpResponseRedirect(next_page)
-
-    response = HttpResponse(content_type='application/ms-excel')
-    response['Content-Disposition'] = 'attachment; filename=login-logs.xlsx'
-    wb.save(response)
-    return response
-
-@login_required
-@sys_staff_required
-def sys_log_file_audit_export_excel(request):
-    """ Export file access logs to excel.
-    """
-    next_page = request.headers.get('referer', None)
-    if not next_page:
-        next_page = SITE_ROOT
-
-    if not is_pro_version():
-        messages.error(request, _('Failed to export excel, this feature is only in professional version.'))
-        return HttpResponseRedirect(next_page)
-
-    start = request.GET.get('start', None)
-    end = request.GET.get('end', None)
-    if not check_time_period_valid(start, end):
-        messages.error(request, _('Failed to export excel, invalid start or end date'))
-        return HttpResponseRedirect(next_page)
-
-    events = get_log_events_by_type_and_time('file_audit', start, end)
-
-    head = [_("User"), _("Type"), _("IP"), _("Device"), _("Date"),
-            _("Library Name"), _("Library ID"), _("Library Owner"), _("File Path")]
-    data_list = []
-
-    events.sort(key=lambda x: x.timestamp, reverse=True)
-    for ev in events:
-        event_type, ev.show_device = generate_file_audit_event_type(ev)
-
-        repo_id = ev.repo_id
-        repo = seafile_api.get_repo(repo_id)
-        if repo:
-            repo_name = repo.name
-            repo_owner = seafile_api.get_repo_owner(repo_id) or \
-                    seafile_api.get_org_repo_owner(repo_id)
-        else:
-            repo_name = _('Deleted')
-            repo_owner = '--'
-
-        username = ev.user if ev.user else _('Anonymous User')
-        date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \
-            ev.timestamp else ''
-
-        row = [username, event_type, ev.ip, ev.show_device,
-               date, repo_name, ev.repo_id, repo_owner, ev.file_path]
-        data_list.append(row)
-
-    wb = write_xls('file-access-logs', head, data_list)
-    if not wb:
-        messages.error(request, _('Failed to export excel'))
-        return HttpResponseRedirect(next_page)
-
-    response = HttpResponse(content_type='application/ms-excel')
-    response['Content-Disposition'] = 'attachment; filename=file-access-logs.xlsx'
-    wb.save(response)
-    return response
-
-@login_required
-@sys_staff_required
-def sys_log_file_update_export_excel(request):
-    """ Export file update logs to excel.
-    """
-    next_page = request.headers.get('referer', None)
-    if not next_page:
-        next_page = SITE_ROOT
-
-    if not is_pro_version():
-        messages.error(request, _('Failed to export excel, this feature is only in professional version.'))
-        return HttpResponseRedirect(next_page)
-
-    start = request.GET.get('start', None)
-    end = request.GET.get('end', None)
-    if not check_time_period_valid(start, end):
-        messages.error(request, _('Failed to export excel, invalid start or end date'))
-        return HttpResponseRedirect(next_page)
-
-    events = get_log_events_by_type_and_time('file_update', start, end)
-
-    head = [_("User"), _("Date"), _("Library Name"), _("Library ID"),
-            _("Library Owner"), _("Action")]
-    data_list = []
-
-    events.sort(key=lambda x: x.timestamp, reverse=True)
-    for ev in events:
-
-        repo_id = ev.repo_id
-        repo = seafile_api.get_repo(repo_id)
-        if repo:
-            repo_name = repo.name
-            repo_owner = seafile_api.get_repo_owner(repo_id) or \
-                    seafile_api.get_org_repo_owner(repo_id)
-        else:
-            repo_name = _('Deleted')
-            repo_owner = '--'
-
-        username = ev.user if ev.user else _('Anonymous User')
-        date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \
-            ev.timestamp else ''
-
-        row = [username, date, repo_name, ev.repo_id, repo_owner, ev.file_oper.strip()]
-        data_list.append(row)
-
-    wb = write_xls('file-update-logs', head, data_list)
-    if not wb:
-        messages.error(request, _('Failed to export excel'))
-        return HttpResponseRedirect(next_page)
-
-    response = HttpResponse(content_type='application/ms-excel')
-    response['Content-Disposition'] = 'attachment; filename=file-update-logs.xlsx'
-    wb.save(response)
-    return response
-
-@login_required
-@sys_staff_required
-def sys_log_perm_audit_export_excel(request):
-    """ Export permission audit logs to excel.
-    """
-    next_page = request.headers.get('referer', None)
-    if not next_page:
-        next_page = SITE_ROOT
-
-    if not is_pro_version():
-        messages.error(request, _('Failed to export excel, this feature is only in professional version.'))
-        return HttpResponseRedirect(next_page)
-
-    start = request.GET.get('start', None)
-    end = request.GET.get('end', None)
-    if not check_time_period_valid(start, end):
-        messages.error(request, _('Failed to export excel, invalid start or end date'))
-        return HttpResponseRedirect(next_page)
-
-    events = get_log_events_by_type_and_time('perm_audit', start, end)
-
-    head = [_("From"), _("To"), _("Action"), _("Permission"), _("Library"),
-            _("Folder Path"), _("Date")]
-    data_list = []
-
-    events.sort(key=lambda x: x.timestamp, reverse=True)
-    for ev in events:
-        repo = seafile_api.get_repo(ev.repo_id)
-        repo_name = repo.repo_name if repo else _('Deleted')
-
-        if '@' in ev.to:
-            to = ev.to
-        elif ev.to.isdigit():
-            group = ccnet_api.get_group(int(ev.to))
-            to = group.group_name if group else _('Deleted')
-        elif 'all' in ev.to:
-            to = _('Organization')
-        else:
-            to = '--'
-
-        if 'add' in ev.etype:
-            action = _('Add')
-        elif 'modify' in ev.etype:
-            action = _('Modify')
-        elif 'delete' in ev.etype:
-            action = _('Delete')
-        else:
-            action = '--'
-
-        if ev.permission == 'rw':
-            permission = _('Read-Write')
-        elif ev.permission == 'r':
-            permission = _('Read-Only')
-        else:
-            permission = '--'
-
-        date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \
-            ev.timestamp else ''
-
-        row = [ev.from_user, to, action, permission, repo_name,
-               ev.file_path, date]
-        data_list.append(row)
-
-    wb = write_xls('perm-audit-logs', head, data_list)
-    if not wb:
-        next_page = request.headers.get('referer', None)
-        if not next_page:
-            next_page = SITE_ROOT
-
-        messages.error(request, _('Failed to export excel'))
-        return HttpResponseRedirect(next_page)
-
-    response = HttpResponse(content_type='application/ms-excel')
-    response['Content-Disposition'] = 'attachment; filename=perm-audit-logs.xlsx'
-    wb.save(response)
-    return response

seahub/urls.py

@@ -121,6 +121,7 @@ from seahub.api2.endpoints.repo_share_links import RepoShareLinks, RepoShareLink
 from seahub.api2.endpoints.repo_upload_links import RepoUploadLinks, RepoUploadLink
 
 # Admin
+from seahub.api2.endpoints.admin.logs_export import SysLogsExport, FileLogsExportStatus, sys_log_export_excel
 from seahub.api2.endpoints.admin.abuse_reports import AdminAbuseReportsView, AdminAbuseReportView
 from seahub.api2.endpoints.admin.revision_tag import AdminTaggedItemsView
 from seahub.api2.endpoints.admin.login_logs import LoginLogs, AdminLoginLogs
@@ -886,21 +887,17 @@ if HAS_FILE_SEARCH:
 
 from seahub.utils import is_pro_version
 if is_pro_version():
-    from seahub.sysadmin_extra.views import \
-        sys_login_admin_export_excel, sys_log_file_audit_export_excel, \
-        sys_log_file_update_export_excel, sys_log_perm_audit_export_excel
     urlpatterns += [
         re_path(r'^api/v2.1/admin/logs/login/$', LoginLogs.as_view(), name='api-v2.1-admin-logs-login'),
-        path('sys/loginadmin/export-excel/', sys_login_admin_export_excel, name='sys_login_admin_export_excel'),
-
         re_path(r'^api/v2.1/admin/logs/file-audit/$', FileAudit.as_view(), name='api-v2.1-admin-logs-file-audit'),
-        path('sys/log/fileaudit/export-excel/', sys_log_file_audit_export_excel, name='sys_log_file_audit_export_excel'),
-
         re_path(r'^api/v2.1/admin/logs/file-update/$', FileUpdate.as_view(), name='api-v2.1-admin-logs-file-update'),
-        path('sys/log/fileupdate/export-excel/', sys_log_file_update_export_excel, name='sys_log_file_update_export_excel'),
-
         re_path(r'^api/v2.1/admin/logs/perm-audit/$', PermAudit.as_view(), name='api-v2.1-admin-logs-perm-audit'),
-        path('sys/log/permaudit/export-excel/', sys_log_perm_audit_export_excel, name='sys_log_perm_audit_export_excel'),
+
+        re_path(r'^api/v2.1/admin/logs/export-excel/$', SysLogsExport.as_view(), name='api-v2.1-admin-logs-export-excel'),
+        re_path(r'^api/v2.1/query-export-status/$', FileLogsExportStatus.as_view(), name='api-v2.1-query-export-status'),
+        path('sys/log/export-excel/', sys_log_export_excel, name='sys_log_export_excel'),
+
+
     ]
 
 if getattr(settings, 'MULTI_TENANCY', False):

seahub/utils/__init__.py

@@ -1165,7 +1165,6 @@ if HAS_OFFICE_CONVERTER:
         try:
             add_office_convert_task(obj_id, doctype, raw_path)
         except Exception as e:
-            print(e)
             logging.exception('failed to add_office_convert_task: %s' % e)
             return _('Internal Server Error')
         return None