From 60c78adc44269e96d6eb8a63e3b830bbdf2d835d Mon Sep 17 00:00:00 2001 From: zMingGit <517046497@qq.com> Date: Wed, 9 Aug 2017 18:14:00 +0800 Subject: [PATCH] [api2] stat_file (#1690) * [api2] stat_file * [send message] update last login info * datetime->data, return number to int * fill data * modify func args * modify algorithm --- seahub/api2/endpoints/admin/statistics.py | 166 +++++++++++++++++++ seahub/api2/endpoints/admin/sysinfo.py | 13 +- seahub/api2/views.py | 8 +- seahub/urls.py | 6 + seahub/utils/__init__.py | 55 +++++- tests/api/endpoints/admin/test_statistics.py | 94 +++++++++++ 6 files changed, 334 insertions(+), 8 deletions(-) create mode 100644 seahub/api2/endpoints/admin/statistics.py create mode 100644 tests/api/endpoints/admin/test_statistics.py diff --git a/seahub/api2/endpoints/admin/statistics.py b/seahub/api2/endpoints/admin/statistics.py new file mode 100644 index 0000000000..164077e021 --- /dev/null +++ b/seahub/api2/endpoints/admin/statistics.py @@ -0,0 +1,166 @@ +# Copyright (c) 2012-2016 Seafile Ltd. +import datetime +import pytz + +from rest_framework.authentication import SessionAuthentication +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework import status +from django.utils import timezone + +from seahub.utils import get_file_ops_stats, get_file_ops_stats_by_day, \ + get_total_storage_stats, get_total_storage_stats_by_day, \ + get_user_activity_stats, get_user_activity_stats_by_day, \ + is_pro_version, EVENTS_ENABLED +from seahub.utils.timeutils import datetime_to_isoformat_timestr +from seahub.settings import TIME_ZONE + +from seahub.api2.authentication import TokenAuthentication +from seahub.api2.throttling import UserRateThrottle +from seahub.api2.utils import api_error + + + +def check_parameter(func): + def _decorated(view, request, *args, **kwargs): + if not is_pro_version() or not EVENTS_ENABLED: + return api_error(status.HTTP_404_NOT_FOUND, 'Events not enabled.') + start_time = request.GET.get("start", "") + end_time = request.GET.get("end", "") + group_by = request.GET.get("group_by", "hour") + if not start_time: + error_msg = "Start time can not be empty" + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + if not end_time: + error_msg = "End time can not be empty" + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + if group_by.lower() not in ["hour", "day"]: + error_msg = "group_by can only be day or hour." + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + try: + start_time = datetime.datetime.strptime(start_time, + "%Y-%m-%d %H:%M:%S") + except: + error_msg = "Start time %s invalid" % start_time + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + try: + end_time = datetime.datetime.strptime(end_time, + "%Y-%m-%d %H:%M:%S") + except: + error_msg = "End time %s invalid" % end_time + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + + return func(view, request, start_time, end_time, group_by) + return _decorated + + +class FileOperationsView(APIView): + """ + Get file operations statistics. + Permission checking: + 1. only admin can perform this action. + """ + authentication_classes = (TokenAuthentication, SessionAuthentication) + throttle_classes = (UserRateThrottle,) + permission_classes = (IsAdminUser,) + + @check_parameter + def get(self, request, start_time, end_time, group_by): + """ + Get records of the specified time range. + param: + start: the start time of the query. + end: the end time of the query. + group_by: group records by day or by hour, default group by hour. + return: + the list of file operations record. + """ + data = get_data_by_hour_or_day(group_by, start_time, end_time, get_file_ops_stats, get_file_ops_stats_by_day) + ops_added_dict = get_init_data(start_time, end_time, group_by) + ops_visited_dict = get_init_data(start_time, end_time, group_by) + ops_deleted_dict = get_init_data(start_time, end_time, group_by) + + for e in data: + if e[1] == 'Added': + ops_added_dict[e[0]] = e[2] + elif e[1] == 'Visited': + ops_visited_dict[e[0]] = e[2] + elif e[1] == 'Deleted': + ops_deleted_dict[e[0]] = e[2] + + res_data = [] + for k, v in ops_added_dict.items(): + res_data.append({'datetime': datetime_to_isoformat_timestr(k), + 'added': v, + 'visited': ops_visited_dict[k], + 'deleted': ops_deleted_dict[k]}) + return Response(sorted(res_data, key=lambda x: x['datetime'])) + + +class TotalStorageView(APIView): + authentication_classes = (TokenAuthentication, SessionAuthentication) + throttle_classes = (UserRateThrottle,) + permission_classes = (IsAdminUser,) + + @check_parameter + def get(self, request, start_time, end_time, group_by): + data = get_data_by_hour_or_day(group_by, start_time, end_time, get_total_storage_stats, get_total_storage_stats_by_day) + + res_data = [] + init_data = get_init_data(start_time, end_time, group_by) + for e in data: + init_data[e[0]] = e[1] + for k, v in init_data.items(): + res_data.append({'datetime': datetime_to_isoformat_timestr(k), 'total_storage': v}) + + return Response(sorted(res_data, key=lambda x: x['datetime'])) + + +class ActiveUsersView(APIView): + authentication_classes = (TokenAuthentication, SessionAuthentication) + throttle_classes = (UserRateThrottle,) + permission_classes = (IsAdminUser,) + + @check_parameter + def get(self, request, start_time, end_time, group_by): + data = get_data_by_hour_or_day(group_by, start_time, end_time, get_user_activity_stats, get_user_activity_stats_by_day) + + res_data = [] + init_data = get_init_data(start_time, end_time, group_by) + for e in data: + init_data[e[0]] = e[1] + for k, v in init_data.items(): + res_data.append({'datetime': datetime_to_isoformat_timestr(k), 'count': v}) + + return Response(sorted(res_data, key=lambda x: x['datetime'])) + + +def get_init_data(start_time, end_time, group_by): + res = {} + if group_by == 'hour': + start_time = start_time.replace(minute=0).replace(second=0) + end_time = end_time.replace(minute=0).replace(second=0) + time_delta = end_time - start_time + date_length = (time_delta.days * 24) + time_delta.seconds/3600 + 1 + else: + start_time = start_time.replace(hour=0).replace(minute=0).replace(second=0) + end_time = end_time.replace(hour=0).replace(minute=0).replace(second=0) + time_delta = end_time - start_time + date_length = time_delta.days + 1 + for offset in range(date_length): + offset = offset if group_by == 'hour' else offset * 24 + dt = start_time + datetime.timedelta(hours=offset) + res[dt] = 0 + return res + + +def get_data_by_hour_or_day(parameter, start_time, end_time, func, func_by_day): + timezone_name = timezone.get_current_timezone_name() + offset = pytz.timezone(timezone_name).localize(datetime.datetime.now()).strftime('%z') + offset = offset[:3] + ':' + offset[3:] + if parameter == "hour": + data = func(start_time, end_time, offset) + elif parameter == "day": + data = func_by_day(start_time, end_time, offset) + return data diff --git a/seahub/api2/endpoints/admin/sysinfo.py b/seahub/api2/endpoints/admin/sysinfo.py index 3b7dad759e..decad55318 100644 --- a/seahub/api2/endpoints/admin/sysinfo.py +++ b/seahub/api2/endpoints/admin/sysinfo.py @@ -82,11 +82,11 @@ class SysInfo(APIView): logger.error(e) inactive_ldap_users = 0 - active_users = active_db_users + active_ldap_users if active_ldap_users > 0 \ - else active_db_users + active_users = active_db_users + active_ldap_users if \ + active_ldap_users > 0 else active_db_users - inactive_users = inactive_db_users + inactive_ldap_users if inactive_ldap_users > 0 \ - else inactive_db_users + inactive_users = inactive_db_users + inactive_ldap_users if \ + inactive_ldap_users > 0 else inactive_db_users # get license info is_pro = is_pro_version() @@ -129,10 +129,11 @@ class SysInfo(APIView): # count current connected devices try: - current_connected_devices_count = TokenV2.objects.get_current_connected_devices_count() + current_connected_devices_count = TokenV2.objects.\ + get_current_connected_devices_count() except Exception as e: logger.error(e) - current_connected_devices_count= 0 + current_connected_devices_count = 0 info = { 'users_count': active_users + inactive_users, diff --git a/seahub/api2/views.py b/seahub/api2/views.py index b0257a7d55..a5d577fe1c 100644 --- a/seahub/api2/views.py +++ b/seahub/api2/views.py @@ -120,7 +120,7 @@ from seaserv import seafserv_threaded_rpc, \ is_group_user, remove_share, get_group, \ get_commit, get_file_id_by_path, MAX_DOWNLOAD_DIR_SIZE, edit_repo, \ ccnet_threaded_rpc, get_personal_groups, seafile_api, \ - create_org, ccnet_api + create_org, ccnet_api, send_message from constance import config @@ -578,6 +578,12 @@ class Repos(APIView): } repos_json.append(repo) + utc_dt = datetime.datetime.utcnow() + timestamp = utc_dt.strftime('%Y-%m-%d %H:%M:%S') + try: + send_message('seahub.stats', 'user-login\t%s\t%s' % (email, timestamp)) + except Exception as e: + logger.error('Error when sending user-login message: %s' % str(e)) response = HttpResponse(json.dumps(repos_json), status=200, content_type=json_content_type) response["enable_encrypted_library"] = config.ENABLE_ENCRYPTED_LIBRARY diff --git a/seahub/urls.py b/seahub/urls.py index 96ebf6702f..7569c09be0 100644 --- a/seahub/urls.py +++ b/seahub/urls.py @@ -56,6 +56,7 @@ from seahub.api2.endpoints.admin.file_audit import FileAudit from seahub.api2.endpoints.admin.file_update import FileUpdate from seahub.api2.endpoints.admin.perm_audit import PermAudit from seahub.api2.endpoints.admin.sysinfo import SysInfo +from seahub.api2.endpoints.admin.statistics import FileOperationsView, TotalStorageView, ActiveUsersView from seahub.api2.endpoints.admin.devices import AdminDevices from seahub.api2.endpoints.admin.device_errors import AdminDeviceErrors from seahub.api2.endpoints.admin.libraries import AdminLibraries, AdminLibrary @@ -255,6 +256,11 @@ urlpatterns = patterns( ## admin::revision-tags url(r'^api/v2.1/admin/revision-tags/tagged-items/$', AdminTaggedItemsView.as_view(), name='api-v2.1-admin-revision-tags-tagged-items'), + ## admin::statistics + url(r'^api/v2.1/admin/statistics/file-operations/$', FileOperationsView.as_view(), name='api-v2.1-admin-statistics-file-operations'), + url(r'^api/v2.1/admin/statistics/total-storage/$', TotalStorageView.as_view(), name='api-v2.1-admin-statistics-total-storage'), + url(r'^api/v2.1/admin/statistics/active-users/$', ActiveUsersView.as_view(), name='api-v2.1-admin-statistics-active-users'), + ## admin::devices url(r'^api/v2.1/admin/devices/$', AdminDevices.as_view(), name='api-v2.1-admin-devices'), url(r'^api/v2.1/admin/device-errors/$', AdminDeviceErrors.as_view(), name='api-v2.1-admin-device-errors'), diff --git a/seahub/utils/__init__.py b/seahub/utils/__init__.py index c6e936f893..7a9e7a2a04 100644 --- a/seahub/utils/__init__.py +++ b/seahub/utils/__init__.py @@ -627,6 +627,20 @@ if EVENTS_CONFIG_FILE: 15th events. """ return _get_events(username, start, count) + + def get_user_activity_stats(start, end, offset): + """ Return user activity record of the specified time + """ + with _get_seafevents_session() as session: + res = seafevents.get_user_activity_stats(session, start, end, offset) + return res + + def get_user_activity_stats_by_day(start, end, offset): + """ + """ + with _get_seafevents_session() as session: + res = seafevents.get_user_activity_stats_by_day(session, start, end, offset) + return res def get_org_user_events(org_id, username, start, count): return _get_events(username, start, count, org_id=org_id) @@ -677,6 +691,34 @@ if EVENTS_CONFIG_FILE: return events if events else None + def get_file_ops_stats(start, end, offset): + """ Return file audit record of specifiy time. + """ + with _get_seafevents_session() as session: + res = seafevents.get_file_ops_stats(session, start, end, offset) + return res + + def get_file_ops_stats_by_day(start, end, offset): + """ return file audit record of sepcifiy time group by day. + """ + with _get_seafevents_session() as session: + res = seafevents.get_file_ops_stats_by_day(session, start, end, offset) + return res + + def get_total_storage_stats(start, end, offset): + """ Return total storage record of specified time. + """ + with _get_seafevents_session() as session: + res = seafevents.get_total_storage_stats(session, start, end, offset) + return res + + def get_total_storage_stats_by_day(start, end, offset): + """ + """ + with _get_seafevents_session() as session: + res = seafevents.get_total_storage_stats_by_day(session, start, end, offset) + return res + def get_file_update_events(email, org_id, repo_id, start, limit): """Return file update events list. (If no file update, return 'None') @@ -688,7 +730,6 @@ if EVENTS_CONFIG_FILE: """ with _get_seafevents_session() as session: events = seafevents.get_file_update_events(session, email, org_id, repo_id, start, limit) - return events if events else None def get_perm_audit_events(email, org_id, repo_id, start, limit): @@ -721,6 +762,10 @@ else: EVENTS_ENABLED = False def get_user_events(): pass + def get_user_activity_stats(): + pass + def get_user_activity_stats_by_day(): + pass def get_log_events_by_time(): pass def get_org_user_events(): @@ -731,6 +776,14 @@ else: pass def get_file_audit_events(): pass + def get_file_ops_stats(): + pass + def get_file_ops_stats_by_day(): + pass + def get_total_storage_stats(): + pass + def get_total_storage_stats_by_day(): + pass def get_file_update_events(): pass def get_perm_audit_events(): diff --git a/tests/api/endpoints/admin/test_statistics.py b/tests/api/endpoints/admin/test_statistics.py new file mode 100644 index 0000000000..ad3f3e4801 --- /dev/null +++ b/tests/api/endpoints/admin/test_statistics.py @@ -0,0 +1,94 @@ +import json +import datetime +from datetime import date + +from mock import patch + +from django.core.urlresolvers import reverse +from seahub.test_utils import BaseTestCase +from seahub.utils.timeutils import datetime_to_isoformat_timestr + + +class FileOperationsInfoText(BaseTestCase): + def setUp(self): + self.login_as(self.admin) + + @patch("seahub.api2.endpoints.admin.statistics.get_file_ops_stats") + @patch("seahub.api2.endpoints.admin.statistics.get_file_ops_stats_by_day") + def test_can_get_file_audit_stats(self, mock_get_file_audit_stats_by_day, mock_get_file_audit_stats): + mock_get_file_audit_stats.return_value = [ + (datetime.datetime(2017, 6, 2, 7, 0), u'Added', 2L), + (datetime.datetime(2017, 6, 2, 7, 0), u'Deleted', 2L), + (datetime.datetime(2017, 6, 2, 7, 0), u'Visited', 2L), + (datetime.datetime(2017, 6, 2, 8, 0), u'Added', 3L), + (datetime.datetime(2017, 6, 2, 8, 0), u'Deleted', 4L), + (datetime.datetime(2017, 6, 2, 8, 0), u'Visited', 5L)] + mock_get_file_audit_stats_by_day.return_value = [ + (datetime.datetime(2017, 6, 2, 4, 2), u'Added', 2L), + (datetime.datetime(2017, 6, 2, 4, 2), u'Deleted', 2L), + (datetime.datetime(2017, 6, 2, 4, 2), u'Visited', 2L), + ] + url = reverse('api-v2.1-admin-statistics-file-operations') + url += "?start=2017-06-01 07:00:00&end=2017-06-03 07:00:00&group_by=hour" + resp = self.client.get(url) + json_resp = json.loads(resp.content) + self.assertEqual(200, resp.status_code) + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 7, 0)), + 'added': 2L, 'deleted': 2L, 'visited': 2L} + assert data in json_resp + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 8, 0)), + 'added': 3L, 'deleted': 4L, 'visited': 5L} + assert data in json_resp + url += "?start=2017-06-01 07:00:00&end=2017-06-03 07:00:00&group_by=day" + resp = self.client.get(url) + json_resp = json.loads(resp.content) + self.assertEqual(200, resp.status_code) + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 4, 2)), + 'added': 2, 'deleted': 2, 'visited': 2} + assert data in json_resp + + @patch("seahub.api2.endpoints.admin.statistics.get_user_activity_stats") + @patch("seahub.api2.endpoints.admin.statistics.get_user_activity_stats_by_day") + def test_can_user_activity_stats(self, mock_stats_by_day, mock_stats): + mock_stats.return_value = [(datetime.datetime(2017, 6, 2, 7, 0), 2L), + (datetime.datetime(2017, 6, 2, 8, 0), 5L)] + mock_stats_by_day.return_value = [(datetime.datetime(2017, 6, 2, 4, 0), 3L)] + url = reverse('api-v2.1-admin-statistics-active-users') + url += "?start=2017-06-01 07:00:00&end=2017-06-03 07:00:00&group_by=hour" + resp = self.client.get(url) + json_resp = json.loads(resp.content) + + self.assertEqual(200, resp.status_code) + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 7, 0)), 'count': 2} + assert data in json_resp + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 8, 0)), 'count': 5} + assert data in json_resp + url += "?start=2017-06-01 07:00:00&end=2017-06-03 07:00:00&group_by=day" + resp = self.client.get(url) + json_resp = json.loads(resp.content) + self.assertEqual(200, resp.status_code) + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 4, 0)), 'count': 3} + assert data in json_resp + + @patch("seahub.api2.endpoints.admin.statistics.get_total_storage_stats") + @patch("seahub.api2.endpoints.admin.statistics.get_total_storage_stats_by_day") + def test_can_get_total_storage_stats(self, mock_stats_by_day, mock_stats): + mock_stats.return_value = [(datetime.datetime(2017, 6, 2, 7, 0), 2L), + (datetime.datetime(2017, 6, 2, 8, 0), 5L)] + mock_stats_by_day.return_value = [(datetime.datetime(2017, 6, 2, 3, 0), 13L)] + url = reverse('api-v2.1-admin-statistics-total-storage') + url += "?start=2017-06-01 07:00:00&end=2017-06-03 07:00:00&group_by=hour" + resp = self.client.get(url) + + json_resp = json.loads(resp.content) + self.assertEqual(200, resp.status_code) + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 7, 0)), 'total_storage': 2} + assert data in json_resp + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 8, 0)), 'total_storage': 5} + assert data in json_resp + url += "?start=2017-06-01 07:00:00&end=2017-06-03 07:00:00&group_by=day" + resp = self.client.get(url) + json_resp = json.loads(resp.content) + self.assertEqual(200, resp.status_code) + data = {'datetime': datetime_to_isoformat_timestr(datetime.datetime(2017, 6, 2, 3, 0)), 'total_storage': 13} + assert data in json_resp