diff --git a/apps/common/utils/timezone.py b/apps/common/utils/timezone.py index 4b60008af..17e44b7bf 100644 --- a/apps/common/utils/timezone.py +++ b/apps/common/utils/timezone.py @@ -1,22 +1,21 @@ -import datetime - import pytz +from datetime import datetime, timedelta, timezone from django.utils import timezone as dj_timezone from rest_framework.fields import DateTimeField -max = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc) +max = datetime.max.replace(tzinfo=timezone.utc) -def astimezone(dt: datetime.datetime, tzinfo: pytz.tzinfo.DstTzInfo): +def astimezone(dt: datetime, tzinfo: pytz.tzinfo.DstTzInfo): assert dj_timezone.is_aware(dt) return tzinfo.normalize(dt.astimezone(tzinfo)) -def as_china_cst(dt: datetime.datetime): +def as_china_cst(dt: datetime): return astimezone(dt, pytz.timezone('Asia/Shanghai')) -def as_current_tz(dt: datetime.datetime): +def as_current_tz(dt: datetime): return astimezone(dt, dj_timezone.get_current_timezone()) @@ -36,6 +35,15 @@ def local_now_date_display(fmt='%Y-%m-%d'): return local_now().strftime(fmt) +def local_zero_hour(fmt='%Y-%m-%d'): + return datetime.strptime(local_now().strftime(fmt), fmt) + + +def local_monday(): + zero_hour_time = local_zero_hour() + return zero_hour_time - timedelta(zero_hour_time.weekday()) + + _rest_dt_field = DateTimeField() dt_parser = _rest_dt_field.to_internal_value dt_formatter = _rest_dt_field.to_representation diff --git a/apps/jumpserver/api.py b/apps/jumpserver/api.py index 59580d178..bfa50da4d 100644 --- a/apps/jumpserver/api.py +++ b/apps/jumpserver/api.py @@ -3,45 +3,51 @@ import time from django.core.cache import cache from django.utils import timezone from django.utils.timesince import timesince -from django.db.models import Count, Max +from django.db.models import Count, Max, F from django.http.response import JsonResponse, HttpResponse from rest_framework.views import APIView from rest_framework.permissions import AllowAny -from collections import Counter +from rest_framework.request import Request from rest_framework.response import Response from users.models import User from assets.models import Asset +from assets.const import AllTypes from terminal.models import Session from terminal.utils import ComponentsPrometheusMetricsUtil from orgs.utils import current_org from common.utils import lazyproperty +from common.utils.timezone import local_now, local_zero_hour from orgs.caches import OrgResourceStatisticsCache - __all__ = ['IndexApi'] class DatesLoginMetricMixin: + request: Request + @lazyproperty def days(self): query_params = self.request.query_params - if query_params.get('monthly'): - return 30 - return 7 + # monthly + count = query_params.get('days') + return count if count else 0 @lazyproperty def sessions_queryset(self): - days = timezone.now() - timezone.timedelta(days=self.days) - sessions_queryset = Session.objects.filter(date_start__gt=days) + days = self.days + if days == 0: + t = local_zero_hour() + else: + t = local_now() - timezone.timedelta(days=days) + sessions_queryset = Session.objects.filter(date_start__gte=t) return sessions_queryset @lazyproperty def session_dates_list(self): - now = timezone.now() + now = local_now() dates = [(now - timezone.timedelta(days=i)).date() for i in range(self.days)] dates.reverse() - # dates = self.sessions_queryset.dates('date_start', 'day') return dates def get_dates_metrics_date(self): @@ -63,7 +69,7 @@ class DatesLoginMetricMixin: def __set_data_to_cache(self, date, tp, count): cache_key = self.get_cache_key(date, tp) - cache.set(cache_key, count, 3600*24*7) + cache.set(cache_key, count, 3600 * 24 * 7) @staticmethod def get_date_start_2_end(d): @@ -162,40 +168,45 @@ class DatesLoginMetricMixin: def dates_total_count_disabled_assets(self): return Asset.objects.filter(is_active=False).count() - # 以下是从week中而来 - def get_dates_login_times_top5_users(self): - users = self.sessions_queryset.values_list('user_id', flat=True) - users = [ - {'user': user, 'total': total} - for user, total in Counter(users).most_common(5) - ] - return users - def get_dates_total_count_login_users(self): return len(set(self.sessions_queryset.values_list('user_id', flat=True))) def get_dates_total_count_login_times(self): return self.sessions_queryset.count() - def get_dates_login_times_top10_assets(self): + @lazyproperty + def get_type_to_assets(self): + result = Asset.objects.annotate(type=F('platform__type')). \ + values('type').order_by('type').annotate(total=Count(1)) + all_types_dict = dict(AllTypes.choices()) + result = list(result) + for i in result: + tp = i['type'] + i['label'] = all_types_dict[tp] + return result + + def get_dates_login_times_assets(self): assets = self.sessions_queryset.values("asset") \ - .annotate(total=Count("asset")) \ - .annotate(last=Max("date_start")).order_by("-total")[:10] + .annotate(total=Count("asset")) \ + .annotate(last=Max("date_start")).order_by("-total") + assets = assets[:10] for asset in assets: asset['last'] = str(asset['last']) return list(assets) - def get_dates_login_times_top10_users(self): + def get_dates_login_times_users(self): users = self.sessions_queryset.values("user_id") \ - .annotate(total=Count("user_id")) \ - .annotate(user=Max('user')) \ - .annotate(last=Max("date_start")).order_by("-total")[:10] + .annotate(total=Count("user_id")) \ + .annotate(user=Max('user')) \ + .annotate(last=Max("date_start")).order_by("-total") + users = users[:10] for user in users: user['last'] = str(user['last']) return list(users) - def get_dates_login_record_top10_sessions(self): - sessions = self.sessions_queryset.order_by('-date_start')[:10] + def get_dates_login_record_sessions(self): + sessions = self.sessions_queryset.order_by('-date_start') + sessions = sessions[:10] for session in sessions: session.avatar_url = User.get_avatar_url("") sessions = [ @@ -229,11 +240,13 @@ class IndexApi(DatesLoginMetricMixin, APIView): if _all or query_params.get('total_count') or query_params.get('total_count_users'): data.update({ 'total_count_users': caches.users_amount, + 'total_count_users_this_week': caches.new_users_amount_this_week, }) if _all or query_params.get('total_count') or query_params.get('total_count_assets'): data.update({ 'total_count_assets': caches.assets_amount, + 'total_count_assets_this_week': caches.new_assets_amount_this_week, }) if _all or query_params.get('total_count') or query_params.get('total_count_online_users'): @@ -246,6 +259,23 @@ class IndexApi(DatesLoginMetricMixin, APIView): 'total_count_online_sessions': caches.total_count_online_sessions, }) + if _all or query_params.get('total_count') or query_params.get('total_count_today_failed_sessions'): + data.update({ + 'total_count_today_failed_sessions': caches.total_count_today_failed_sessions, + }) + if _all or query_params.get('total_count') or query_params.get('total_count_today_login_users'): + data.update({ + 'total_count_today_login_users': caches.total_count_today_login_users, + }) + if _all or query_params.get('total_count') or query_params.get('total_count_today_active_assets'): + data.update({ + 'total_count_today_active_assets': caches.total_count_today_active_assets, + }) + if _all or query_params.get('total_count') or query_params.get('total_count_type_to_assets_amount'): + data.update({ + 'total_count_type_to_assets_amount': self.get_type_to_assets, + }) + if _all or query_params.get('dates_metrics'): data.update({ 'dates_metrics_date': self.get_dates_metrics_date(), @@ -274,24 +304,19 @@ class IndexApi(DatesLoginMetricMixin, APIView): 'dates_total_count_login_times': self.get_dates_total_count_login_times(), }) - if _all or query_params.get('dates_login_times_top5_users'): - data.update({ - 'dates_login_times_top5_users': self.get_dates_login_times_top5_users(), - }) - if _all or query_params.get('dates_login_times_top10_assets'): data.update({ - 'dates_login_times_top10_assets': self.get_dates_login_times_top10_assets(), + 'dates_login_times_top10_assets': self.get_dates_login_times_assets(), }) if _all or query_params.get('dates_login_times_top10_users'): data.update({ - 'dates_login_times_top10_users': self.get_dates_login_times_top10_users(), + 'dates_login_times_top10_users': self.get_dates_login_times_users(), }) if _all or query_params.get('dates_login_record_top10_sessions'): data.update({ - 'dates_login_record_top10_sessions': self.get_dates_login_record_top10_sessions() + 'dates_login_record_top10_sessions': self.get_dates_login_record_sessions() }) return JsonResponse(data, status=200) @@ -353,4 +378,3 @@ class PrometheusMetricsApi(HealthApiMixin): util = ComponentsPrometheusMetricsUtil() metrics_text = util.get_prometheus_metrics_text() return HttpResponse(metrics_text, content_type='text/plain; version=0.0.4; charset=utf-8') - diff --git a/apps/orgs/caches.py b/apps/orgs/caches.py index a17cd832b..6b73c43b8 100644 --- a/apps/orgs/caches.py +++ b/apps/orgs/caches.py @@ -1,11 +1,14 @@ from django.db.transaction import on_commit + from orgs.models import Organization from orgs.tasks import refresh_org_cache_task from orgs.utils import current_org, tmp_to_org - from common.cache import Cache, IntegerField from common.utils import get_logger +from common.utils.timezone import local_zero_hour, local_monday from users.models import UserGroup, User +from audits.models import UserLoginLog +from audits.const import LoginStatusChoices from assets.models import Node, Domain, Asset, Account from terminal.models import Session from perms.models import AssetPermission @@ -35,30 +38,35 @@ class OrgRelatedCache(Cache): """ 在事务提交之后再发送信号,防止因事务的隔离性导致未获得最新的数据 """ + def func(): logger.debug(f'CACHE: Send refresh task {self}.{fields}') refresh_org_cache_task.delay(self, *fields) + on_commit(func) def expire(self, *fields): def func(): super(OrgRelatedCache, self).expire(*fields) + on_commit(func) class OrgResourceStatisticsCache(OrgRelatedCache): users_amount = IntegerField() - groups_amount = IntegerField(queryset=UserGroup.objects) - assets_amount = IntegerField() + new_users_amount_this_week = IntegerField() + new_assets_amount_this_week = IntegerField() nodes_amount = IntegerField(queryset=Node.objects) - accounts_amount = IntegerField(queryset=Account.objects) domains_amount = IntegerField(queryset=Domain.objects) - # gateways_amount = IntegerField(queryset=Gateway.objects) + groups_amount = IntegerField(queryset=UserGroup.objects) + accounts_amount = IntegerField(queryset=Account.objects) asset_perms_amount = IntegerField(queryset=AssetPermission.objects) - total_count_online_users = IntegerField() total_count_online_sessions = IntegerField() + total_count_today_login_users = IntegerField() + total_count_today_active_assets = IntegerField() + total_count_today_failed_sessions = IntegerField() def __init__(self, org): super().__init__() @@ -70,18 +78,56 @@ class OrgResourceStatisticsCache(OrgRelatedCache): def get_current_org(self): return self.org + def get_users(self): + return User.get_org_users(self.org) + + @staticmethod + def get_assets(): + return Asset.objects.all() + def compute_users_amount(self): - amount = User.get_org_users(self.org).count() - return amount + users = self.get_users() + return users.count() + + def compute_new_users_amount_this_week(self): + monday_time = local_monday() + users = self.get_users().filter(date_joined__gte=monday_time) + return users.count() def compute_assets_amount(self): - if self.org.is_root(): - return Asset.objects.all().count() - node = Node.org_root() - return node.assets_amount + assets = self.get_assets() + return assets.count() - def compute_total_count_online_users(self): - return Session.objects.filter(is_finished=False).values_list('user_id').distinct().count() + def compute_new_assets_amount_this_week(self): + monday_time = local_monday() + assets = self.get_assets().filter(date_created__gte=monday_time) + return assets.count() - def compute_total_count_online_sessions(self): + @staticmethod + def compute_total_count_online_users(): + return Session.objects.filter( + is_finished=False + ).values_list('user_id').distinct().count() + + @staticmethod + def compute_total_count_online_sessions(): return Session.objects.filter(is_finished=False).count() + + @staticmethod + def compute_total_count_today_login_users(): + t = local_zero_hour() + return UserLoginLog.objects.filter( + datetime__gte=t, status=LoginStatusChoices.success + ).values('username').distinct().count() + + @staticmethod + def compute_total_count_today_active_assets(): + t = local_zero_hour() + return Session.objects.filter( + date_start__gte=t, is_success=False + ).values('asset_id').distinct().count() + + @staticmethod + def compute_total_count_today_failed_sessions(): + t = local_zero_hour() + return Session.objects.filter(date_start__gte=t, is_success=False).count() diff --git a/apps/orgs/signal_handlers/cache.py b/apps/orgs/signal_handlers/cache.py index b3e06362d..9d3ce5c85 100644 --- a/apps/orgs/signal_handlers/cache.py +++ b/apps/orgs/signal_handlers/cache.py @@ -2,8 +2,9 @@ from django.db.models.signals import post_save, pre_delete, pre_save, post_delet from django.dispatch import receiver from orgs.models import Organization -from assets.models import Node +from assets.models import Node, Account from perms.models import AssetPermission +from audits.models import UserLoginLog from users.models import UserGroup, User from users.signals import pre_user_leave_org from terminal.models import Session @@ -74,12 +75,15 @@ def on_user_delete_refresh_cache(sender, instance, **kwargs): class OrgResourceStatisticsRefreshUtil: model_cache_field_mapper = { - AssetPermission: ['asset_perms_amount'], - Domain: ['domains_amount'], Node: ['nodes_amount'], - Asset: ['assets_amount'], + Domain: ['domains_amount'], UserGroup: ['groups_amount'], - RoleBinding: ['users_amount'] + Account: ['accounts_amount'], + UserLoginLog: ['total_count_today_login_users'], + RoleBinding: ['users_amount', 'new_users_amount_this_week'], + Asset: ['assets_amount', 'new_assets_amount_this_week'], + AssetPermission: ['asset_perms_amount'], + } @classmethod