From bd9b1f1a5b2bfe41b4bce3280f07ae53981af4ba Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 13 Feb 2023 19:42:42 +0800 Subject: [PATCH 1/3] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20audit=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/const/database.py | 2 +- apps/assets/const/protocol.py | 3 + apps/assets/serializers/platform.py | 4 ++ apps/audits/signal_handlers/activity_log.py | 26 ++++----- apps/locale/ja/LC_MESSAGES/django.po | 2 +- apps/locale/zh/LC_MESSAGES/django.po | 2 +- apps/orgs/signal_handlers/common.py | 62 +++++++++++++-------- 7 files changed, 62 insertions(+), 39 deletions(-) diff --git a/apps/assets/const/database.py b/apps/assets/const/database.py index 623b5b291..875cd64a0 100644 --- a/apps/assets/const/database.py +++ b/apps/assets/const/database.py @@ -63,7 +63,7 @@ class DatabaseTypes(BaseType): cls.SQLSERVER: [{'name': 'SQLServer'}], cls.CLICKHOUSE: [{'name': 'ClickHouse'}], cls.MONGODB: [{'name': 'MongoDB'}], - cls.REDIS: [{'name': 'Redis'}], + cls.REDIS: [{'name': 'Redis'}, {'name': 'Redis6+'}], } @classmethod diff --git a/apps/assets/const/protocol.py b/apps/assets/const/protocol.py index a5f1d413a..b8efc654f 100644 --- a/apps/assets/const/protocol.py +++ b/apps/assets/const/protocol.py @@ -96,6 +96,9 @@ class Protocol(ChoicesMixin, models.TextChoices): 'port': 6379, 'required': True, 'secret_types': ['password'], + 'setting': { + 'auth_username': True, + } }, } diff --git a/apps/assets/serializers/platform.py b/apps/assets/serializers/platform.py index b3bc86e4d..15136930c 100644 --- a/apps/assets/serializers/platform.py +++ b/apps/assets/serializers/platform.py @@ -38,6 +38,10 @@ class ProtocolSettingSerializer(serializers.Serializer): ) script = serializers.JSONField(default=list, label=_("Script")) + # Redis + auth_username = serializers.BooleanField(default=False, label=_("Auth with username"), + help_text=_("Auth with username")) + class PlatformAutomationSerializer(serializers.ModelSerializer): class Meta: diff --git a/apps/audits/signal_handlers/activity_log.py b/apps/audits/signal_handlers/activity_log.py index ce1b99fe2..6766d5d53 100644 --- a/apps/audits/signal_handlers/activity_log.py +++ b/apps/audits/signal_handlers/activity_log.py @@ -4,18 +4,18 @@ from celery import signals from django.db.models.signals import post_save from django.utils.translation import ugettext_lazy as _ -from audits.models import ActivityLog -from assets.models import Asset, Node from accounts.const import AutomationTypes from accounts.models import AccountBackupAutomation +from assets.models import Asset, Node +from audits.models import ActivityLog from common.utils import get_object_or_none +from jumpserver.utils import current_request from ops.celery import app from orgs.utils import tmp_to_root_org from terminal.models import Session from users.models import User -from jumpserver.utils import current_request - from ..const import ActivityChoices +from ..models import UserLoginLog class ActivityLogHandler(object): @@ -118,8 +118,8 @@ class ActivityLogHandler(object): @staticmethod def login_log_for_activity(obj): login_status = _('Success') if obj.status else _('Failed') - detail = _('User {} login into this service.[{}]').format( - obj.username, login_status + detail = _('User {} login this system {}').format( + obj.name, login_status ) user_id = User.objects.filter(username=obj.username).values('id').first() return user_id['id'], detail, ActivityChoices.login_log @@ -164,12 +164,11 @@ def on_celery_task_pre_run_for_activity_log(task_id='', **kwargs): activities.append( ActivityLog(id=activity_id, detail_id=task_id) ) - ActivityLog.objects.bulk_update(activities, ('detail_id', )) + ActivityLog.objects.bulk_update(activities, ('detail_id',)) -@post_save.connect -def on_object_created( - sender, instance=None, created=False, update_fields=None, **kwargs +def on_session_or_login_log_created( + sender, instance=None, created=False, **kwargs ): handler_mapping = { 'Session': activity_handler.session_for_activity, @@ -179,12 +178,13 @@ def on_object_created( if not created or model_name not in handler_mapping: return - resource_id, detail, a_type = handler_mapping[model_name](instance) + resource_id, detail, act_type = handler_mapping[model_name](instance) ActivityLog.objects.create( - resource_id=resource_id, type=a_type, + resource_id=resource_id, type=act_type, detail=detail, detail_id=instance.id ) - +for sender in [Session, UserLoginLog]: + post_save.connect(on_session_or_login_log_created, sender=sender) diff --git a/apps/locale/ja/LC_MESSAGES/django.po b/apps/locale/ja/LC_MESSAGES/django.po index 4819919d4..5e0b9d49c 100644 --- a/apps/locale/ja/LC_MESSAGES/django.po +++ b/apps/locale/ja/LC_MESSAGES/django.po @@ -1765,7 +1765,7 @@ msgid "User {} has executed change auth plan for this account.({})" msgstr "ユーザー {} はこのアカウントのために改密計画を実行しました。({})" #: audits/handler.py:168 -msgid "User {} login into this service.[{}]" +msgid "User [{}] login this system.[{}]" msgstr "ユーザー {} がサービスにログインしました。[{}]" #: audits/models.py:32 audits/models.py:59 audits/models.py:101 diff --git a/apps/locale/zh/LC_MESSAGES/django.po b/apps/locale/zh/LC_MESSAGES/django.po index e84e4b3ec..56becba06 100644 --- a/apps/locale/zh/LC_MESSAGES/django.po +++ b/apps/locale/zh/LC_MESSAGES/django.po @@ -1752,7 +1752,7 @@ msgid "User {} has executed change auth plan for this account.({})" msgstr "用户 {} 为这个账号执行了改密计划.({})" #: audits/handler.py:168 -msgid "User {} login into this service.[{}]" +msgid "User [{}] login this system.[{}]" msgstr "用户 {} 登录了服务.[{}]" #: audits/models.py:32 audits/models.py:59 audits/models.py:101 diff --git a/apps/orgs/signal_handlers/common.py b/apps/orgs/signal_handlers/common.py index dd4168c5c..f6d5c4132 100644 --- a/apps/orgs/signal_handlers/common.py +++ b/apps/orgs/signal_handlers/common.py @@ -4,12 +4,13 @@ from collections import defaultdict from functools import partial from django.conf import settings -from django.db.models.signals import post_save, pre_delete, m2m_changed +from django.db.models.signals import post_save, pre_delete, m2m_changed, post_delete from django.db.utils import ProgrammingError, OperationalError from django.dispatch import receiver from django.utils.functional import LazyObject from common.const.signals import PRE_REMOVE, POST_REMOVE +from common.decorators import delay_run from common.decorators import on_transaction_commit from common.signals import django_ready from common.utils import get_logger @@ -51,9 +52,21 @@ def subscribe_orgs_mapping_expire(sender, **kwargs): ) +@delay_run(ttl=5) +def expire_user_orgs(*args): + User.expire_users_rbac_perms_cache() + + +@receiver(post_save, sender=Organization) +def on_org_create(sender, instance, created=False, **kwargs): + if created: + return + expire_user_orgs() + + # 创建对应的root @receiver(post_save, sender=Organization) -def on_org_create_or_update(sender, instance, created=False, **kwargs): +def on_org_create_or_update(sender, instance, **kwargs): # 必须放到最开始, 因为下面调用Node.save方法时会获取当前组织的org_id(即instance.org_id), 如果不过期会找不到 expire_orgs_mapping_for_memory(instance.id) @@ -77,7 +90,26 @@ def on_org_delete(sender, instance, **kwargs): root_node.delete() -def _remove_users(model, users, org, user_field_name='users'): +@receiver(post_delete, sender=Organization) +def on_org_delete(sender, instance, **kwargs): + expire_user_orgs() + + +@receiver(post_save, sender=User) +@on_transaction_commit +def on_user_created_set_default_org(sender, instance, created, **kwargs): + if not instance.id: + # 用户已被手动删除，instance.orgs 时会使用 id 进行查找报错，所以判断不存在id时不做处理 + return + if not created: + return + if instance.orgs.count() > 0: + return + with tmp_to_org(Organization.default()): + Organization.default().add_member(instance) + + +def _remove_user_resource(model, users, org, user_field_name='users'): with tmp_to_org(org): if not isinstance(users, (tuple, list, set)): users = (users,) @@ -103,8 +135,8 @@ def _remove_users(model, users, org, user_field_name='users'): objects = model.objects.filter(id__in=object_id_user_ids_map.keys()) send_m2m_change_signal = partial( - m2m_changed.send, - sender=m2m_model, reverse=reverse, model=User, using=model.objects.db + m2m_changed.send, sender=m2m_model, reverse=reverse, + model=User, using=model.objects.db ) for obj in objects: @@ -126,7 +158,7 @@ def _remove_users(model, users, org, user_field_name='users'): def _clear_users_from_org(org, users): """ - 清理用户在该组织下的相关数据 + 清理用户在该组织下的相关数据, 包括用户组, 资产授权, 用户授权 """ if not users: return @@ -134,23 +166,7 @@ def _clear_users_from_org(org, users): models = (AssetPermission, UserGroup) for m in models: - _remove_users(m, users, org) - - # _remove_users(CommandFilterRule, users, org, user_field_name='reviewers') - - -@receiver(post_save, sender=User) -@on_transaction_commit -def on_user_created_set_default_org(sender, instance, created, **kwargs): - if not instance.id: - # 用户已被手动删除，instance.orgs 时会使用 id 进行查找报错，所以判断不存在id时不做处理 - return - if not created: - return - if instance.orgs.count() > 0: - return - with tmp_to_org(Organization.default()): - Organization.default().add_member(instance) + _remove_user_resource(m, users, org) @receiver(post_user_leave_org) From b3244d0215b6f6aa2dad1636ea05aefd298a8181 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 14 Feb 2023 13:44:14 +0800 Subject: [PATCH 2/3] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20redis=20?= =?UTF-8?q?=E5=B9=B3=E5=8F=B0=E5=8D=8F=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/const/database.py | 15 ++++++++++++++- apps/assets/const/types.py | 2 +- apps/audits/signal_handlers/activity_log.py | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/apps/assets/const/database.py b/apps/assets/const/database.py index 875cd64a0..7bf1aaf44 100644 --- a/apps/assets/const/database.py +++ b/apps/assets/const/database.py @@ -63,7 +63,20 @@ class DatabaseTypes(BaseType): cls.SQLSERVER: [{'name': 'SQLServer'}], cls.CLICKHOUSE: [{'name': 'ClickHouse'}], cls.MONGODB: [{'name': 'MongoDB'}], - cls.REDIS: [{'name': 'Redis'}, {'name': 'Redis6+'}], + cls.REDIS: [ + { + 'name': 'Redis', + 'protocols_setting': { + 'redis': {'auth_username': False} + } + }, + { + 'name': 'Redis6+', + 'protocols_setting': { + 'redis': {'auth_username': True} + } + } + ] } @classmethod diff --git a/apps/assets/const/types.py b/apps/assets/const/types.py index c42695840..509c2f58e 100644 --- a/apps/assets/const/types.py +++ b/apps/assets/const/types.py @@ -304,7 +304,7 @@ class AllTypes(ChoicesMixin): setting = _protocols_setting.get(p['name'], {}) p['required'] = p.pop('required', False) p['default'] = p.pop('default', False) - p['setting'] = {**setting, **p.get('setting', {})} + p['setting'] = {**p.get('setting', {}), **setting} platform_data = { **default_platform_data, **d, diff --git a/apps/audits/signal_handlers/activity_log.py b/apps/audits/signal_handlers/activity_log.py index 6766d5d53..8254dce93 100644 --- a/apps/audits/signal_handlers/activity_log.py +++ b/apps/audits/signal_handlers/activity_log.py @@ -119,7 +119,7 @@ class ActivityLogHandler(object): def login_log_for_activity(obj): login_status = _('Success') if obj.status else _('Failed') detail = _('User {} login this system {}').format( - obj.name, login_status + obj.username, login_status ) user_id = User.objects.filter(username=obj.username).values('id').first() return user_id['id'], detail, ActivityChoices.login_log From 8b0a9673068a2ed0fd2238e658fd7adc77cbab76 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 14 Feb 2023 15:53:08 +0800 Subject: [PATCH 3/3] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20audit=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/asset/host.py | 5 ++--- apps/assets/serializers/platform.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/assets/api/asset/host.py b/apps/assets/api/asset/host.py index 3e11b8ad5..d923b2b84 100644 --- a/apps/assets/api/asset/host.py +++ b/apps/assets/api/asset/host.py @@ -1,8 +1,8 @@ -from assets.models import Host, Asset -from assets.serializers import HostSerializer, HostInfoSerializer from rest_framework.decorators import action from rest_framework.response import Response +from assets.models import Host, Asset +from assets.serializers import HostSerializer, HostInfoSerializer from .asset import AssetViewSet __all__ = ['HostViewSet'] @@ -22,4 +22,3 @@ class HostViewSet(AssetViewSet): def info(self, *args, **kwargs): asset = super().get_object() return Response(asset.info) - diff --git a/apps/assets/serializers/platform.py b/apps/assets/serializers/platform.py index 15136930c..0bbb0aa51 100644 --- a/apps/assets/serializers/platform.py +++ b/apps/assets/serializers/platform.py @@ -39,8 +39,7 @@ class ProtocolSettingSerializer(serializers.Serializer): script = serializers.JSONField(default=list, label=_("Script")) # Redis - auth_username = serializers.BooleanField(default=False, label=_("Auth with username"), - help_text=_("Auth with username")) + auth_username = serializers.BooleanField(default=False, label=_("Auth with username")) class PlatformAutomationSerializer(serializers.ModelSerializer):