From ab5b85d9b57198d8d622aafac6bf790fac44f631 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Tue, 17 Jan 2023 12:43:07 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97=EF=BC=8Cactivity=E6=97=A5=E5=BF=97=E9=83=BD?= =?UTF-8?q?=E5=AD=98=E5=85=A5=E6=93=8D=E4=BD=9C=E6=97=A5=E5=BF=97=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/audits/api.py | 2 +- apps/audits/backends/db.py | 70 ++++++++++++++----- apps/audits/handler.py | 42 +++++------ .../migrations/0020_auto_20230112_1034.py | 18 ----- .../migrations/0020_auto_20230117_1004.py | 50 +++++++++++++ apps/audits/models.py | 8 ++- apps/audits/serializers.py | 14 +++- apps/audits/signal_handlers.py | 16 ++--- 8 files changed, 148 insertions(+), 72 deletions(-) delete mode 100644 apps/audits/migrations/0020_auto_20230112_1034.py create mode 100644 apps/audits/migrations/0020_auto_20230117_1004.py diff --git a/apps/audits/api.py b/apps/audits/api.py index 383b0d45b..a8c178ac2 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -106,7 +106,7 @@ class OperateLogViewSet(RetrieveModelMixin, ListModelMixin, OrgGenericViewSet): return super().get_serializer_class() def get_queryset(self): - qs = OperateLog.objects.filter(is_activity=False) + qs = OperateLog.objects.all() es_config = settings.OPERATE_LOG_ELASTICSEARCH_CONFIG if es_config: engine_mod = import_module(TYPE_ENGINE_MAPPING['es']) diff --git a/apps/audits/backends/db.py b/apps/audits/backends/db.py index a3d941f8e..b24efaaa4 100644 --- a/apps/audits/backends/db.py +++ b/apps/audits/backends/db.py @@ -5,9 +5,12 @@ from audits.models import OperateLog class OperateLogStore(object): + # 用不可见字符分割前后数据，节省存储-> diff: {'key': 'before\0after'} + SEP = '\0' + def __init__(self, config): self.model = OperateLog - self.max_length = 1024 + self.max_length = 2048 self.max_length_tip_msg = _( 'The text content is too long. Use Elasticsearch to store operation logs' ) @@ -16,27 +19,62 @@ class OperateLogStore(object): def ping(timeout=None): return True + @classmethod + def convert_before_after_to_diff(cls, before, after): + if not isinstance(before, dict): + before = dict() + if not isinstance(after, dict): + after = dict() + + diff = dict() + keys = set(before.keys()) | set(after.keys()) + for k in keys: + before_value = before.get(k, '') + after_value = after.get(k, '') + diff[k] = '%s%s%s' % (before_value, cls.SEP, after_value) + return diff + + @classmethod + def convert_diff_to_before_after(cls, diff): + before, after = dict(), dict() + if not diff: + return before, after + + for k, v in diff.items(): + before_value, after_value = v.split(cls.SEP, 1) + before[k], after[k] = before_value, after_value + return before, after + + @classmethod + def convert_diff_friendly(cls, raw_diff): + diff_list = list() + for k, v in raw_diff.items(): + before, after = v.split(cls.SEP, 1) + diff_list.append({ + 'field': k, + 'before': before if before else _('empty'), + 'after': after if after else _('empty'), + }) + return diff_list + def save(self, **kwargs): - before_limit, after_limit = None, None log_id = kwargs.get('id', '') - before = kwargs.get('before') or {} - after = kwargs.get('after') or {} - if len(str(before)) > self.max_length: - before_limit = {str(_('Tips')): self.max_length_tip_msg} - if len(str(after)) > self.max_length: - after_limit = {str(_('Tips')): self.max_length_tip_msg} + before = kwargs.pop('before') or {} + after = kwargs.pop('after') or {} op_log = self.model.objects.filter(pk=log_id).first() if op_log is not None: - op_log_before = op_log.before or {} - op_log_after = op_log.after or {} - if not before_limit: - before.update(op_log_before) - if not after_limit: - after.update(op_log_after) + op_log_diff = op_log.diff or {} + op_before, op_after = self.convert_diff_to_before_after(op_log_diff) + before.update(op_before) + after.update(op_after) else: op_log = self.model(**kwargs) - op_log.before = before_limit if before_limit else before - op_log.after = after_limit if after_limit else after + diff = self.convert_before_after_to_diff(before, after) + if len(str(diff)) > self.max_length: + limit = {str(_('Tips')): self.max_length_tip_msg} + diff = self.convert_before_after_to_diff(limit, limit) + + op_log.diff = diff op_log.save() diff --git a/apps/audits/handler.py b/apps/audits/handler.py index 13ea6f16a..7aff4c427 100644 --- a/apps/audits/handler.py +++ b/apps/audits/handler.py @@ -11,7 +11,6 @@ from common.utils.encode import Singleton from common.local import encrypted_field_set from settings.serializers import SettingsSerializer from jumpserver.utils import current_request -from audits.models import OperateLog from orgs.utils import get_current_org_id from .backends import get_operate_log_storage @@ -21,25 +20,6 @@ from .const import ActionChoices logger = get_logger(__name__) -class ModelClient: - @staticmethod - def save(**kwargs): - log_id = kwargs.get('id', '') - op_log = OperateLog.objects.filter(pk=log_id).first() - if op_log is not None: - raw_after = op_log.after or {} - raw_before = op_log.before or {} - cur_before = kwargs.get('before') or {} - cur_after = kwargs.get('after') or {} - raw_before.update(cur_before) - raw_after.update(cur_after) - op_log.before = raw_before - op_log.after = raw_after - op_log.save() - else: - OperateLog.objects.create(**kwargs) - - class OperatorLogHandler(metaclass=Singleton): CACHE_KEY = 'OPERATOR_LOG_CACHE_KEY' @@ -156,28 +136,42 @@ class OperatorLogHandler(metaclass=Singleton): # 否则会话结束，录像文件结束操作的会话记录都会体现出来 params = {} action = kwargs.get('data', {}).get('action', 'create') + detail = _( + '{} used account[{}], login method[{}] login the asset.' + ).format( + resource.user, resource.account, resource.login_from_display + ) if action == ActionChoices.create: params = { 'action': ActionChoices.connect, 'resource_id': str(resource.asset_id), - 'user': resource.user + 'user': resource.user, 'detail': detail } return params @staticmethod def _get_ChangeSecretRecord_params(resource, **kwargs): + detail = _( + 'User {} has executed change auth plan for this account.({})' + ).format( + resource.created_by, _(resource.status.title()) + ) return { - 'action': ActionChoices.change_auth, + 'action': ActionChoices.change_auth, 'detail': detail, 'resource_id': str(resource.account_id), } @staticmethod def _get_UserLoginLog_params(resource, **kwargs): username = resource.username + login_status = _('Success') if resource.status else _('Failed') + detail = _('User {} login into this service.[{}]').format( + resource.username, login_status + ) user_id = User.objects.filter(username=username).\ values_list('id', flat=True)[0] return { - 'action': ActionChoices.login, + 'action': ActionChoices.login, 'detail': detail, 'resource_id': str(user_id), } @@ -185,7 +179,6 @@ class OperatorLogHandler(metaclass=Singleton): param_func = getattr(self, '_get_%s_params' % object_name, None) if param_func is not None: params = param_func(resource, data=data) - data['is_activity'] = True data.update(params) return data @@ -228,6 +221,7 @@ class OperatorLogHandler(metaclass=Singleton): op_handler = OperatorLogHandler() +# 理论上操作日志的唯一入口 create_or_update_operate_log = op_handler.create_or_update_operate_log cache_instance_before_data = op_handler.cache_instance_before_data get_instance_current_with_cache_diff = op_handler.get_instance_current_with_cache_diff diff --git a/apps/audits/migrations/0020_auto_20230112_1034.py b/apps/audits/migrations/0020_auto_20230112_1034.py deleted file mode 100644 index d3dd634fa..000000000 --- a/apps/audits/migrations/0020_auto_20230112_1034.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.2.14 on 2023-01-12 02:34 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('audits', '0019_alter_operatelog_options'), - ] - - operations = [ - migrations.AddField( - model_name='operatelog', - name='is_activity', - field=models.BooleanField(default=False, verbose_name='Is Activity'), - ), - ] diff --git a/apps/audits/migrations/0020_auto_20230117_1004.py b/apps/audits/migrations/0020_auto_20230117_1004.py new file mode 100644 index 000000000..d3c2c4c70 --- /dev/null +++ b/apps/audits/migrations/0020_auto_20230117_1004.py @@ -0,0 +1,50 @@ +# Generated by Django 3.2.14 on 2023-01-17 02:04 + +import common.db.encoder +from django.db import migrations, models + +from audits.backends.db import OperateLogStore + + +def migrate_operate_log_after_before(apps, schema_editor): + operate_log_model = apps.get_model("audits", "OperateLog") + db_alias = schema_editor.connection.alias + count, batch_size = 0, 1000 + + while True: + operate_logs = [] + queryset = operate_log_model.objects.using(db_alias).all()[count:count + batch_size] + if not queryset: + break + count += len(queryset) + for inst in queryset: + before, after, diff = inst.before, inst.after, dict() + if not any([before, after]): + continue + diff = OperateLogStore.convert_before_after_to_diff(before, after) + inst.diff = diff + operate_logs.append(inst) + operate_log_model.objects.bulk_update(operate_logs, ['diff']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('audits', '0019_alter_operatelog_options'), + ] + + operations = [ + migrations.AddField( + model_name='operatelog', + name='diff', + field=models.JSONField(default=dict, encoder=common.db.encoder.ModelJSONFieldEncoder, null=True), + ), + migrations.AddField( + model_name='operatelog', + name='detail', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Detail'), + ), + migrations.RunPython(migrate_operate_log_after_before), + migrations.RemoveField(model_name='operatelog', name='after', ), + migrations.RemoveField(model_name='operatelog', name='before', ), + ] diff --git a/apps/audits/models.py b/apps/audits/models.py index 370a9513f..86c7597a7 100644 --- a/apps/audits/models.py +++ b/apps/audits/models.py @@ -58,9 +58,8 @@ class OperateLog(OrgModelMixin): ) remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True) datetime = models.DateTimeField(auto_now=True, verbose_name=_('Datetime'), db_index=True) - before = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True) - after = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True) - is_activity = models.BooleanField(default=False, verbose_name=(_('Is Activity'))) + diff = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True) + detail = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Detail')) def __str__(self): return "<{}> {} <{}>".format(self.user, self.action, self.resource) @@ -139,6 +138,9 @@ class UserLoginLog(models.Model): max_length=32, default="", verbose_name=_("Authentication backend") ) + def __str__(self): + return '%s(%s)' % (self.username, self.city) + @property def backend_display(self): return gettext(self.backend) diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index 8fd8a10dc..6700c3b68 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -3,6 +3,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers +from audits.backends.db import OperateLogStore from common.serializers.fields import LabeledChoiceField from common.utils.timezone import as_current_tz from ops.models.job import JobAuditLog @@ -68,7 +69,13 @@ class UserLoginLogSerializer(serializers.ModelSerializer): class OperateLogActionDetailSerializer(serializers.ModelSerializer): class Meta: model = models.OperateLog - fields = ('before', 'after') + fields = ('diff',) + + def to_representation(self, instance): + data = super().to_representation(instance) + diff = OperateLogStore.convert_diff_friendly(data['diff']) + data['diff'] = diff + return data class OperateLogSerializer(serializers.ModelSerializer): @@ -109,5 +116,8 @@ class ActivitiesOperatorLogSerializer(serializers.Serializer): @staticmethod def get_content(obj): action = obj.action.replace('_', ' ').capitalize() - ctn = _('User {} {} this resource.').format(obj.user, _(action)) + if not obj.detail: + ctn = _('User {} {} this resource.').format(obj.user, _(action)) + else: + ctn = obj.detail return ctn diff --git a/apps/audits/signal_handlers.py b/apps/audits/signal_handlers.py index a7649bbe5..8cc396496 100644 --- a/apps/audits/signal_handlers.py +++ b/apps/audits/signal_handlers.py @@ -285,11 +285,11 @@ def on_user_auth_failed(sender, username, request, reason='', **kwargs): @receiver(django_ready) def on_django_start_set_operate_log_monitor_models(sender, **kwargs): - exclude_label = { + exclude_apps = { 'django_cas_ng', 'captcha', 'admin', 'jms_oidc_rp', 'django_celery_beat', 'contenttypes', 'sessions', 'auth' } - exclude_object_name = { + exclude_models = { 'UserPasswordHistory', 'ContentType', 'SiteMessage', 'SiteMessageUsers', 'PlatformAutomation', 'PlatformProtocol', 'Protocol', @@ -305,10 +305,10 @@ def on_django_start_set_operate_log_monitor_models(sender, **kwargs): 'FTPLog', 'OperateLog', 'PasswordChangeLog' } for i, app in enumerate(apps.get_models(), 1): - app_label = app._meta.app_label - app_object_name = app._meta.object_name - if app_label in exclude_label or \ - app_object_name in exclude_object_name or \ - app_object_name.endswith('Execution'): + app_name = app._meta.app_label + model_name = app._meta.object_name + if app_name in exclude_apps or \ + model_name in exclude_models or \ + model_name.endswith('Execution'): continue - MODELS_NEED_RECORD.add(app_object_name) + MODELS_NEED_RECORD.add(model_name)