From 7c4931b6afa340b0d269724f43d5867eb8225c64 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Thu, 13 Mar 2025 17:52:41 +0800 Subject: [PATCH] perf: execution type --- apps/accounts/api/automations/backup.py | 2 +- .../accounts/api/automations/change_secret.py | 2 +- .../automations/change_secret_dashboard.py | 25 +++++++++-------- .../accounts/api/automations/check_account.py | 2 +- .../api/automations/gather_account.py | 2 +- apps/accounts/api/automations/push_account.py | 2 +- apps/assets/automations/base/manager.py | 11 +++++--- .../0015_automationexecution_type.py | 28 +++++++++++++++++++ apps/assets/models/automations/base.py | 1 + 9 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 apps/assets/migrations/0015_automationexecution_type.py diff --git a/apps/accounts/api/automations/backup.py b/apps/accounts/api/automations/backup.py index cf3c0c184..f53358988 100644 --- a/apps/accounts/api/automations/backup.py +++ b/apps/accounts/api/automations/backup.py @@ -32,5 +32,5 @@ class BackupAccountExecutionViewSet(AutomationExecutionViewSet): def get_queryset(self): queryset = super().get_queryset() - queryset = queryset.filter(automation__type=self.tp) + queryset = queryset.filter(type=self.tp) return queryset diff --git a/apps/accounts/api/automations/change_secret.py b/apps/accounts/api/automations/change_secret.py index db849011e..61a30eb52 100644 --- a/apps/accounts/api/automations/change_secret.py +++ b/apps/accounts/api/automations/change_secret.py @@ -130,7 +130,7 @@ class ChangSecretExecutionViewSet(AutomationExecutionViewSet): def get_queryset(self): queryset = super().get_queryset() - queryset = queryset.filter(automation__type=self.tp) + queryset = queryset.filter(type=self.tp) return queryset diff --git a/apps/accounts/api/automations/change_secret_dashboard.py b/apps/accounts/api/automations/change_secret_dashboard.py index 9c7b9266f..4dc73cfd6 100644 --- a/apps/accounts/api/automations/change_secret_dashboard.py +++ b/apps/accounts/api/automations/change_secret_dashboard.py @@ -9,6 +9,7 @@ from rest_framework.views import APIView from accounts.const import AutomationTypes, ChangeSecretRecordStatusChoice from accounts.models import ChangeSecretAutomation, AutomationExecution, ChangeSecretRecord from assets.models import Node, Asset +from common.const import Status from common.utils import lazyproperty from common.utils.timezone import local_zero_hour, local_now from ops.celery import app @@ -97,17 +98,19 @@ class ChangeSecretDashboardApi(APIView): return qs.count() return self.get_queryset_date_filter(qs, field).count() - @staticmethod - def get_status_counts(records): - pending = ChangeSecretRecordStatusChoice.pending - failed = ChangeSecretRecordStatusChoice.failed - total_ids = {str(i) for i in records.exclude(status=pending).values('execution_id').distinct()} - failed_ids = {str(i) for i in records.filter(status=failed).values('execution_id').distinct()} - total = len(total_ids) - failed = len(total_ids & failed_ids) + def get_status_counts(self, executions): + executions = executions.filter(type=self.tp) + total, failed, success = 0, 0, 0 + for status in executions.values_list('status', flat=True): + total += 1 + if status in [Status.failed, Status.error]: + failed += 1 + elif status == Status.success: + success += 1 + return { 'total_count_change_secret_executions': total, - 'total_count_success_change_secret_executions': total - failed, + 'total_count_success_change_secret_executions': success, 'total_count_failed_change_secret_executions': failed, } @@ -131,8 +134,8 @@ class ChangeSecretDashboardApi(APIView): data['total_count_change_secret_assets'] = self.get_change_secret_asset_queryset().count() if _all or query_params.get('total_count_change_secret_status'): - records = self.get_queryset_date_filter(self.change_secret_records_queryset, 'date_finished') - data.update(self.get_status_counts(records)) + executions = self.get_queryset_date_filter(AutomationExecution.objects.all(), 'date_start') + data.update(self.get_status_counts(executions)) if _all or query_params.get('daily_success_and_failure_metrics'): success, failed = self.get_daily_success_and_failure_metrics() diff --git a/apps/accounts/api/automations/check_account.py b/apps/accounts/api/automations/check_account.py index 64ff46054..c47c456d4 100644 --- a/apps/accounts/api/automations/check_account.py +++ b/apps/accounts/api/automations/check_account.py @@ -55,7 +55,7 @@ class CheckAccountExecutionViewSet(AutomationExecutionViewSet): def get_queryset(self): queryset = super().get_queryset() - queryset = queryset.filter(automation__type=self.tp) + queryset = queryset.filter(type=self.tp) return queryset @action(methods=["get"], detail=False, url_path="adhoc") diff --git a/apps/accounts/api/automations/gather_account.py b/apps/accounts/api/automations/gather_account.py index a4dbb0931..732aa01f8 100644 --- a/apps/accounts/api/automations/gather_account.py +++ b/apps/accounts/api/automations/gather_account.py @@ -46,7 +46,7 @@ class DiscoverAccountsExecutionViewSet(AutomationExecutionViewSet): def get_queryset(self): queryset = super().get_queryset() - queryset = queryset.filter(automation__type=self.tp) + queryset = queryset.filter(type=self.tp) return queryset @xframe_options_sameorigin diff --git a/apps/accounts/api/automations/push_account.py b/apps/accounts/api/automations/push_account.py index 925e21e06..643f58d61 100644 --- a/apps/accounts/api/automations/push_account.py +++ b/apps/accounts/api/automations/push_account.py @@ -38,7 +38,7 @@ class PushAccountExecutionViewSet(AutomationExecutionViewSet): def get_queryset(self): queryset = super().get_queryset() - queryset = queryset.filter(automation__type=self.tp) + queryset = queryset.filter(type=self.tp) return queryset diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 4f2948288..87a5fe7d9 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -3,10 +3,10 @@ import json import logging import os import shutil +import time from collections import defaultdict from socket import gethostname -import time import yaml from django.conf import settings from django.template.loader import render_to_string @@ -175,7 +175,7 @@ class BaseManager: self.do_run(*args, **kwargs) except Exception as e: logging.exception(e) - self.status = 'error' + self.status = Status.error finally: self.post_run() @@ -485,8 +485,11 @@ class BasePlaybookManager(PlaybookPrepareMixin, BaseManager): self.on_host_error(host, error, detail) def post_run(self): - if self.summary['fail_assets']: - self.status = 'failed' + if any( + self.summary.get(key, 0) > 0 + for key in ("fail_assets", "fail_accounts") + ): + self.status = Status.failed super().post_run() def on_runner_success(self, runner, cb): diff --git a/apps/assets/migrations/0015_automationexecution_type.py b/apps/assets/migrations/0015_automationexecution_type.py new file mode 100644 index 000000000..c48dac755 --- /dev/null +++ b/apps/assets/migrations/0015_automationexecution_type.py @@ -0,0 +1,28 @@ +# Generated by Django 4.1.13 on 2025-03-13 09:14 + +from django.db import migrations, models + + +def migrate_execution_type(apps, schema_editor): + execution_model = apps.get_model('assets', 'AutomationExecution') + execution_objs = [] + for execution in execution_model.objects.all(): + snapshot = execution.snapshot + execution.type = snapshot.get('type', '') + execution_objs.append(execution) + execution_model.objects.bulk_update(execution_objs, ['type']) + + +class Migration(migrations.Migration): + dependencies = [ + ('assets', '0014_alter_automationexecution_duration'), + ] + + operations = [ + migrations.AddField( + model_name='automationexecution', + name='type', + field=models.CharField(default='', max_length=16, verbose_name='Type'), + ), + migrations.RunPython(migrate_execution_type) + ] diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index 07a167773..a95cd63bf 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -142,6 +142,7 @@ class AutomationExecution(OrgModelMixin): null=True, verbose_name=_("Date start"), db_index=True ) date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished")) + type = models.CharField(default='', max_length=16, verbose_name=_("Type")) duration = models.DecimalField(default=0, max_digits=10, decimal_places=2, verbose_name=_("Duration")) snapshot = EncryptJsonDictTextField( default=dict, blank=True, null=True, verbose_name=_("Automation snapshot")