diff --git a/apps/accounts/api/automations/backup.py b/apps/accounts/api/automations/backup.py index 74ddfaf76..cf3c0c184 100644 --- a/apps/accounts/api/automations/backup.py +++ b/apps/accounts/api/automations/backup.py @@ -9,11 +9,11 @@ from orgs.mixins.api import OrgBulkModelViewSet from .base import AutomationExecutionViewSet __all__ = [ - 'AccountBackupPlanViewSet', 'BackupAccountExecutionViewSet' + 'BackupAccountViewSet', 'BackupAccountExecutionViewSet' ] -class AccountBackupPlanViewSet(OrgBulkModelViewSet): +class BackupAccountViewSet(OrgBulkModelViewSet): model = BackupAccountAutomation filterset_fields = ('name',) search_fields = filterset_fields @@ -21,8 +21,13 @@ class AccountBackupPlanViewSet(OrgBulkModelViewSet): class BackupAccountExecutionViewSet(AutomationExecutionViewSet): - serializer_class = serializers.BackupAccountExecutionSerializer - http_method_names = ['get', 'post', 'options'] + rbac_perms = ( + ("list", "accounts.view_backupaccountexecution"), + ("retrieve", "accounts.view_backupaccountexecution"), + ("create", "accounts.add_backupaccountexecution"), + ("report", "accounts.view_backupaccountexecution"), + ) + tp = AutomationTypes.backup_account def get_queryset(self): diff --git a/apps/accounts/automations/backup_account/handlers.py b/apps/accounts/automations/backup_account/handlers.py index f8422ea42..b10cc0fc4 100644 --- a/apps/accounts/automations/backup_account/handlers.py +++ b/apps/accounts/automations/backup_account/handlers.py @@ -3,15 +3,17 @@ import time from collections import defaultdict, OrderedDict from django.conf import settings +from django.db.models import F from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from xlsxwriter import Workbook from accounts.const import AccountBackupType -from accounts.models.automations.backup_account import BackupAccountAutomation +from accounts.models import BackupAccountAutomation, Account from accounts.notifications import AccountBackupExecutionTaskMsg, AccountBackupByObjStorageExecutionTaskMsg from accounts.serializers import AccountSecretSerializer from assets.const import AllTypes +from common.const import Status from common.utils.file import encrypt_and_compress_zip_file, zip_files from common.utils.timezone import local_now_filename, local_now_display from terminal.models.component.storage import ReplayStorage @@ -74,9 +76,9 @@ class BaseAccountHandler: class AssetAccountHandler(BaseAccountHandler): @staticmethod - def get_filename(plan_name): + def get_filename(name): filename = os.path.join( - PATH, f'{plan_name}-{local_now_filename()}-{time.time()}.xlsx' + PATH, f'{name}-{local_now_filename()}-{time.time()}.xlsx' ) return filename @@ -118,32 +120,41 @@ class AssetAccountHandler(BaseAccountHandler): cls.handler_secret(data, section) data_map.update(cls.add_rows(data, header_fields, sheet_name)) number_of_backup_accounts = _('Number of backup accounts') - print('\n\033[33m- {}: {}\033[0m'.format(number_of_backup_accounts, accounts.count())) + print('\033[33m- {}: {}\033[0m'.format(number_of_backup_accounts, accounts.count())) return data_map class AccountBackupHandler: - def __init__(self, execution): + def __init__(self, manager, execution): + self.manager = manager self.execution = execution - self.plan_name = self.execution.plan.name - self.is_frozen = False # 任务状态冻结标志 + self.name = self.execution.snapshot.get('name', '-') + + def get_accounts(self): + # TODO 可以优化一下查询 在账号上做 category 的缓存 避免数据量大时连表操作 + types = self.execution.snapshot.get('types', []) + self.manager.summary['total_types'] = len(types) + qs = Account.objects.filter( + asset__platform__type__in=types + ).annotate(type=F('asset__platform__type')) + return qs def create_excel(self, section='complete'): - hint = _('Generating asset or application related backup information files') + hint = _('Generating asset related backup information files') print( - '\n' f'\033[32m>>> {hint}\033[0m' '' ) - # Print task start date + time_start = time.time() files = [] - accounts = self.execution.backup_accounts + accounts = self.get_accounts() + self.manager.summary['total_accounts'] = accounts.count() data_map = AssetAccountHandler.create_data_map(accounts, section) if not data_map: return files - filename = AssetAccountHandler.get_filename(self.plan_name) + filename = AssetAccountHandler.get_filename(self.name) wb = Workbook(filename) for sheet, data in data_map.items(): @@ -164,19 +175,18 @@ class AccountBackupHandler: return recipients = User.objects.filter(id__in=list(recipients)) print( - '\n' f'\033[32m>>> {_("Start sending backup emails")}\033[0m' '' ) - plan_name = self.plan_name + name = self.name for user in recipients: if not user.secret_key: attachment_list = [] else: - attachment = os.path.join(PATH, f'{plan_name}-{local_now_filename()}-{time.time()}.zip') + attachment = os.path.join(PATH, f'{name}-{local_now_filename()}-{time.time()}.zip') encrypt_and_compress_zip_file(attachment, user.secret_key, files) - attachment_list = [attachment, ] - AccountBackupExecutionTaskMsg(plan_name, user).publish(attachment_list) + attachment_list = [attachment] + AccountBackupExecutionTaskMsg(name, user).publish(attachment_list) for file in files: os.remove(file) @@ -186,49 +196,37 @@ class AccountBackupHandler: return recipients = ReplayStorage.objects.filter(id__in=list(recipients)) print( - '\n' '\033[32m>>> 📃 ---> sftp \033[0m' '' ) - plan_name = self.plan_name + name = self.name encrypt_file = _('Encrypting files using encryption password') for rec in recipients: - attachment = os.path.join(PATH, f'{plan_name}-{local_now_filename()}-{time.time()}.zip') + attachment = os.path.join(PATH, f'{name}-{local_now_filename()}-{time.time()}.zip') if password: print(f'\033[32m>>> {encrypt_file}\033[0m') encrypt_and_compress_zip_file(attachment, password, files) else: zip_files(attachment, files) attachment_list = attachment - AccountBackupByObjStorageExecutionTaskMsg(plan_name, rec).publish(attachment_list) + AccountBackupByObjStorageExecutionTaskMsg(name, rec).publish(attachment_list) file_sent_to = _('The backup file will be sent to') print('{}: {}({})'.format(file_sent_to, rec.name, rec.id)) for file in files: os.remove(file) - def step_perform_task_update(self, is_success, reason): - self.execution.reason = reason[:1024] - self.execution.is_success = is_success - self.execution.save() - def _run(self): - is_success = False - error = '-' try: - backup_type = self.execution.snapshot.get('backup_type', AccountBackupType.email.value) - if backup_type == AccountBackupType.email.value: + backup_type = self.execution.snapshot.get('backup_type', AccountBackupType.email) + if backup_type == AccountBackupType.email: self.backup_by_email() - elif backup_type == AccountBackupType.object_storage.value: + elif backup_type == AccountBackupType.object_storage: self.backup_by_obj_storage() except Exception as e: - self.is_frozen = True - print(e) error = str(e) - else: - is_success = True - finally: - reason = error - self.step_perform_task_update(is_success, reason) + print(f'\033[31m>>> {error}\033[0m') + self.execution.status = Status.error + self.execution.summary['error'] = error def backup_by_obj_storage(self): object_id = self.execution.snapshot.get('id') @@ -265,7 +263,7 @@ class AccountBackupHandler: f'\033[31m>>> {warn_text}\033[0m' '' ) - raise RecipientsNotFound('Not Found Recipients') + return if recipients_part_one and recipients_part_two: print(f'\033[32m>>> {split_help_text}\033[0m') files = self.create_excel(section='front') @@ -279,16 +277,5 @@ class AccountBackupHandler: self.send_backup_mail(files, recipients) def run(self): - plan_start = _('Plan start') - time_cost = _('Duration') - error = _('An exception occurred during task execution') - print('{}: {}'.format(plan_start, local_now_display())) - time_start = time.time() - try: - self._run() - except Exception as e: - print(error) - print(e) - finally: - timedelta = round((time.time() - time_start), 2) - print('{}: {}s'.format(time_cost, timedelta)) + print('{}: {}'.format(_('Plan start'), local_now_display())) + self._run() diff --git a/apps/accounts/automations/backup_account/manager.py b/apps/accounts/automations/backup_account/manager.py index 700a07f75..70f1f591b 100644 --- a/apps/accounts/automations/backup_account/manager.py +++ b/apps/accounts/automations/backup_account/manager.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # -import time from django.utils.translation import gettext_lazy as _ from assets.automations.base.manager import BaseManager -from common.db.utils import safe_db_connection from common.utils.timezone import local_now_display from .handlers import AccountBackupHandler @@ -14,23 +12,19 @@ class AccountBackupManager(BaseManager): def do_run(self): execution = self.execution account_backup_execution_being_executed = _('The account backup plan is being executed') - print(f'\n\033[33m# {account_backup_execution_being_executed}\033[0m') - handler = AccountBackupHandler(execution) + print(f'\033[33m# {account_backup_execution_being_executed}\033[0m') + handler = AccountBackupHandler(self, execution) handler.run() def send_report_if_need(self): pass - def update_execution(self): - timedelta = int(time.time() - self.time_start) - self.execution.timedelta = timedelta - - with safe_db_connection(): - self.execution.save(update_fields=['timedelta', ]) - def print_summary(self): print('\n\n' + '-' * 80) plan_execution_end = _('Plan execution end') print('{} {}\n'.format(plan_execution_end, local_now_display())) time_cost = _('Duration') print('{}: {}s'.format(time_cost, self.duration)) + + def get_report_template(self): + return "accounts/backup_account_report.html" diff --git a/apps/accounts/automations/endpoint.py b/apps/accounts/automations/endpoint.py index db600e791..84323efad 100644 --- a/apps/accounts/automations/endpoint.py +++ b/apps/accounts/automations/endpoint.py @@ -18,7 +18,7 @@ class ExecutionManager: AutomationTypes.gather_accounts: GatherAccountsManager, AutomationTypes.verify_gateway_account: VerifyGatewayAccountManager, AutomationTypes.check_account: CheckAccountManager, - 'backup_account': AccountBackupManager, + AutomationTypes.backup_account: AccountBackupManager, } def __init__(self, execution): diff --git a/apps/accounts/models/automations/backup_account.py b/apps/accounts/models/automations/backup_account.py index 77fa6cda4..2830fd162 100644 --- a/apps/accounts/models/automations/backup_account.py +++ b/apps/accounts/models/automations/backup_account.py @@ -79,35 +79,3 @@ class BackupAccountAutomation(AccountBaseAutomation): def save(self, *args, **kwargs): self.type = AutomationTypes.backup_account super().save(*args, **kwargs) - -# class AccountBackupExecution(AutomationExecution): -# plan = models.ForeignKey( -# 'AccountBackupAutomation', related_name='execution', on_delete=models.CASCADE, -# verbose_name=_('Account backup plan') -# ) -# -# class Meta: -# verbose_name = _('Account backup execution') -# -# @property -# def types(self): -# types = self.snapshot.get('types') -# return types -# -# @lazyproperty -# def backup_accounts(self): -# from accounts.models import Account -# # TODO 可以优化一下查询 在账号上做 category 的缓存 避免数据量大时连表操作 -# qs = Account.objects.filter( -# asset__platform__type__in=self.types -# ).annotate(type=F('asset__platform__type')) -# return qs -# -# @property -# def manager_type(self): -# return 'backup_account' -# -# def start(self): -# from accounts.automations.endpoint import ExecutionManager -# manager = ExecutionManager(execution=self) -# return manager.run() diff --git a/apps/accounts/models/automations/base.py b/apps/accounts/models/automations/base.py index 8a2464c7e..799cbd3d7 100644 --- a/apps/accounts/models/automations/base.py +++ b/apps/accounts/models/automations/base.py @@ -40,6 +40,9 @@ class AutomationExecution(AssetAutomationExecution): ('view_pushaccountexecution', _('Can view push account execution')), ('add_pushaccountexecution', _('Can add push account execution')), + + ('view_backupaccountexecution', _('Can view backup account execution')), + ('add_backupaccountexecution', _('Can add backup account execution')), ] @property diff --git a/apps/accounts/serializers/automations/backup.py b/apps/accounts/serializers/automations/backup.py index c62c76e7a..eec829663 100644 --- a/apps/accounts/serializers/automations/backup.py +++ b/apps/accounts/serializers/automations/backup.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- # from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers +from accounts.const import AutomationTypes from accounts.models import BackupAccountAutomation -from common.const.choices import Trigger -from common.serializers.fields import LabeledChoiceField, EncryptedField +from common.serializers.fields import EncryptedField from common.utils import get_logger from .base import BaseAutomationSerializer logger = get_logger(__file__) -__all__ = ['BackupAccountSerializer', 'BackupAccountExecutionSerializer'] +__all__ = ['BackupAccountSerializer'] class BackupAccountSerializer(BaseAutomationSerializer): @@ -41,14 +40,6 @@ class BackupAccountSerializer(BaseAutomationSerializer): 'types': {'label': _('Asset type')} } - -class BackupAccountExecutionSerializer(serializers.ModelSerializer): - trigger = LabeledChoiceField(choices=Trigger.choices, label=_("Trigger mode"), read_only=True) - - class Meta: - model = BackupAccountAutomation - read_only_fields = [ - 'id', 'date_start', 'timedelta', 'snapshot', - 'trigger', 'reason', 'is_success', 'org_id' - ] - fields = read_only_fields + ['plan'] + @property + def model_type(self): + return AutomationTypes.backup_account diff --git a/apps/accounts/serializers/automations/base.py b/apps/accounts/serializers/automations/base.py index 086f1b297..4bb9038b2 100644 --- a/apps/accounts/serializers/automations/base.py +++ b/apps/accounts/serializers/automations/base.py @@ -26,15 +26,14 @@ class BaseAutomationSerializer(PeriodTaskSerializerMixin, BulkOrgResourceModelSe class Meta: read_only_fields = [ 'date_created', 'date_updated', 'created_by', - 'periodic_display', 'executed_amount' + 'periodic_display', 'executed_amount', 'type' ] fields = read_only_fields + [ 'id', 'name', 'is_periodic', 'interval', 'crontab', 'comment', - 'type', 'accounts', 'nodes', 'assets', 'is_active', + 'accounts', 'nodes', 'assets', 'is_active', ] extra_kwargs = { 'name': {'required': True}, - 'type': {'read_only': True}, 'executed_amount': {'label': _('Executions')}, } @@ -58,7 +57,7 @@ class AutomationExecutionSerializer(serializers.ModelSerializer): class Meta: model = AutomationExecution read_only_fields = [ - 'trigger', 'date_start', 'date_finished', 'snapshot', 'status' + 'trigger', 'date_start', 'date_finished', 'snapshot', 'status', 'duration' ] fields = ['id', 'automation', 'type'] + read_only_fields diff --git a/apps/accounts/templates/accounts/backup_account_report.html b/apps/accounts/templates/accounts/backup_account_report.html new file mode 100644 index 000000000..80d71e7ef --- /dev/null +++ b/apps/accounts/templates/accounts/backup_account_report.html @@ -0,0 +1,78 @@ +{% load i18n %} + +
+

{% trans 'The following is a summary of account backup tasks, please review and handle them' %}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
任务汇总:
{% trans 'Task name' %}:{{ execution.automation.name }}
{% trans 'Date start' %}:{{ execution.date_start | date:"Y/m/d H:i:s" }}
{% trans 'Date end' %}:{{ execution.date_finished | date:"Y/m/d H:i:s" }}
{% trans 'Time using' %}:{{ execution.duration }}s
{% trans 'Account count' %}:{{ summary.total_accounts }}
{% trans 'Type count' %}:{{ summary.total_types }}
+
+ + + diff --git a/apps/accounts/urls.py b/apps/accounts/urls.py index 09764d067..6df017ee5 100644 --- a/apps/accounts/urls.py +++ b/apps/accounts/urls.py @@ -15,7 +15,7 @@ router.register(r'gathered-accounts', api.GatheredAccountViewSet, 'gathered-acco router.register(r'account-secrets', api.AccountSecretsViewSet, 'account-secret') router.register(r'account-templates', api.AccountTemplateViewSet, 'account-template') router.register(r'account-template-secrets', api.AccountTemplateSecretsViewSet, 'account-template-secret') -router.register(r'account-backup-plans', api.AccountBackupPlanViewSet, 'account-backup') +router.register(r'account-backup-plans', api.BackupAccountViewSet, 'account-backup') router.register(r'account-backup-plan-executions', api.BackupAccountExecutionViewSet, 'account-backup-execution') router.register(r'change-secret-automations', api.ChangeSecretAutomationViewSet, 'change-secret-automation') router.register(r'change-secret-executions', api.ChangSecretExecutionViewSet, 'change-secret-execution') diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 1b0ee0865..d182eb88a 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -98,7 +98,7 @@ class BaseManager: self.summary = defaultdict(int) self.result = defaultdict(list) self.duration = 0 - self.status = 'success' + self.status = Status.success def get_assets_group_by_platform(self): return self.execution.all_assets_group_by_platform() diff --git a/apps/assets/serializers/automations/base.py b/apps/assets/serializers/automations/base.py index 522049d17..62ffc3542 100644 --- a/apps/assets/serializers/automations/base.py +++ b/apps/assets/serializers/automations/base.py @@ -42,17 +42,17 @@ class AutomationExecutionSerializer(serializers.ModelSerializer): class Meta: model = AutomationExecution read_only_fields = [ - 'trigger', 'date_start', 'date_finished', 'snapshot', 'status' + 'trigger', 'date_start', 'date_finished', 'snapshot', 'status', 'duration' ] fields = ['id', 'automation'] + read_only_fields @staticmethod def get_status(obj): - if obj.status == 'success': - return _("Success") - elif obj.status == 'pending': - return _("Pending") - return obj.status + from common.const import Status + status = Status._member_map_.get(obj.status) + if status is None: + return obj.status + return status.label @staticmethod def get_snapshot(obj):