feat: 逃生通道

This commit is contained in:
feng626
2022-01-10 19:02:18 +08:00
committed by Jiangjie.Bai
parent 5cdc4c3c28
commit b0932e5137
23 changed files with 849 additions and 125 deletions

View File

@@ -10,3 +10,4 @@ from .domain import *
from .cmd_filter import *
from .gathered_user import *
from .favorite_asset import *
from .backup import *

55
apps/assets/api/backup.py Normal file
View File

@@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
#
from rest_framework import status, mixins, viewsets
from rest_framework.response import Response
from common.permissions import IsOrgAdmin
from orgs.mixins.api import OrgBulkModelViewSet
from .. import serializers
from ..tasks import execute_account_backup_plan
from ..models import (
AccountBackupPlan, AccountBackupPlanExecution
)
__all__ = [
'AccountBackupPlanViewSet', 'AccountBackupPlanExecutionViewSet'
]
class AccountBackupPlanViewSet(OrgBulkModelViewSet):
model = AccountBackupPlan
filter_fields = ('name',)
search_fields = filter_fields
ordering_fields = ('name',)
ordering = ('name',)
serializer_class = serializers.AccountBackupPlanSerializer
permission_classes = (IsOrgAdmin,)
class AccountBackupPlanExecutionViewSet(
mixins.CreateModelMixin, mixins.ListModelMixin,
mixins.RetrieveModelMixin, viewsets.GenericViewSet
):
serializer_class = serializers.AccountBackupPlanExecutionSerializer
search_fields = ('trigger', 'plan_id')
filterset_fields = search_fields
permission_classes = (IsOrgAdmin,)
def get_queryset(self):
queryset = AccountBackupPlanExecution.objects.all()
return queryset
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
pid = serializer.data.get('plan')
task = execute_account_backup_plan.delay(
pid=pid, trigger=AccountBackupPlanExecution.Trigger.manual
)
return Response({'task': task.id}, status=status.HTTP_201_CREATED)
def filter_queryset(self, queryset):
queryset = super().filter_queryset(queryset)
queryset = queryset.order_by('-date_start')
return queryset

View File

@@ -0,0 +1,62 @@
# Generated by Django 3.1.13 on 2022-01-12 11:59
import common.db.encoder
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import uuid
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('assets', '0083_auto_20211215_1436'),
]
operations = [
migrations.CreateModel(
name='AccountBackupPlan',
fields=[
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('name', models.CharField(max_length=128, verbose_name='Name')),
('is_periodic', models.BooleanField(default=False)),
('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')),
('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')),
('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('types', models.IntegerField(choices=[(255, 'All'), (1, 'Asset'), (2, 'Application')], default=255, verbose_name='Type')),
('comment', models.TextField(blank=True, verbose_name='Comment')),
('recipients', models.ManyToManyField(blank=True, related_name='recipient_escape_route_plans', to=settings.AUTH_USER_MODEL, verbose_name='Recipient')),
],
options={
'verbose_name': 'Account backup plan',
'ordering': ['name'],
'unique_together': {('name', 'org_id')},
},
),
migrations.AlterField(
model_name='systemuser',
name='protocol',
field=models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('redis', 'Redis'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol'),
),
migrations.CreateModel(
name='AccountBackupPlanExecution',
fields=[
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('date_start', models.DateTimeField(auto_now_add=True, verbose_name='Date start')),
('timedelta', models.FloatField(default=0.0, null=True, verbose_name='Time')),
('plan_snapshot', models.JSONField(blank=True, default=dict, encoder=common.db.encoder.ModelJSONFieldEncoder, null=True, verbose_name='Escape route snapshot')),
('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')], default='manual', max_length=128, verbose_name='Trigger mode')),
('reason', models.CharField(blank=True, max_length=1024, null=True, verbose_name='Reason')),
('is_success', models.BooleanField(default=False, verbose_name='Is success')),
('plan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='execution', to='assets.accountbackupplan', verbose_name='Account backup plan')),
],
options={
'verbose_name': 'Account backup execution',
},
),
]

View File

@@ -12,3 +12,4 @@ from .utils import *
from .authbook import *
from .gathered_user import *
from .favorite_asset import *
from .backup import *

View File

@@ -0,0 +1,145 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
import uuid
from celery import current_task
from django.db import models
from django.utils.translation import ugettext_lazy as _
from orgs.mixins.models import OrgModelMixin
from ops.mixin import PeriodTaskModelMixin
from common.utils import get_logger
from common.db.encoder import ModelJSONFieldEncoder
from common.db.models import BitOperationChoice
from common.mixins.models import CommonModelMixin
__all__ = ['AccountBackupPlan', 'AccountBackupPlanExecution', 'Type']
logger = get_logger(__file__)
class Type(BitOperationChoice):
NONE = 0
ALL = 0xff
Asset = 0b1
App = 0b1 << 1
DB_CHOICES = (
(ALL, _('All')),
(Asset, _('Asset')),
(App, _('Application'))
)
NAME_MAP = {
ALL: "all",
Asset: "asset",
App: "application"
}
NAME_MAP_REVERSE = {v: k for k, v in NAME_MAP.items()}
CHOICES = []
for i, j in DB_CHOICES:
CHOICES.append((NAME_MAP[i], j))
class AccountBackupPlan(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
types = models.IntegerField(choices=Type.DB_CHOICES, default=Type.ALL, verbose_name=_('Type'))
recipients = models.ManyToManyField(
'users.User', related_name='recipient_escape_route_plans', blank=True,
verbose_name=_("Recipient")
)
comment = models.TextField(blank=True, verbose_name=_('Comment'))
def __str__(self):
return f'{self.name}({self.org_id})'
class Meta:
ordering = ['name']
unique_together = [('name', 'org_id')]
verbose_name = _('Account backup plan')
def get_register_task(self):
from ..tasks import execute_account_backup_plan
name = "account_backup_plan_period_{}".format(str(self.id)[:8])
task = execute_account_backup_plan.name
args = (str(self.id), AccountBackupPlanExecution.Trigger.timing)
kwargs = {}
return name, task, args, kwargs
def to_attr_json(self):
return {
'name': self.name,
'is_periodic': self.is_periodic,
'interval': self.interval,
'crontab': self.crontab,
'org_id': self.org_id,
'created_by': self.created_by,
'types': Type.value_to_choices(self.types),
'recipients': {
str(recipient.id): (str(recipient), bool(recipient.secret_key))
for recipient in self.recipients.all()
}
}
def execute(self, trigger):
try:
hid = current_task.request.id
except AttributeError:
hid = str(uuid.uuid4())
execution = AccountBackupPlanExecution.objects.create(
id=hid, plan=self, plan_snapshot=self.to_attr_json(), trigger=trigger
)
return execution.start()
class AccountBackupPlanExecution(OrgModelMixin):
class Trigger(models.TextChoices):
manual = 'manual', _('Manual trigger')
timing = 'timing', _('Timing trigger')
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
date_start = models.DateTimeField(
auto_now_add=True, verbose_name=_('Date start')
)
timedelta = models.FloatField(
default=0.0, verbose_name=_('Time'), null=True
)
plan_snapshot = models.JSONField(
encoder=ModelJSONFieldEncoder, default=dict,
blank=True, null=True, verbose_name=_('Escape route snapshot')
)
trigger = models.CharField(
max_length=128, default=Trigger.manual, choices=Trigger.choices,
verbose_name=_('Trigger mode')
)
reason = models.CharField(
max_length=1024, blank=True, null=True, verbose_name=_('Reason')
)
is_success = models.BooleanField(default=False, verbose_name=_('Is success'))
plan = models.ForeignKey(
'AccountBackupPlan', related_name='execution', on_delete=models.CASCADE,
verbose_name=_('Account backup plan')
)
class Meta:
verbose_name = _('Account backup execution')
@property
def types(self):
types = self.plan_snapshot.get('types')
return types
@property
def recipients(self):
recipients = self.plan_snapshot.get('recipients')
if not recipients:
return []
return recipients.values()
def start(self):
from ..task_handlers import ExecutionManager
manager = ExecutionManager(execution=self)
return manager.run()

View File

@@ -0,0 +1,25 @@
from django.utils.translation import ugettext_lazy as _
from users.models import User
from common.tasks import send_mail_attachment_async
class AccountBackupExecutionTaskMsg(object):
subject = _('Notification of account backup route task results')
def __init__(self, name: str, user: User):
self.name = name
self.user = user
@property
def message(self):
name = self.name
if self.user.secret_key:
return _('{} - The account backup passage task has been completed. See the attachment for details').format(name)
return _("{} - The account backup passage task has been completed: the encryption password has not been set - "
"please go to personal information -> file encryption password to set the encryption password").format(name)
def publish(self, attachment_list=None):
send_mail_attachment_async.delay(
self.subject, self.message, [self.user.email], attachment_list
)

View File

@@ -11,3 +11,4 @@ from .cmd_filter import *
from .gathered_user import *
from .favorite_asset import *
from .account import *
from .backup import *

View File

@@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
#
from django.utils.translation import ugettext as _
from rest_framework import serializers
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from ops.mixin import PeriodTaskSerializerMixin
from common.utils import get_logger
from .base import TypesField
from ..models import AccountBackupPlan, AccountBackupPlanExecution
logger = get_logger(__file__)
__all__ = ['AccountBackupPlanSerializer', 'AccountBackupPlanExecutionSerializer']
class AccountBackupPlanSerializer(PeriodTaskSerializerMixin, BulkOrgResourceModelSerializer):
types = TypesField(required=False, allow_null=True, label=_("Actions"))
class Meta:
model = AccountBackupPlan
fields = [
'id', 'name', 'is_periodic', 'interval', 'crontab', 'date_created',
'date_updated', 'created_by', 'periodic_display', 'comment',
'recipients', 'types'
]
extra_kwargs = {
'name': {'required': True},
'periodic_display': {'label': _('Periodic perform')},
'recipients': {'label': _('Recipient'), 'help_text': _(
'Currently only mail sending is supported'
)}
}
class AccountBackupPlanExecutionSerializer(serializers.ModelSerializer):
trigger_display = serializers.ReadOnlyField(
source='get_trigger_display', label=_('Trigger mode')
)
class Meta:
model = AccountBackupPlanExecution
fields = [
'id', 'date_start', 'timedelta', 'plan_snapshot', 'trigger', 'reason',
'is_success', 'plan', 'org_id', 'recipients', 'trigger_display'
]
read_only_fields = (
'id', 'date_start', 'timedelta', 'plan_snapshot', 'trigger', 'reason',
'is_success', 'org_id', 'recipients'
)

View File

@@ -6,6 +6,7 @@ from django.utils.translation import ugettext as _
from rest_framework import serializers
from common.utils import ssh_pubkey_gen, ssh_private_key_gen, validate_ssh_private_key
from assets.models import Type
class AuthSerializer(serializers.ModelSerializer):
@@ -70,3 +71,24 @@ class AuthSerializerMixin(serializers.ModelSerializer):
def update(self, instance, validated_data):
self.clean_auth_fields(validated_data)
return super().update(instance, validated_data)
class TypesField(serializers.MultipleChoiceField):
def __init__(self, *args, **kwargs):
kwargs['choices'] = Type.CHOICES
super().__init__(*args, **kwargs)
def to_representation(self, value):
return Type.value_to_choices(value)
def to_internal_value(self, data):
if data is None:
return data
return Type.choices_to_value(data)
class ActionsDisplayField(TypesField):
def to_representation(self, value):
values = super().to_representation(value)
choices = dict(Type.CHOICES)
return [choices.get(i) for i in values]

View File

@@ -0,0 +1 @@
from .endpoint import *

View File

@@ -0,0 +1,203 @@
import os
import time
import pandas as pd
from collections import defaultdict
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from assets.models import AuthBook, Asset, BaseUser, ProtocolsMixin
from assets.notifications import AccountBackupExecutionTaskMsg
from applications.models import Account, Application
from applications.const import AppType
from users.models import User
from common.utils import get_logger
from common.utils.timezone import local_now_display
from common.utils.file import encrypt_and_compress_zip_file
logger = get_logger(__file__)
PATH = os.path.join(os.path.dirname(settings.BASE_DIR), 'tmp')
class AssetAccountHandler:
@staticmethod
def get_filename(plan_name):
filename = os.path.join(
PATH, f'{plan_name}-{_("Asset")}-{local_now_display()}-{time.time()}.xlsx'
)
return filename
@staticmethod
def create_df():
df_dict = defaultdict(list)
label_key = AuthBook._meta.verbose_name
accounts = AuthBook.objects.all().prefetch_related('systemuser', 'asset')
for account in accounts:
account.load_auth()
protocol = account.asset.protocol
protocol_label = getattr(ProtocolsMixin.Protocol, protocol).label
row = {
getattr(Asset, 'hostname').field.verbose_name: account.asset.hostname,
getattr(Asset, 'ip').field.verbose_name: account.asset.ip,
}
secret_row = AccountBackupHandler.create_secret_row(account)
row.update(secret_row)
row.update({
getattr(Asset, 'protocol').field.verbose_name: protocol_label,
getattr(AuthBook, 'version').field.verbose_name: account.version
})
df_dict[label_key].append(row)
for k, v in df_dict.items():
df_dict[k] = pd.DataFrame(v)
return df_dict
class AppAccountHandler:
@staticmethod
def get_filename(plan_name):
filename = os.path.join(
PATH, f'{plan_name}-{_("Application")}-{local_now_display()}-{time.time()}.xlsx'
)
return filename
@staticmethod
def create_df():
df_dict = defaultdict(list)
accounts = Account.objects.all().prefetch_related('systemuser', 'app')
for account in accounts:
account.load_auth()
app_type = account.app.type
if app_type == 'postgresql':
label_key = getattr(AppType, 'pgsql').label
else:
label_key = getattr(AppType, app_type).label
row = {
getattr(Application, 'name').field.verbose_name: account.app.name,
getattr(Application, 'attrs').field.verbose_name: account.app.attrs
}
secret_row = AccountBackupHandler.create_secret_row(account)
row.update(secret_row)
row.update({
getattr(Account, 'version').field.verbose_name: account.version
})
df_dict[label_key].append(row)
for k, v in df_dict.items():
df_dict[k] = pd.DataFrame(v)
return df_dict
HANDLER_MAP = {
'asset': AssetAccountHandler,
'application': AppAccountHandler
}
class AccountBackupHandler:
def __init__(self, execution):
self.execution = execution
self.plan_name = self.execution.plan.name
self.is_frozen = False # 任务状态冻结标志
def create_excel(self):
logger.info(
'\n'
'\033[32m>>> 正在生成资产及应用相关备份信息文件\033[0m'
''
)
# Print task start date
time_start = time.time()
info = {}
for account_type in self.execution.types:
if account_type in HANDLER_MAP:
account_handler = HANDLER_MAP[account_type]
df = account_handler.create_df()
filename = account_handler.get_filename(self.plan_name)
info[filename] = df
for filename, df_dict in info.items():
with pd.ExcelWriter(filename) as w:
for sheet, df in df_dict.items():
sheet = sheet.replace(' ', '-')
getattr(df, 'to_excel')(w, sheet_name=sheet, index=False)
timedelta = round((time.time() - time_start), 2)
logger.info('步骤完成: 用时 {}s'.format(timedelta))
return list(info.keys())
def send_backup_mail(self, files):
recipients = self.execution.plan_snapshot.get('recipients')
if not recipients:
return
recipients = User.objects.filter(id__in=list(recipients))
logger.info(
'\n'
'\033[32m>>> 发送备份邮件\033[0m'
''
)
plan_name = self.plan_name
for user in recipients:
if not user.secret_key:
attachment_list = []
else:
password = user.secret_key.encode('utf8')
attachment = os.path.join(PATH, f'{plan_name}-{local_now_display()}-{time.time()}.zip')
encrypt_and_compress_zip_file(attachment, password, files)
attachment_list = [attachment, ]
AccountBackupExecutionTaskMsg(plan_name, user).publish(attachment_list)
logger.info('邮件已发送至{}({})'.format(user, user.email))
for file in files:
os.remove(file)
def step_perform_task_update(self, is_success, reason):
self.execution.reason = reason[:1024]
self.execution.is_success = is_success
self.execution.save()
logger.info('已完成对任务状态的更新')
def step_finished(self, is_success):
if is_success:
logger.info('任务执行成功')
else:
logger.error('任务执行失败')
def _run(self):
is_success = False
error = '-'
try:
files = self.create_excel()
self.send_backup_mail(files)
except Exception as e:
self.is_frozen = True
logger.error('任务执行被异常中断')
logger.info('下面打印发生异常的 Traceback 信息 : ')
logger.error(e, exc_info=True)
error = str(e)
else:
is_success = True
finally:
reason = error
self.step_perform_task_update(is_success, reason)
self.step_finished(is_success)
def run(self):
logger.info('任务开始: {}'.format(local_now_display()))
time_start = time.time()
try:
self._run()
except Exception as e:
logger.error('任务运行出现异常')
logger.error('下面显示异常 Traceback 信息: ')
logger.error(e, exc_info=True)
finally:
logger.info('\n任务结束: {}'.format(local_now_display()))
timedelta = round((time.time() - time_start), 2)
logger.info('用时: {}'.format(timedelta))
@staticmethod
def create_secret_row(instance):
row = {
getattr(BaseUser, 'username').field.verbose_name: instance.username,
getattr(BaseUser, 'password').field.verbose_name: instance.password,
getattr(BaseUser, 'private_key').field.verbose_name: instance.private_key,
getattr(BaseUser, 'public_key').field.verbose_name: instance.public_key
}
return row

View File

@@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
#
import time
from django.utils import timezone
from common.utils import get_logger
from common.utils.timezone import local_now_display
from .handlers import AccountBackupHandler
logger = get_logger(__name__)
class AccountBackupExecutionManager:
def __init__(self, execution):
self.execution = execution
self.date_start = timezone.now()
self.time_start = time.time()
self.date_end = None
self.time_end = None
self.timedelta = 0
def do_run(self):
execution = self.execution
logger.info('\n\033[33m# 账号备份计划正在执行\033[0m')
handler = AccountBackupHandler(execution)
handler.run()
def pre_run(self):
self.execution.date_start = self.date_start
self.execution.save()
def post_run(self):
self.time_end = time.time()
self.date_end = timezone.now()
logger.info('\n\n' + '-' * 80)
logger.info('计划执行结束 {}\n'.format(local_now_display()))
self.timedelta = self.time_end - self.time_start
logger.info('用时: {}s'.format(self.timedelta))
self.execution.timedelta = self.timedelta
self.execution.save()
def run(self):
self.pre_run()
self.do_run()
self.post_run()

View File

@@ -0,0 +1,10 @@
from .backup.manager import AccountBackupExecutionManager
class ExecutionManager:
manager_type = {
'backup': AccountBackupExecutionManager
}
def __new__(cls, execution):
return AccountBackupExecutionManager(execution)

View File

@@ -9,3 +9,4 @@ from .gather_asset_hardware_info import *
from .push_system_user import *
from .system_user_connectivity import *
from .nodes_amount import *
from .backup import *

View File

@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
#
from celery import shared_task
from common.utils import get_object_or_none, get_logger
from orgs.utils import tmp_to_org, tmp_to_root_org
from assets.models import AccountBackupPlan
logger = get_logger(__file__)
@shared_task
def execute_account_backup_plan(pid, trigger):
with tmp_to_root_org():
plan = get_object_or_none(AccountBackupPlan, pk=pid)
if not plan:
logger.error("No account backup route plan found: {}".format(pid))
return
with tmp_to_org(plan.org):
plan.execute(trigger)

View File

@@ -26,6 +26,8 @@ router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset')
router.register(r'system-users-assets-relations', api.SystemUserAssetRelationViewSet, 'system-users-assets-relation')
router.register(r'system-users-nodes-relations', api.SystemUserNodeRelationViewSet, 'system-users-nodes-relation')
router.register(r'system-users-users-relations', api.SystemUserUserRelationViewSet, 'system-users-users-relation')
router.register(r'backup', api.AccountBackupPlanViewSet, 'backup')
router.register(r'backup-execution', api.AccountBackupPlanExecutionViewSet, 'backup-execution')
cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')