From 0044f1126223a70a36966b8c012127fcfba7acc4 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Fri, 11 Nov 2022 19:20:17 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=89=A7=E8=A1=8Cadhoc=E5=92=8Cplayboo?= =?UTF-8?q?k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../0111_alter_automationexecution_status.py | 18 ++ .../migrations/0015_auto_20221111_1919.py | 28 +++ apps/ops/ansible/utils.py | 2 +- apps/ops/api/__init__.py | 2 + apps/ops/api/adhoc.py | 44 +---- apps/ops/api/celery.py | 15 +- apps/ops/api/job.py | 41 +++++ apps/ops/api/playbook.py | 28 +++ .../ops/migrations/0029_auto_20221111_1919.py | 171 ++++++++++++++++++ apps/ops/models/__init__.py | 1 + apps/ops/models/adhoc.py | 34 +++- apps/ops/models/base.py | 15 +- apps/ops/models/celery.py | 2 +- apps/ops/models/job.py | 154 ++++++++++++++++ apps/ops/models/playbook.py | 44 ++--- apps/ops/serializers/adhoc.py | 81 +++------ apps/ops/serializers/job.py | 29 +++ apps/ops/serializers/playbook.py | 28 +++ apps/ops/tasks.py | 39 +++- apps/ops/urls/api_urls.py | 17 +- .../migrations/0032_auto_20221111_1919.py | 31 ++++ ...22_alter_applyassetticket_apply_actions.py | 18 ++ 22 files changed, 680 insertions(+), 162 deletions(-) create mode 100644 apps/assets/migrations/0111_alter_automationexecution_status.py create mode 100644 apps/audits/migrations/0015_auto_20221111_1919.py create mode 100644 apps/ops/api/job.py create mode 100644 apps/ops/api/playbook.py create mode 100644 apps/ops/migrations/0029_auto_20221111_1919.py create mode 100644 apps/ops/models/job.py create mode 100644 apps/ops/serializers/job.py create mode 100644 apps/ops/serializers/playbook.py create mode 100644 apps/perms/migrations/0032_auto_20221111_1919.py create mode 100644 apps/tickets/migrations/0022_alter_applyassetticket_apply_actions.py diff --git a/apps/assets/migrations/0111_alter_automationexecution_status.py b/apps/assets/migrations/0111_alter_automationexecution_status.py new file mode 100644 index 000000000..5ccaf638f --- /dev/null +++ b/apps/assets/migrations/0111_alter_automationexecution_status.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-11 11:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0110_changesecretrecord_asset'), + ] + + operations = [ + migrations.AlterField( + model_name='automationexecution', + name='status', + field=models.CharField(default='pending', max_length=16, verbose_name='Status'), + ), + ] diff --git a/apps/audits/migrations/0015_auto_20221111_1919.py b/apps/audits/migrations/0015_auto_20221111_1919.py new file mode 100644 index 000000000..b638474ee --- /dev/null +++ b/apps/audits/migrations/0015_auto_20221111_1919.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.14 on 2022-11-11 11:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('audits', '0014_auto_20220505_1902'), + ] + + operations = [ + migrations.AlterField( + model_name='ftplog', + name='operate', + field=models.CharField(choices=[('mkdir', 'Mkdir'), ('rmdir', 'Rmdir'), ('delete', 'Delete'), ('upload', 'Upload'), ('rename', 'Rename'), ('symlink', 'Symlink'), ('download', 'Download')], max_length=16, verbose_name='Operate'), + ), + migrations.AlterField( + model_name='operatelog', + name='action', + field=models.CharField(choices=[('view', 'View'), ('update', 'Update'), ('delete', 'Delete'), ('create', 'Create')], max_length=16, verbose_name='Action'), + ), + migrations.AlterField( + model_name='userloginlog', + name='status', + field=models.BooleanField(choices=[(1, 'Success'), (0, 'Failed')], default=1, verbose_name='Status'), + ), + ] diff --git a/apps/ops/ansible/utils.py b/apps/ops/ansible/utils.py index 478badc56..daeb98b14 100644 --- a/apps/ops/ansible/utils.py +++ b/apps/ops/ansible/utils.py @@ -3,4 +3,4 @@ from django.conf import settings def get_ansible_task_log_path(task_id): from ops.utils import get_task_log_path - return get_task_log_path(settings.ANSIBLE_LOG_DIR, task_id, level=3) + return get_task_log_path(settings.CELERY_LOG_DIR, task_id, level=2) diff --git a/apps/ops/api/__init__.py b/apps/ops/api/__init__.py index 8eb5356e4..e82f3fb2e 100644 --- a/apps/ops/api/__init__.py +++ b/apps/ops/api/__init__.py @@ -2,3 +2,5 @@ # from .adhoc import * from .celery import * +from .job import * +from .playbook import * diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index 71e818fbb..ce7e8ad57 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -1,52 +1,22 @@ # -*- coding: utf-8 -*- # -from django.shortcuts import get_object_or_404 -from rest_framework import viewsets, generics -from rest_framework.views import Response - -from common.drf.serializers import CeleryTaskExecutionSerializer -from ..models import AdHoc, AdHocExecution +from rest_framework import viewsets +from ..models import AdHoc from ..serializers import ( - AdHocSerializer, - AdHocExecutionSerializer, - AdHocDetailSerializer, + AdHocSerializer, AdhocListSerializer, ) __all__ = [ - 'AdHocViewSet', 'AdHocExecutionViewSet' + 'AdHocViewSet' ] class AdHocViewSet(viewsets.ModelViewSet): queryset = AdHoc.objects.all() - serializer_class = AdHocSerializer def get_serializer_class(self): - if self.action == 'retrieve': - return AdHocDetailSerializer - return super().get_serializer_class() - - -class AdHocExecutionViewSet(viewsets.ModelViewSet): - queryset = AdHocExecution.objects.all() - serializer_class = AdHocExecutionSerializer - - def get_queryset(self): - task_id = self.request.query_params.get('task') - adhoc_id = self.request.query_params.get('adhoc') - - if task_id: - task = get_object_or_404(AdHoc, id=task_id) - adhocs = task.adhoc.all() - self.queryset = self.queryset.filter(adhoc__in=adhocs) - - if adhoc_id: - adhoc = get_object_or_404(AdHoc, id=adhoc_id) - self.queryset = self.queryset.filter(adhoc=adhoc) - return self.queryset - - - - + if self.action != 'list': + return AdhocListSerializer + return AdHocSerializer diff --git a/apps/ops/api/celery.py b/apps/ops/api/celery.py index 61d13db17..4376a9232 100644 --- a/apps/ops/api/celery.py +++ b/apps/ops/api/celery.py @@ -98,6 +98,11 @@ class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet): return queryset +class CelerySummaryAPIView(generics.RetrieveAPIView): + def get(self, request, *args, **kwargs): + pass + + class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet): queryset = CeleryTask.objects.all() serializer_class = CeleryTaskSerializer @@ -107,11 +112,11 @@ class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet): class CeleryTaskExecutionViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet): serializer_class = CeleryTaskExecutionSerializer http_method_names = ('get', 'head', 'options',) + queryset = CeleryTaskExecution.objects.all() def get_queryset(self): - task_id = self.kwargs.get("task_pk") + task_id = self.request.query_params.get('task_id') if task_id: - task = CeleryTask.objects.get(pk=task_id) - return CeleryTaskExecution.objects.filter(name=task.name) - else: - return CeleryTaskExecution.objects.none() + task = get_object_or_404(CeleryTask, id=task_id) + self.queryset = self.queryset.filter(name=task.name) + return self.queryset diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py new file mode 100644 index 000000000..eaad4514c --- /dev/null +++ b/apps/ops/api/job.py @@ -0,0 +1,41 @@ +from django.shortcuts import get_object_or_404 +from rest_framework import viewsets + +from ops.models import Job, JobExecution +from ops.serializers.job import JobSerializer, JobExecutionSerializer + +__all__ = ['JobViewSet', 'JobExecutionViewSet'] + +from ops.tasks import run_ops_job, run_ops_job_executions + + +class JobViewSet(viewsets.ModelViewSet): + serializer_class = JobSerializer + queryset = Job.objects.all() + + def get_queryset(self): + return self.queryset.filter(instant=False) + + def perform_create(self, serializer): + instance = serializer.save() + if instance.instant: + run_ops_job.delay(instance.id) + + +class JobExecutionViewSet(viewsets.ModelViewSet): + serializer_class = JobExecutionSerializer + queryset = JobExecution.objects.all() + http_method_names = ('get', 'post', 'head', 'options',) + + def perform_create(self, serializer): + instance = serializer.save() + run_ops_job_executions.delay(instance.id) + + def get_queryset(self): + job_id = self.request.query_params.get('job_id') + job_type = self.request.query_params.get('type') + if job_id: + self.queryset = self.queryset.filter(job_id=job_id) + if job_type: + self.queryset = self.queryset.filter(job__type=job_type) + return self.queryset diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py new file mode 100644 index 000000000..2cb846c0e --- /dev/null +++ b/apps/ops/api/playbook.py @@ -0,0 +1,28 @@ +import os +import zipfile + +from django.conf import settings +from rest_framework import viewsets +from ..models import Playbook +from ..serializers.playbook import PlaybookSerializer + +__all__ = ["PlaybookViewSet"] + + +def unzip_playbook(src, dist): + fz = zipfile.ZipFile(src, 'r') + for file in fz.namelist(): + fz.extract(file, dist) + + +class PlaybookViewSet(viewsets.ModelViewSet): + queryset = Playbook.objects.all() + serializer_class = PlaybookSerializer + + def perform_create(self, serializer): + instance = serializer.save() + src_path = os.path.join(settings.MEDIA_ROOT, instance.path.name) + dest_path = os.path.join(settings.DATA_DIR, "ops", "playbook", instance.id.__str__()) + if os.path.exists(dest_path): + os.makedirs(dest_path) + unzip_playbook(src_path, dest_path) diff --git a/apps/ops/migrations/0029_auto_20221111_1919.py b/apps/ops/migrations/0029_auto_20221111_1919.py new file mode 100644 index 000000000..072e59867 --- /dev/null +++ b/apps/ops/migrations/0029_auto_20221111_1919.py @@ -0,0 +1,171 @@ +# Generated by Django 3.2.14 on 2022-11-11 11:19 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('assets', '0111_alter_automationexecution_status'), + ('ops', '0028_celerytask_last_published_time'), + ] + + operations = [ + migrations.CreateModel( + name='Job', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=128, null=True, verbose_name='Name')), + ('instant', models.BooleanField(default=False)), + ('args', models.CharField(blank=True, default='', max_length=1024, null=True, verbose_name='Args')), + ('module', models.CharField(choices=[('shell', 'Shell'), ('win_shell', 'Powershell')], default='shell', max_length=128, null=True, verbose_name='Module')), + ('type', models.CharField(choices=[('adhoc', 'Adhoc'), ('playbook', 'Playbook')], default='adhoc', max_length=128, verbose_name='Type')), + ('runas', models.CharField(default='root', max_length=128, verbose_name='Runas')), + ('runas_policy', models.CharField(choices=[('privileged_only', 'Privileged Only'), ('privileged_first', 'Privileged First'), ('skip', 'Skip')], default='skip', max_length=128, verbose_name='Runas policy')), + ('assets', models.ManyToManyField(to='assets.Asset', verbose_name='Assets')), + ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='JobExecution', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('task_id', models.UUIDField(null=True)), + ('status', models.CharField(default='running', max_length=16, verbose_name='Status')), + ('result', models.JSONField(blank=True, null=True, verbose_name='Result')), + ('summary', models.JSONField(default=dict, verbose_name='Summary')), + ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), + ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), + ('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')), + ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ('job', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='ops.job')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AlterUniqueTogether( + name='playbooktemplate', + unique_together=None, + ), + migrations.RemoveField( + model_name='adhoc', + name='account', + ), + migrations.RemoveField( + model_name='adhoc', + name='account_policy', + ), + migrations.RemoveField( + model_name='adhoc', + name='assets', + ), + migrations.RemoveField( + model_name='adhoc', + name='crontab', + ), + migrations.RemoveField( + model_name='adhoc', + name='date_last_run', + ), + migrations.RemoveField( + model_name='adhoc', + name='interval', + ), + migrations.RemoveField( + model_name='adhoc', + name='is_periodic', + ), + migrations.RemoveField( + model_name='adhoc', + name='last_execution', + ), + migrations.RemoveField( + model_name='adhoc', + name='org_id', + ), + migrations.RemoveField( + model_name='playbook', + name='account', + ), + migrations.RemoveField( + model_name='playbook', + name='account_policy', + ), + migrations.RemoveField( + model_name='playbook', + name='assets', + ), + migrations.RemoveField( + model_name='playbook', + name='comment', + ), + migrations.RemoveField( + model_name='playbook', + name='crontab', + ), + migrations.RemoveField( + model_name='playbook', + name='date_last_run', + ), + migrations.RemoveField( + model_name='playbook', + name='interval', + ), + migrations.RemoveField( + model_name='playbook', + name='is_periodic', + ), + migrations.RemoveField( + model_name='playbook', + name='last_execution', + ), + migrations.RemoveField( + model_name='playbook', + name='org_id', + ), + migrations.RemoveField( + model_name='playbook', + name='template', + ), + migrations.AlterField( + model_name='adhoc', + name='module', + field=models.CharField(choices=[('shell', 'Shell'), ('win_shell', 'Powershell')], default='shell', max_length=128, verbose_name='Module'), + ), + migrations.AlterField( + model_name='playbook', + name='name', + field=models.CharField(max_length=128, null=True, verbose_name='Name'), + ), + migrations.AlterField( + model_name='playbook', + name='path', + field=models.FileField(upload_to='playbooks/'), + ), + migrations.DeleteModel( + name='PlaybookExecution', + ), + migrations.DeleteModel( + name='PlaybookTemplate', + ), + migrations.AddField( + model_name='job', + name='playbook', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='ops.playbook', verbose_name='Playbook'), + ), + ] diff --git a/apps/ops/models/__init__.py b/apps/ops/models/__init__.py index 93b630dd6..b6edb768d 100644 --- a/apps/ops/models/__init__.py +++ b/apps/ops/models/__init__.py @@ -4,3 +4,4 @@ from .adhoc import * from .celery import * from .playbook import * +from .job import * diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index 22fd0f054..e94223fb9 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -1,29 +1,43 @@ # ~*~ coding: utf-8 ~*~ import os.path +import uuid from django.db import models from django.utils.translation import ugettext_lazy as _ +from common.db.models import BaseCreateUpdateModel from common.utils import get_logger from .base import BaseAnsibleJob, BaseAnsibleExecution from ..ansible import AdHocRunner __all__ = ["AdHoc", "AdHocExecution"] - logger = get_logger(__file__) -class AdHoc(BaseAnsibleJob): - pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all') - module = models.CharField(max_length=128, default='shell', verbose_name=_('Module')) - args = models.CharField(max_length=1024, default='', verbose_name=_('Args')) - last_execution = models.ForeignKey('AdHocExecution', verbose_name=_("Last execution"), - on_delete=models.SET_NULL, null=True, blank=True) +class AdHoc(BaseCreateUpdateModel): + class Modules(models.TextChoices): + shell = 'shell', _('Shell') + winshell = 'win_shell', _('Powershell') - def get_register_task(self): - from ops.tasks import run_adhoc - return "run_adhoc_{}".format(self.id), run_adhoc, (str(self.id),), {} + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + name = models.CharField(max_length=128, verbose_name=_('Name')) + pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all') + module = models.CharField(max_length=128, choices=Modules.choices, default=Modules.shell, + verbose_name=_('Module')) + args = models.CharField(max_length=1024, default='', verbose_name=_('Args')) + owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + + @property + def row_count(self): + if len(self.args) == 0: + return 0 + count = str(self.args).count('\n') + return count + 1 + + @property + def size(self): + return len(self.args) def __str__(self): return "{}: {}".format(self.module, self.args) diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py index a37982673..3d6d1438d 100644 --- a/apps/ops/models/base.py +++ b/apps/ops/models/base.py @@ -17,7 +17,8 @@ class BaseAnsibleJob(PeriodTaskModelMixin, JMSOrgBaseModel): assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) account_policy = models.CharField(max_length=128, default='root', verbose_name=_('Account policy')) - last_execution = models.ForeignKey('BaseAnsibleExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True) + last_execution = models.ForeignKey('BaseAnsibleExecution', verbose_name=_("Last execution"), + on_delete=models.SET_NULL, null=True) date_last_run = models.DateTimeField(null=True, verbose_name=_('Date last run')) class Meta: @@ -118,12 +119,6 @@ class BaseAnsibleExecution(models.Model): def is_success(self): return self.status == 'success' - @property - def time_cost(self): - if self.date_finished and self.date_start: - return (self.date_finished - self.date_start).total_seconds() - return None - @property def short_id(self): return str(self.id).split('-')[-1] @@ -134,4 +129,8 @@ class BaseAnsibleExecution(models.Model): return self.date_finished - self.date_start return None - + @property + def time_cost(self): + if self.date_finished and self.date_start: + return (self.date_finished - self.date_start).total_seconds() + return None diff --git a/apps/ops/models/celery.py b/apps/ops/models/celery.py index 55f05129f..01251f8b2 100644 --- a/apps/ops/models/celery.py +++ b/apps/ops/models/celery.py @@ -19,7 +19,7 @@ class CeleryTask(models.Model): def meta(self): task = app.tasks.get(self.name, None) return { - "verbose_name": getattr(task, 'verbose_name', None), + "display_name": getattr(task, 'verbose_name', None), "comment": getattr(task, 'comment', None), "queue": getattr(task, 'queue', 'default') } diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py new file mode 100644 index 000000000..9199ab902 --- /dev/null +++ b/apps/ops/models/job.py @@ -0,0 +1,154 @@ +import os +import uuid +import logging + +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from celery import current_task + +from common.db.models import BaseCreateUpdateModel + +__all__ = ["Job", "JobExecution"] + +from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner + + +class Job(BaseCreateUpdateModel): + class Types(models.TextChoices): + adhoc = 'adhoc', _('Adhoc') + playbook = 'playbook', _('Playbook') + + class RunasPolicies(models.TextChoices): + privileged_only = 'privileged_only', _('Privileged Only') + privileged_first = 'privileged_first', _('Privileged First') + skip = 'skip', _('Skip') + + class Modules(models.TextChoices): + shell = 'shell', _('Shell') + winshell = 'win_shell', _('Powershell') + + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + name = models.CharField(max_length=128, null=True, verbose_name=_('Name')) + instant = models.BooleanField(default=False) + args = models.CharField(max_length=1024, default='', verbose_name=_('Args'), null=True, blank=True) + module = models.CharField(max_length=128, choices=Modules.choices, default=Modules.shell, + verbose_name=_('Module'), null=True) + playbook = models.ForeignKey('ops.Playbook', verbose_name=_("Playbook"), null=True, on_delete=models.SET_NULL) + type = models.CharField(max_length=128, choices=Types.choices, default=Types.adhoc, verbose_name=_("Type")) + owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) + runas = models.CharField(max_length=128, default='root', verbose_name=_('Runas')) + runas_policy = models.CharField(max_length=128, choices=RunasPolicies.choices, default=RunasPolicies.skip, + verbose_name=_('Runas policy')) + + @property + def inventory(self): + return JMSInventory(self.assets.all(), self.runas_policy, self.runas) + + def create_execution(self): + return self.executions.create() + + +class JobExecution(BaseCreateUpdateModel): + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + task_id = models.UUIDField(null=True) + status = models.CharField(max_length=16, verbose_name=_('Status'), default='running') + job = models.ForeignKey(Job, on_delete=models.CASCADE, related_name='executions', null=True) + result = models.JSONField(blank=True, null=True, verbose_name=_('Result')) + summary = models.JSONField(default=dict, verbose_name=_('Summary')) + creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) + date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True) + date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished")) + + def get_runner(self): + inv = self.job.inventory + inv.write_to_file(self.inventory_path) + + if self.job.type == 'adhoc': + runner = AdHocRunner( + self.inventory_path, self.job.module, module_args=self.job.args, + pattern="all", project_dir=self.private_dir + ) + elif self.job.type == 'playbook': + runner = PlaybookRunner( + self.inventory_path, self.job.playbook.work_path + ) + else: + raise Exception("unsupported job type") + return runner + + @property + def short_id(self): + return str(self.id).split('-')[-1] + + @property + def timedelta(self): + if self.date_start and self.date_finished: + return self.date_finished - self.date_start + return None + + @property + def is_finished(self): + return self.status in ['success', 'failed'] + + @property + def is_success(self): + return self.status == 'success' + + @property + def time_cost(self): + if self.date_finished and self.date_start: + return (self.date_finished - self.date_start).total_seconds() + return None + + @property + def inventory_path(self): + return os.path.join(self.private_dir, 'inventory', 'hosts') + + @property + def private_dir(self): + uniq = self.date_created.strftime('%Y%m%d_%H%M%S') + '_' + self.short_id + job_name = self.job.name if self.job.name else 'instant' + return os.path.join(settings.ANSIBLE_DIR, job_name, uniq) + + def set_error(self, error): + this = self.__class__.objects.get(id=self.id) # 重新获取一次，避免数据库超时连接超时 + this.status = 'failed' + this.summary['error'] = str(error) + this.finish_task() + + def set_result(self, cb): + status_mapper = { + 'successful': 'success', + } + this = self.__class__.objects.get(id=self.id) + this.status = status_mapper.get(cb.status, cb.status) + this.summary = cb.summary + this.result = cb.result + this.finish_task() + + def finish_task(self): + self.date_finished = timezone.now() + self.save(update_fields=['result', 'status', 'summary', 'date_finished']) + + def set_celery_id(self): + if not current_task: + return + task_id = current_task.request.root_id + self.task_id = task_id + + def start(self, **kwargs): + self.date_start = timezone.now() + self.set_celery_id() + self.save() + runner = self.get_runner() + try: + cb = runner.run(**kwargs) + self.set_result(cb) + return cb + except Exception as e: + logging.error(e, exc_info=True) + self.set_error(e) diff --git a/apps/ops/models/playbook.py b/apps/ops/models/playbook.py index a0c11db3b..6e0155288 100644 --- a/apps/ops/models/playbook.py +++ b/apps/ops/models/playbook.py @@ -1,39 +1,19 @@ +import os.path +import uuid + +from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ -from orgs.mixins.models import JMSOrgBaseModel -from .base import BaseAnsibleExecution, BaseAnsibleJob +from common.db.models import BaseCreateUpdateModel -class PlaybookTemplate(JMSOrgBaseModel): - name = models.CharField(max_length=128, verbose_name=_("Name")) - path = models.FilePathField(verbose_name=_("Path")) - comment = models.TextField(verbose_name=_("Comment"), blank=True) - - def __str__(self): - return self.name - - class Meta: - ordering = ['name'] - verbose_name = _("Playbook template") - unique_together = [('org_id', 'name')] - - -class Playbook(BaseAnsibleJob): - path = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) +class Playbook(BaseCreateUpdateModel): + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + name = models.CharField(max_length=128, verbose_name=_('Name'), null=True) + path = models.FileField(upload_to='playbooks/') owner = models.ForeignKey('users.User', verbose_name=_("Owner"), on_delete=models.SET_NULL, null=True) - comment = models.TextField(blank=True, verbose_name=_("Comment")) - template = models.ForeignKey('PlaybookTemplate', verbose_name=_("Template"), on_delete=models.SET_NULL, null=True) - last_execution = models.ForeignKey('PlaybookExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) - def get_register_task(self): - name = "automation_strategy_period_{}".format(str(self.id)[:8]) - task = execute_automation_strategy.name - args = (str(self.id), Trigger.timing) - kwargs = {} - return name, task, args, kwargs - - -class PlaybookExecution(BaseAnsibleExecution): - task = models.ForeignKey('Playbook', verbose_name=_("Task"), on_delete=models.CASCADE) - path = models.FilePathField(max_length=1024, verbose_name=_("Run dir")) + @property + def work_path(self): + return os.path.join(settings.DATA_DIR, "ops", "playbook", self.id.__str__()) diff --git a/apps/ops/serializers/adhoc.py b/apps/ops/serializers/adhoc.py index 5df047bfa..2120c8c50 100644 --- a/apps/ops/serializers/adhoc.py +++ b/apps/ops/serializers/adhoc.py @@ -1,11 +1,33 @@ # ~*~ coding: utf-8 ~*~ from __future__ import unicode_literals -from rest_framework import serializers -from django.shortcuts import reverse +import datetime + +from rest_framework import serializers + +from common.drf.fields import ReadableHiddenField from ..models import AdHoc, AdHocExecution +class AdHocSerializer(serializers.ModelSerializer): + owner = ReadableHiddenField(default=serializers.CurrentUserDefault()) + row_count = serializers.IntegerField(read_only=True) + size = serializers.IntegerField(read_only=True) + + class Meta: + model = AdHoc + fields = ["id", "name", "module", "owner", "row_count", "size", "date_created", "date_updated"] + + +class AdhocListSerializer(AdHocSerializer): + row_count = serializers.IntegerField(read_only=True) + size = serializers.IntegerField(read_only=True) + + class Meta: + model = AdHoc + fields = ["id", "name", "module", "row_count", "size", "args", "owner", "date_created", "date_updated"] + + class AdHocExecutionSerializer(serializers.ModelSerializer): stat = serializers.SerializerMethodField() last_success = serializers.ListField(source='success_hosts') @@ -49,26 +71,6 @@ class AdHocExecutionExcludeResultSerializer(AdHocExecutionSerializer): ] -class AdHocSerializer(serializers.ModelSerializer): - tasks = serializers.ListField() - - class Meta: - model = AdHoc - fields_mini = ['id'] - fields_small = fields_mini + [ - 'tasks', "pattern", "args", "date_created", - ] - fields_fk = ["last_execution"] - fields_m2m = ["assets"] - fields = fields_small + fields_fk + fields_m2m - read_only_fields = [ - 'date_created' - ] - extra_kwargs = { - "become": {'write_only': True} - } - - class AdHocExecutionNestSerializer(serializers.ModelSerializer): last_success = serializers.ListField(source='success_hosts') last_failure = serializers.DictField(source='failed_hosts') @@ -80,38 +82,3 @@ class AdHocExecutionNestSerializer(serializers.ModelSerializer): 'last_success', 'last_failure', 'last_run', 'timedelta', 'is_finished', 'is_success' ) - - -class AdHocDetailSerializer(AdHocSerializer): - latest_execution = AdHocExecutionNestSerializer(allow_null=True) - task_name = serializers.CharField(source='task.name') - - class Meta(AdHocSerializer.Meta): - fields = AdHocSerializer.Meta.fields + [ - 'latest_execution', 'created_by', 'task_name' - ] - - -# class CommandExecutionSerializer(serializers.ModelSerializer): -# result = serializers.JSONField(read_only=True) -# log_url = serializers.SerializerMethodField() -# -# class Meta: -# model = CommandExecution -# fields_mini = ['id'] -# fields_small = fields_mini + [ -# 'command', 'result', 'log_url', -# 'is_finished', 'date_created', 'date_finished' -# ] -# fields_m2m = ['hosts'] -# fields = fields_small + fields_m2m -# read_only_fields = [ -# 'result', 'is_finished', 'log_url', 'date_created', -# 'date_finished' -# ] -# ref_name = 'OpsCommandExecution' -# -# @staticmethod -# def get_log_url(obj): -# return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id}) - diff --git a/apps/ops/serializers/job.py b/apps/ops/serializers/job.py new file mode 100644 index 000000000..64f26dd81 --- /dev/null +++ b/apps/ops/serializers/job.py @@ -0,0 +1,29 @@ +from django.db import transaction +from rest_framework import serializers + +from common.drf.fields import ReadableHiddenField +from ops.models import Job, JobExecution + +_all_ = [] + + +class JobSerializer(serializers.ModelSerializer): + owner = ReadableHiddenField(default=serializers.CurrentUserDefault()) + + class Meta: + model = Job + fields = [ + "id", "name", "instant", "type", "module", "args", "playbook", "assets", "runas_policy", "runas", "owner", + "date_created", + "date_updated" + ] + + +class JobExecutionSerializer(serializers.ModelSerializer): + class Meta: + model = JobExecution + read_only_fields = ["id", "task_id", "timedelta", "time_cost", 'is_finished', 'date_start', 'date_created', + 'is_success', 'task_id', 'short_id'] + fields = read_only_fields + [ + "job" + ] diff --git a/apps/ops/serializers/playbook.py b/apps/ops/serializers/playbook.py new file mode 100644 index 000000000..7ca165501 --- /dev/null +++ b/apps/ops/serializers/playbook.py @@ -0,0 +1,28 @@ +import os + +from rest_framework import serializers + +from common.drf.fields import ReadableHiddenField +from ops.models import Playbook + + +def parse_playbook_name(path): + file_name = os.path.split(path)[-1] + return file_name.split(".")[-2] + + +class PlaybookSerializer(serializers.ModelSerializer): + owner = ReadableHiddenField(default=serializers.CurrentUserDefault()) + + def create(self, validated_data): + name = validated_data.get('name') + if not name: + path = validated_data.get('path').name + validated_data['name'] = parse_playbook_name(path) + return super().create(validated_data) + + class Meta: + model = Playbook + fields = [ + "id", "name", "path", "date_created", "owner", "date_updated" + ] diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 97868a884..e802970c7 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -2,6 +2,7 @@ import os import random import subprocess +import time from django.conf import settings from celery import shared_task, subtask @@ -21,7 +22,7 @@ from .celery.utils import ( create_or_update_celery_periodic_tasks, get_celery_periodic_task, disable_celery_periodic_task, delete_celery_periodic_task ) -from .models import CeleryTaskExecution, AdHoc, Playbook +from .models import CeleryTaskExecution, Playbook, Job, JobExecution from .notifications import ServerPerformanceCheckUtil logger = get_logger(__file__) @@ -31,6 +32,33 @@ def rerun_task(): pass +@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task")) +def run_ops_job(job_id, **kwargs): + job = get_object_or_none(Job, id=job_id) + execution = job.create_execution() + try: + execution.start() + except SoftTimeLimitExceeded: + execution.set_error('Run timeout') + logger.error("Run adhoc timeout") + except Exception as e: + execution.set_error(e) + logger.error("Start adhoc execution error: {}".format(e)) + + +@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task execution")) +def run_ops_job_executions(execution_id, **kwargs): + execution = get_object_or_none(JobExecution, id=execution_id) + try: + execution.start() + except SoftTimeLimitExceeded: + execution.set_error('Run timeout') + logger.error("Run adhoc timeout") + except Exception as e: + execution.set_error(e) + logger.error("Start adhoc execution error: {}".format(e)) + + @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task")) def run_adhoc(tid, **kwargs): """ @@ -156,18 +184,23 @@ def hello(name, callback=None): return gettext("Hello") -@shared_task(verbose_name="Hello Error", comment="an test shared task error") +@shared_task(verbose_name=_("Hello Error"), comment="an test shared task error") def hello_error(): raise Exception("must be error") -@shared_task(verbose_name="Hello Random", comment="some time error and some time success") +@shared_task(verbose_name=_("Hello Random"), comment="some time error and some time success") def hello_random(): i = random.randint(0, 1) if i == 1: raise Exception("must be error") +@shared_task(verbose_name="Hello Running", comment="an task running 1m") +def hello_running(sec=60): + time.sleep(sec) + + @shared_task def hello_callback(result): print(result) diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index 1edbd16e7..a8b71734f 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals from django.urls import path from rest_framework.routers import DefaultRouter from rest_framework_bulk.routes import BulkRouter -from rest_framework_nested import routers from .. import api @@ -13,23 +12,25 @@ app_name = "ops" router = DefaultRouter() bulk_router = BulkRouter() -router.register(r'adhoc', api.AdHocViewSet, 'adhoc') -router.register(r'adhoc-executions', api.AdHocExecutionViewSet, 'execution') +router.register(r'adhocs', api.AdHocViewSet, 'adhoc') +router.register(r'playbooks', api.PlaybookViewSet, 'playbook') +router.register(r'jobs', api.JobViewSet, 'job') +router.register(r'job-executions', api.JobExecutionViewSet, 'job-execution') + router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task') router.register(r'tasks', api.CeleryTaskViewSet, 'task') - -task_router = routers.NestedDefaultRouter(router, r'tasks', lookup='task') -task_router.register(r'executions', api.CeleryTaskExecutionViewSet, 'task-execution') +router.register(r'task-executions', api.CeleryTaskExecutionViewSet, 'task-executions') urlpatterns = [ + path('ansible/job-execution//log/', api.AnsibleTaskLogApi.as_view(), name='job-execution-log'), + path('celery/task//task-execution//log/', api.CeleryTaskExecutionLogApi.as_view(), name='celery-task-execution-log'), path('celery/task//task-execution//result/', api.CeleryResultApi.as_view(), name='celery-task-execution-result'), - path('ansible/task-execution//log/', api.AnsibleTaskLogApi.as_view(), name='ansible-task-log'), ] -urlpatterns += (router.urls + bulk_router.urls + task_router.urls) +urlpatterns += (router.urls + bulk_router.urls) diff --git a/apps/perms/migrations/0032_auto_20221111_1919.py b/apps/perms/migrations/0032_auto_20221111_1919.py new file mode 100644 index 000000000..3f3c56533 --- /dev/null +++ b/apps/perms/migrations/0032_auto_20221111_1919.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.14 on 2022-11-11 11:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0111_alter_automationexecution_status'), + ('perms', '0031_auto_20220816_1600'), + ] + + operations = [ + migrations.CreateModel( + name='PermedAccount', + fields=[ + ], + options={ + 'verbose_name': 'Permed account', + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('assets.account',), + ), + migrations.AlterField( + model_name='assetpermission', + name='actions', + field=models.IntegerField(default=0, verbose_name='Actions'), + ), + ] diff --git a/apps/tickets/migrations/0022_alter_applyassetticket_apply_actions.py b/apps/tickets/migrations/0022_alter_applyassetticket_apply_actions.py new file mode 100644 index 000000000..96f645e0d --- /dev/null +++ b/apps/tickets/migrations/0022_alter_applyassetticket_apply_actions.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-11 11:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tickets', '0021_auto_20220921_1814'), + ] + + operations = [ + migrations.AlterField( + model_name='applyassetticket', + name='apply_actions', + field=models.IntegerField(default=1, verbose_name='Actions'), + ), + ]