From 55c95c58f61618508ac6d52c75ca102ad1b4a78e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=80=81=E5=B9=BF?= Date: Wed, 18 Dec 2019 15:37:53 +0800 Subject: [PATCH] Add new model to operate log (#3546) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] 添加一下model到operate log, [platform,remoteapppermission,changeauthplan,gatherusertask] * [Bugfix] 修改了返回platform的几个位置，修改了command execution的url * [Update] 优化ops task表结构，避免列表页查询几十次sql, 优化了基础的encryptjsonfield * [Update] 修改adhoc 返回的become字段，避免密码泄露 * [Update] 修改变量名称 --- apps/assets/migrations/0044_platform.py | 3 + apps/assets/models/base.py | 3 +- apps/assets/models/user.py | 3 +- apps/audits/signals_handler.py | 1 + apps/common/fields/form.py | 3 +- apps/common/fields/model.py | 23 +- apps/common/tests.py | 3 +- apps/common/utils/encode.py | 7 +- apps/jumpserver/settings/libs.py | 2 +- apps/ops/api/adhoc.py | 2 + apps/ops/celery/decorator.py | 6 +- apps/ops/celery/utils.py | 3 - .../ops/migrations/0009_auto_20191217_1713.py | 72 +++ .../ops/migrations/0010_auto_20191217_1758.py | 68 +++ apps/ops/models/adhoc.py | 311 +++++------- apps/ops/serializers/adhoc.py | 79 +-- apps/ops/templates/ops/adhoc_detail.html | 2 +- .../ops/command_execution_create.html | 470 +++++++++--------- apps/ops/templates/ops/task_adhoc.html | 10 +- apps/ops/templates/ops/task_detail.html | 16 +- apps/ops/templates/ops/task_list.html | 44 +- apps/ops/urls/view_urls.py | 2 +- apps/ops/views/command.py | 4 +- apps/perms/serializers/user_permission.py | 1 + apps/perms/utils/asset_permission.py | 9 +- apps/settings/models.py | 4 +- apps/templates/_base_asset_tree_list.html | 1 - apps/templates/_nav.html | 2 +- apps/templates/_nav_user.html | 4 +- .../migrations/0017_auto_20191125_0931.py | 3 +- apps/users/models/user.py | 4 +- 31 files changed, 647 insertions(+), 518 deletions(-) create mode 100644 apps/ops/migrations/0009_auto_20191217_1713.py create mode 100644 apps/ops/migrations/0010_auto_20191217_1758.py diff --git a/apps/assets/migrations/0044_platform.py b/apps/assets/migrations/0044_platform.py index f66d00642..8d45a8ee3 100644 --- a/apps/assets/migrations/0044_platform.py +++ b/apps/assets/migrations/0044_platform.py @@ -40,6 +40,9 @@ class Migration(migrations.Migration): ('internal', models.BooleanField(default=False, verbose_name='Internal')), ('comment', models.TextField(blank=True, null=True, verbose_name='Comment')), ], + options={ + 'verbose_name': 'Platform' + } ), migrations.RunPython(create_internal_platform) ] diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index b49813c27..895e0134d 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -11,14 +11,13 @@ from django.utils.translation import ugettext_lazy as _ from django.conf import settings from common.utils import ( - get_signer, ssh_key_string_to_obj, ssh_key_gen, get_logger + signer, ssh_key_string_to_obj, ssh_key_gen, get_logger ) from common.validators import alphanumeric from common import fields from orgs.mixins.models import OrgModelMixin from .utils import private_key_validator, Connectivity -signer = get_signer() logger = get_logger(__file__) diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 443c32981..dd2504f00 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -10,14 +10,13 @@ from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django.core.validators import MinValueValidator, MaxValueValidator -from common.utils import get_signer +from common.utils import signer from .base import AssetUser from .asset import Asset __all__ = ['AdminUser', 'SystemUser'] logger = logging.getLogger(__name__) -signer = get_signer() class AdminUser(AssetUser): diff --git a/apps/audits/signals_handler.py b/apps/audits/signals_handler.py index 772a7fcd5..e7c61313d 100644 --- a/apps/audits/signals_handler.py +++ b/apps/audits/signals_handler.py @@ -27,6 +27,7 @@ MODELS_NEED_RECORD = ( 'User', 'UserGroup', 'Asset', 'Node', 'AdminUser', 'SystemUser', 'Domain', 'Gateway', 'Organization', 'AssetPermission', 'CommandFilter', 'CommandFilterRule', 'License', 'Setting', 'Account', 'SyncInstanceTask', + 'Platform', 'RemoteAppPermission', 'ChangeAuthPlan', 'GatherUserTask', ) diff --git a/apps/common/fields/form.py b/apps/common/fields/form.py index 156e331dd..c4cdc78ad 100644 --- a/apps/common/fields/form.py +++ b/apps/common/fields/form.py @@ -6,9 +6,8 @@ from django import forms from django.utils import six from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from ..utils import get_signer +from ..utils import signer -signer = get_signer() __all__ = [ 'FormDictField', 'FormEncryptCharField', 'FormEncryptDictField', diff --git a/apps/common/fields/model.py b/apps/common/fields/model.py index e1bd4e1e7..ddb61f7c0 100644 --- a/apps/common/fields/model.py +++ b/apps/common/fields/model.py @@ -4,7 +4,7 @@ import json from django.db import models from django.utils.translation import ugettext_lazy as _ -from ..utils import get_signer +from ..utils import signer __all__ = [ @@ -12,8 +12,8 @@ __all__ = [ 'JsonCharField', 'JsonTextField', 'JsonListCharField', 'JsonListTextField', 'JsonDictCharField', 'JsonDictTextField', 'EncryptCharField', 'EncryptTextField', 'EncryptMixin', 'EncryptJsonDictTextField', + 'EncryptJsonDictCharField', ] -signer = get_signer() class JsonMixin: @@ -108,14 +108,24 @@ class JsonTextField(JsonMixin, models.TextField): class EncryptMixin: + """ + EncryptMixin要放在最前面 + """ def from_db_value(self, value, expression, connection, context): - if value is not None: - return signer.unsign(value) - return None + if value is None: + return value + value = signer.unsign(value) + sp = super() + if hasattr(sp, 'from_db_value'): + return sp.from_db_value(value, expression, connection, context) + return value def get_prep_value(self, value): if value is None: return value + sp = super() + if hasattr(sp, 'get_prep_value'): + value = sp.get_prep_value(value) return signer.sign(value) @@ -150,3 +160,6 @@ class EncryptJsonDictTextField(EncryptMixin, JsonDictTextField): pass +class EncryptJsonDictCharField(EncryptMixin, JsonDictCharField): + pass + diff --git a/apps/common/tests.py b/apps/common/tests.py index a9edb8f69..5fd1a7ddb 100644 --- a/apps/common/tests.py +++ b/apps/common/tests.py @@ -2,11 +2,10 @@ from django.test import TestCase # Create your tests here. -from .utils import random_string, get_signer +from .utils import random_string, signer def test_signer_len(): - signer = get_signer() results = {} for i in range(1, 4096): s = random_string(i) diff --git a/apps/common/utils/encode.py b/apps/common/utils/encode.py index 6da79ff48..097e28292 100644 --- a/apps/common/utils/encode.py +++ b/apps/common/utils/encode.py @@ -184,8 +184,11 @@ def encrypt_password(password, salt=None): def get_signer(): - signer = Signer(settings.SECRET_KEY) - return signer + s = Signer(settings.SECRET_KEY) + return s + + +signer = get_signer() def ensure_last_char_is_ascii(data): diff --git a/apps/jumpserver/settings/libs.py b/apps/jumpserver/settings/libs.py index 35de39b33..69b580d66 100644 --- a/apps/jumpserver/settings/libs.py +++ b/apps/jumpserver/settings/libs.py @@ -105,7 +105,7 @@ CELERY_TASK_SERIALIZER = 'pickle' CELERY_RESULT_SERIALIZER = 'pickle' CELERY_RESULT_BACKEND = CELERY_BROKER_URL CELERY_ACCEPT_CONTENT = ['json', 'pickle'] -CELERY_RESULT_EXPIRES = 3600 +CELERY_RESULT_EXPIRES = 600 # CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s' # CELERY_WORKER_LOG_FORMAT = '%(message)s' # CELERY_WORKER_TASK_LOG_FORMAT = '%(task_id)s %(task_name)s %(message)s' diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index 1bf2b4901..38c59d701 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -4,6 +4,7 @@ from django.shortcuts import get_object_or_404 from rest_framework import viewsets, generics from rest_framework.views import Response +from django.db.models import Count, Q from common.permissions import IsOrgAdmin from common.serializers import CeleryTaskSerializer @@ -31,6 +32,7 @@ class TaskViewSet(viewsets.ModelViewSet): queryset = queryset.filter(created_by=current_org.id) else: queryset = queryset.filter(created_by='') + queryset = queryset.select_related('latest_history') return queryset diff --git a/apps/ops/celery/decorator.py b/apps/ops/celery/decorator.py index c60c02b6a..971d5b863 100644 --- a/apps/ops/celery/decorator.py +++ b/apps/ops/celery/decorator.py @@ -33,11 +33,14 @@ def get_after_app_ready_tasks(): def register_as_period_task( crontab=None, interval=None, name=None, + args=(), kwargs=None, description=''): """ Warning: Task must be have not any args and kwargs :param crontab: "* * * * *" :param interval: 60*60*60 + :param args: () + :param kwargs: {} :param description: " :param name: "" :return: @@ -58,7 +61,8 @@ def register_as_period_task( 'task': task, 'interval': interval, 'crontab': crontab, - 'args': (), + 'args': args, + 'kwargs': kwargs if kwargs else {}, 'enabled': True, 'description': description } diff --git a/apps/ops/celery/utils.py b/apps/ops/celery/utils.py index 2c46f225f..55ce4a9d1 100644 --- a/apps/ops/celery/utils.py +++ b/apps/ops/celery/utils.py @@ -74,8 +74,6 @@ def create_or_update_celery_periodic_tasks(tasks): kwargs=json.dumps(detail.get('kwargs', {})), description=detail.get('description') or '' ) - print(defaults) - task = PeriodicTask.objects.update_or_create( defaults=defaults, name=name, ) @@ -101,4 +99,3 @@ def get_celery_task_log_path(task_id): path = os.path.join(settings.CELERY_LOG_DIR, rel_path) os.makedirs(os.path.dirname(path), exist_ok=True) return path - diff --git a/apps/ops/migrations/0009_auto_20191217_1713.py b/apps/ops/migrations/0009_auto_20191217_1713.py new file mode 100644 index 000000000..75ba632d8 --- /dev/null +++ b/apps/ops/migrations/0009_auto_20191217_1713.py @@ -0,0 +1,72 @@ +# Generated by Django 2.2.7 on 2019-12-17 09:13 + +from django.db import migrations, models +import django.db.models.deletion +from django.core.exceptions import ObjectDoesNotExist + + +def migrate_task_data(apps, schema_editor): + task_model = apps.get_model("ops", "Task") + db_alias = schema_editor.connection.alias + tasks = task_model.objects.using(db_alias).all() + for task in tasks: + try: + latest_history = task.history.latest() + except ObjectDoesNotExist: + latest_history = None + try: + latest_adhoc = task.adhoc.latest() + except ObjectDoesNotExist: + latest_adhoc = None + if latest_history and latest_history.adhoc: + latest_history.hosts_amount = latest_history.adhoc.hosts.count() + latest_history.save() + total_run_amount = task.history.all().count() + success_run_amount = task.history.filter(is_success=True).count() + task.latest_history = latest_history + task.latest_adhoc = latest_adhoc + task.total_run_amount = total_run_amount + task.success_run_amount = success_run_amount + task.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0008_auto_20190919_2100'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='latest_adhoc', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='task_latest', to='ops.AdHoc'), + ), + migrations.AddField( + model_name='task', + name='latest_history', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='task_latest', to='ops.AdHocRunHistory'), + ), + migrations.AddField( + model_name='task', + name='success_run_amount', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='task', + name='total_run_amount', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='adhocrunhistory', + name='hosts_amount', + field=models.IntegerField(default=0, verbose_name='Host amount'), + ), + migrations.AddField( + model_name='adhocrunhistory', + name='task_display', + field=models.CharField(blank=True, default='', max_length=128, + verbose_name='Task display'), + ), + migrations.RunPython(migrate_task_data), + ] diff --git a/apps/ops/migrations/0010_auto_20191217_1758.py b/apps/ops/migrations/0010_auto_20191217_1758.py new file mode 100644 index 000000000..7e999a33b --- /dev/null +++ b/apps/ops/migrations/0010_auto_20191217_1758.py @@ -0,0 +1,68 @@ +# Generated by Django 2.2.7 on 2019-12-17 09:58 + +import common.fields.model +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0009_auto_20191217_1713'), + ] + + operations = [ + migrations.RemoveField( + model_name='adhoc', + name='_hosts', + ), + migrations.AlterField( + model_name='adhoc', + name='_become', + field=common.fields.model.EncryptJsonDictCharField(blank=True, default='', max_length=1024, verbose_name='Become'), + ), + migrations.AlterField( + model_name='adhoc', + name='_options', + field=common.fields.model.JsonDictCharField(default='', max_length=1024, verbose_name='Options'), + ), + migrations.AlterField( + model_name='adhoc', + name='_tasks', + field=common.fields.model.JsonListTextField(verbose_name='Tasks'), + ), + migrations.RenameField( + model_name='adhoc', + old_name='_become', + new_name='become', + ), + migrations.RenameField( + model_name='adhoc', + old_name='_options', + new_name='options', + ), + migrations.RenameField( + model_name='adhoc', + old_name='_tasks', + new_name='tasks', + ), + migrations.AlterField( + model_name='adhocrunhistory', + name='_result', + field=common.fields.model.JsonDictTextField(blank=True, null=True, verbose_name='Adhoc raw result'), + ), + migrations.AlterField( + model_name='adhocrunhistory', + name='_summary', + field=common.fields.model.JsonDictTextField(blank=True, null=True, verbose_name='Adhoc result summary'), + ), + migrations.RenameField( + model_name='adhocrunhistory', + old_name='_result', + new_name='result', + ), + migrations.RenameField( + model_name='adhocrunhistory', + old_name='_summary', + new_name='summary', + ), + ] diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index 3eebb6636..3e0220b68 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -1,6 +1,5 @@ # ~*~ coding: utf-8 ~*~ -import json import uuid import os import time @@ -13,11 +12,16 @@ from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from django_celery_beat.models import PeriodicTask -from common.utils import get_signer, get_logger, lazyproperty -from orgs.utils import set_to_root_org -from ..celery.utils import delete_celery_periodic_task, \ - create_or_update_celery_periodic_tasks, \ +from common.utils import get_logger, lazyproperty +from common.fields.model import ( + JsonListTextField, JsonDictCharField, EncryptJsonDictCharField, + JsonDictTextField, +) +from orgs.utils import set_to_root_org, get_current_org, set_current_org +from ..celery.utils import ( + delete_celery_periodic_task, create_or_update_celery_periodic_tasks, disable_celery_periodic_task +) from ..ansible import AdHocRunner, AnsibleError from ..inventory import JMSInventory @@ -25,7 +29,6 @@ __all__ = ["Task", "AdHoc", "AdHocRunHistory"] logger = get_logger(__file__) -signer = get_signer() class Task(models.Model): @@ -44,14 +47,17 @@ class Task(models.Model): created_by = models.CharField(max_length=128, blank=True, default='') date_created = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name=_("Date created")) date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) - __latest_adhoc = None + latest_adhoc = models.ForeignKey('ops.AdHoc', on_delete=models.SET_NULL, null=True, related_name='task_latest') + latest_history = models.ForeignKey('ops.AdHocRunHistory', on_delete=models.SET_NULL, null=True, related_name='task_latest') + total_run_amount = models.IntegerField(default=0) + success_run_amount = models.IntegerField(default=0) _ignore_auto_created_by = True @property def short_id(self): return str(self.id).split('-')[-1] - @property + @lazyproperty def versions(self): return self.adhoc.all().count() @@ -78,73 +84,67 @@ class Task(models.Model): @property def assets_amount(self): - return self.latest_adhoc.hosts.count() - - @lazyproperty - def latest_adhoc(self): - return self.get_latest_adhoc() - - @lazyproperty - def latest_history(self): - try: - return self.history.all().latest() - except AdHocRunHistory.DoesNotExist: - return None + if self.latest_history: + return self.latest_history.hosts_amount + return 0 def get_latest_adhoc(self): + if self.latest_adhoc: + return self.latest_adhoc try: - return self.adhoc.all().latest() + adhoc = self.adhoc.all().latest() + self.latest_adhoc = adhoc + self.save() + return adhoc except AdHoc.DoesNotExist: return None @property def history_summary(self): - history = self.get_run_history() - total = len(history) - success = len([history for history in history if history.is_success]) - failed = len([history for history in history if not history.is_success]) + total = self.total_run_amount + success = self.success_run_amount + failed = total - success return {'total': total, 'success': success, 'failed': failed} def get_run_history(self): return self.history.all() - def run(self, record=True): - set_to_root_org() - if self.latest_adhoc: - return self.latest_adhoc.run(record=record) + def run(self): + latest_adhoc = self.get_latest_adhoc() + if latest_adhoc: + return latest_adhoc.run() else: return {'error': 'No adhoc'} - def save(self, force_insert=False, force_update=False, using=None, - update_fields=None): + def register_as_period_task(self): from ..tasks import run_ansible_task - super().save( - force_insert=force_insert, force_update=force_update, - using=using, update_fields=update_fields, - ) + interval = None + crontab = None - if self.is_periodic: - interval = None - crontab = None + if self.interval: + interval = self.interval + elif self.crontab: + crontab = self.crontab - if self.interval: - interval = self.interval - elif self.crontab: - crontab = self.crontab - - tasks = { - self.__str__(): { - "task": run_ansible_task.name, - "interval": interval, - "crontab": crontab, - "args": (str(self.id),), - "kwargs": {"callback": self.callback}, - "enabled": True, - } + tasks = { + self.__str__(): { + "task": run_ansible_task.name, + "interval": interval, + "crontab": crontab, + "args": (str(self.id),), + "kwargs": {"callback": self.callback}, + "enabled": True, } - create_or_update_celery_periodic_tasks(tasks) + } + create_or_update_celery_periodic_tasks(tasks) + + def save(self, **kwargs): + instance = super().save(**kwargs) + if self.is_periodic: + self.register_as_period_task() else: disable_celery_periodic_task(self.__str__()) + return instance def delete(self, using=None, keep_parents=False): super().delete(using=using, keep_parents=keep_parents) @@ -153,7 +153,7 @@ class Task(models.Model): @property def schedule(self): try: - return PeriodicTask.objects.get(name=self.name) + return PeriodicTask.objects.get(name=str(self)) except PeriodicTask.DoesNotExist: return None @@ -172,7 +172,6 @@ class AdHoc(models.Model): task: A task reference _tasks: [{'name': 'task_name', 'action': {'module': '', 'args': ''}, 'other..': ''}, ] _options: ansible options, more see ops.ansible.runner.Options - _hosts: ["hostname1", "hostname2"], hostname must be unique key of cmdb run_as_admin: if true, then need get every host admin user run it, because every host may be have different admin user, so we choise host level run_as: username(Add the uniform AssetUserManager and change it to username) _become: May be using become [sudo, su] options. {method: "sudo", user: "user", pass: "pass"] @@ -180,31 +179,16 @@ class AdHoc(models.Model): """ id = models.UUIDField(default=uuid.uuid4, primary_key=True) task = models.ForeignKey(Task, related_name='adhoc', on_delete=models.CASCADE) - _tasks = models.TextField(verbose_name=_('Tasks')) + tasks = JsonListTextField(verbose_name=_('Tasks')) pattern = models.CharField(max_length=64, default='{}', verbose_name=_('Pattern')) - _options = models.CharField(max_length=1024, default='', verbose_name=_('Options')) - _hosts = models.TextField(blank=True, verbose_name=_('Hosts')) # ['hostname1', 'hostname2'] + options = JsonDictCharField(max_length=1024, default='', verbose_name=_('Options')) hosts = models.ManyToManyField('assets.Asset', verbose_name=_("Host")) run_as_admin = models.BooleanField(default=False, verbose_name=_('Run as admin')) run_as = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Username')) - _become = models.CharField(max_length=1024, default='', blank=True, verbose_name=_("Become")) + become = EncryptJsonDictCharField(max_length=1024, default='', blank=True, verbose_name=_("Become")) created_by = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Create by')) date_created = models.DateTimeField(auto_now_add=True, db_index=True) - @property - def tasks(self): - try: - return json.loads(self._tasks) - except: - return [] - - @tasks.setter - def tasks(self, item): - if item and isinstance(item, list): - self._tasks = json.dumps(item) - else: - raise SyntaxError('Tasks should be a list: {}'.format(item)) - @property def inventory(self): if self.become: @@ -223,97 +207,22 @@ class AdHoc(models.Model): return inventory @property - def become(self): - if self._become: - return json.loads(signer.unsign(self._become)) - else: - return {} + def become_display(self): + if self.become: + return self.become.get("user", "") + return "" - def run(self, record=True): - set_to_root_org() - if record: - return self._run_and_record() - else: - return self._run_only() - - def _run_and_record(self): + def run(self): try: hid = current_task.request.id except AttributeError: hid = str(uuid.uuid4()) - history = AdHocRunHistory(id=hid, adhoc=self, task=self.task) + history = AdHocRunHistory( + id=hid, adhoc=self, task=self.task, + task_display=str(self.task) + ) history.save() - time_start = time.time() - date_start = timezone.now() - is_success = False - summary = {} - raw = '' - - try: - date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - print(_("{} Start task: {}").format(date_start_s, self.task.name)) - raw, summary = self._run_only() - is_success = summary.get('success', False) - except Exception as e: - logger.error(e, exc_info=True) - raw = {"dark": {"all": str(e)}, "contacted": []} - finally: - date_end = timezone.now() - date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S') - print(_("{} Task finish").format(date_end_s)) - print('.\n\n.') - try: - summary_text = json.dumps(summary) - except json.JSONDecodeError: - summary_text = '{}' - AdHocRunHistory.objects.filter(id=history.id).update( - date_start=date_start, - is_finished=True, - is_success=is_success, - date_finished=timezone.now(), - timedelta=time.time() - time_start, - _summary=summary_text - ) - return raw, summary - - def _run_only(self): - Task.objects.filter(id=self.task.id).update(date_updated=timezone.now()) - runner = AdHocRunner(self.inventory, options=self.options) - try: - result = runner.run( - self.tasks, - self.pattern, - self.task.name, - ) - return result.results_raw, result.results_summary - except AnsibleError as e: - logger.warn("Failed run adhoc {}, {}".format(self.task.name, e)) - pass - - @become.setter - def become(self, item): - """ - :param item: { - method: "sudo", - user: "user", - pass: "pass", - } - :return: - """ - # self._become = signer.sign(json.dumps(item)).decode('utf-8') - self._become = signer.sign(json.dumps(item)) - - @property - def options(self): - if self._options: - _options = json.loads(self._options) - if isinstance(_options, dict): - return _options - return {} - - @options.setter - def options(self, item): - self._options = json.dumps(item) + return history.start() @property def short_id(self): @@ -328,6 +237,8 @@ class AdHoc(models.Model): def save(self, **kwargs): instance = super().save(**kwargs) + self.task.latest_adhoc = instance + self.task.save() return instance def __str__(self): @@ -356,19 +267,25 @@ class AdHocRunHistory(models.Model): """ id = models.UUIDField(default=uuid.uuid4, primary_key=True) task = models.ForeignKey(Task, related_name='history', on_delete=models.SET_NULL, null=True) + task_display = models.CharField(max_length=128, blank=True, default='', verbose_name=_("Task display")) + hosts_amount = models.IntegerField(default=0, verbose_name=_("Host amount")) adhoc = models.ForeignKey(AdHoc, related_name='history', on_delete=models.SET_NULL, null=True) date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time')) date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time')) timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True) is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) is_success = models.BooleanField(default=False, verbose_name=_('Is success')) - _result = models.TextField(blank=True, null=True, verbose_name=_('Adhoc raw result')) - _summary = models.TextField(blank=True, null=True, verbose_name=_('Adhoc result summary')) + result = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc raw result')) + summary = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc result summary')) @property def short_id(self): return str(self.id).split('-')[-1] + @property + def adhoc_short_id(self): + return str(self.adhoc_id).split('-')[-1] + @property def log_path(self): dt = datetime.datetime.now().strftime('%Y-%m-%d') @@ -377,30 +294,58 @@ class AdHocRunHistory(models.Model): os.makedirs(log_dir) return os.path.join(log_dir, str(self.id) + '.log') - @property - def result(self): - if self._result: - return json.loads(self._result) - else: - return {} - - @result.setter - def result(self, item): - self._result = json.dumps(item) - - @property - def summary(self): - if self._summary: - return json.loads(self._summary) - else: - return {"ok": {}, "dark": {}} - - @summary.setter - def summary(self, item): + def start_runner(self): + runner = AdHocRunner(self.adhoc.inventory, options=self.adhoc.options) try: - self._summary = json.dumps(item) - except json.JSONDecodeError: - self._summary = json.dumps({}) + result = runner.run( + self.adhoc.tasks, + self.adhoc.pattern, + self.task.name, + ) + return result.results_raw, result.results_summary + except AnsibleError as e: + logger.warn("Failed run adhoc {}, {}".format(self.task.name, e)) + return {}, {} + + def start(self): + self.task.latest_history = self + self.task.save() + current_org = get_current_org() + set_to_root_org() + time_start = time.time() + date_start = timezone.now() + is_success = False + summary = {} + raw = '' + + try: + date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + print(_("{} Start task: {}").format(date_start_s, self.task.name)) + raw, summary = self.start_runner() + is_success = summary.get('success', False) + except Exception as e: + logger.error(e, exc_info=True) + raw = {"dark": {"all": str(e)}, "contacted": []} + finally: + date_end = timezone.now() + date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S') + print(_("{} Task finish").format(date_end_s)) + print('.\n\n.') + task = Task.objects.get(id=self.task_id) + task.total_run_amount = models.F('total_run_amount') + 1 + if is_success: + task.success_run_amount = models.F('success_run_amount') + 1 + task.save() + AdHocRunHistory.objects.filter(id=self.id).update( + date_start=date_start, + is_finished=True, + is_success=is_success, + date_finished=timezone.now(), + timedelta=time.time() - time_start, + summary=summary + ) + set_current_org(current_org) + return raw, summary @property def success_hosts(self): diff --git a/apps/ops/serializers/adhoc.py b/apps/ops/serializers/adhoc.py index a737af1c4..6a3464654 100644 --- a/apps/ops/serializers/adhoc.py +++ b/apps/ops/serializers/adhoc.py @@ -6,59 +6,74 @@ from django.shortcuts import reverse from ..models import Task, AdHoc, AdHocRunHistory, CommandExecution -class TaskSerializer(serializers.ModelSerializer): - class Meta: - model = Task - fields = [ - 'id', 'name', 'interval', 'crontab', 'is_periodic', - 'is_deleted', 'comment', 'created_by', 'date_created', - 'versions', 'is_success', 'timedelta', 'assets_amount', - 'date_updated', 'history_summary', - ] - - -class AdHocSerializer(serializers.ModelSerializer): - class Meta: - model = AdHoc - exclude = ('_tasks', '_options', '_hosts', '_become') - - def get_field_names(self, declared_fields, info): - fields = super().get_field_names(declared_fields, info) - fields.extend(['tasks', 'options', 'hosts', 'become', 'short_id']) - return fields - - class AdHocRunHistorySerializer(serializers.ModelSerializer): - task = serializers.SerializerMethodField() - adhoc_short_id = serializers.SerializerMethodField() stat = serializers.SerializerMethodField() class Meta: model = AdHocRunHistory - exclude = ('_result', '_summary') - - @staticmethod - def get_adhoc_short_id(obj): - return obj.adhoc.short_id + fields = '__all__' @staticmethod def get_task(obj): - return obj.adhoc.task.id + return obj.task.id @staticmethod def get_stat(obj): return { - "total": obj.adhoc.hosts.count(), + "total": obj.hosts_amount, "success": len(obj.summary.get("contacted", [])), "failed": len(obj.summary.get("dark", [])), } def get_field_names(self, declared_fields, info): fields = super().get_field_names(declared_fields, info) - fields.extend(['summary', 'short_id']) + fields.extend(['short_id', 'adhoc_short_id']) return fields +class AdHocRunHistoryExcludeResultSerializer(AdHocRunHistorySerializer): + def get_field_names(self, declared_fields, info): + fields = super().get_field_names(declared_fields, info) + fields = [i for i in fields if i not in ['result', 'summary']] + return fields + + +class TaskSerializer(serializers.ModelSerializer): + latest_history = AdHocRunHistoryExcludeResultSerializer(read_only=True) + + class Meta: + model = Task + fields = [ + 'id', 'name', 'interval', 'crontab', 'is_periodic', + 'is_deleted', 'comment', 'created_by', 'date_created', + 'date_updated', 'latest_history', + ] + read_only_fields = [ + 'is_deleted', 'created_by', 'date_created', 'date_updated', + 'latest_adhoc', 'latest_history', 'total_run_amount', + 'success_run_amount', + ] + + +class AdHocSerializer(serializers.ModelSerializer): + become_display = serializers.ReadOnlyField() + + class Meta: + model = AdHoc + fields = [ + "id", "task", 'tasks', "pattern", "options", + "hosts", "run_as_admin", "run_as", "become", + "created_by", "date_created", "short_id", + "become_display", + ] + read_only_fields = [ + 'created_by', 'date_created' + ] + extra_kwargs = { + "become": {'write_only': True} + } + + class CommandExecutionSerializer(serializers.ModelSerializer): result = serializers.JSONField(read_only=True) log_url = serializers.SerializerMethodField() diff --git a/apps/ops/templates/ops/adhoc_detail.html b/apps/ops/templates/ops/adhoc_detail.html index 55c0960a2..777207da4 100644 --- a/apps/ops/templates/ops/adhoc_detail.html +++ b/apps/ops/templates/ops/adhoc_detail.html @@ -78,7 +78,7 @@ {% endif %} {% trans 'Become' %} - {{ object.become.user }} + {{ object.become_display }} {% trans 'Created by' %} diff --git a/apps/ops/templates/ops/command_execution_create.html b/apps/ops/templates/ops/command_execution_create.html index f89dc98f6..95ce617ca 100644 --- a/apps/ops/templates/ops/command_execution_create.html +++ b/apps/ops/templates/ops/command_execution_create.html @@ -4,17 +4,12 @@ {% load bootstrap3 %} {% block custom_head_css_js %} - + - - - - + + + + @@ -37,18 +32,18 @@ overflow: auto; } - body ::-webkit-scrollbar-track { + #term ::-webkit-scrollbar-track { -webkit-box-shadow: inset 0 0 2px rgba(0, 0, 0, 0.3); background-color: #272323; border-radius: 6px; } - body ::-webkit-scrollbar { + #term ::-webkit-scrollbar { width: 8px; height: 8px; } - body ::-webkit-scrollbar-thumb { + #term ::-webkit-scrollbar-thumb { background-color: #494141; border-radius: 6px; } @@ -58,8 +53,8 @@ {% block content %}
-
-
+
+
@@ -73,37 +68,30 @@
-
+
-
+
-
+
-
- +
+
- {% for s in system_users %} {% endfor %} - +
@@ -114,230 +102,236 @@ {% endblock %} {% block custom_foot_js %} - + }); + editor = CodeMirror.fromTextArea(document.getElementById("command-text"), { + lineNumbers: true, + lineWrapping: true, + mode: "shell" + }); + editor.setSize(600, 100); + var charWidth = editor.defaultCharWidth(), basePadding = 4; + editor.on("renderLine", function (cm, line, elt) { + var off = CodeMirror.countColumn(line.text, null, cm.getOption("tabSize")) * charWidth; + elt.style.textIndent = "-" + off + "px"; + elt.style.paddingLeft = (basePadding + off) + "px"; + }); + editor.refresh(); + initTree(); + initResultTerminal(); + }).on('click', '.btn-execute', function () { + execute() + }) + {% endblock %} diff --git a/apps/ops/templates/ops/task_adhoc.html b/apps/ops/templates/ops/task_adhoc.html index 9e544d6c5..49035c725 100644 --- a/apps/ops/templates/ops/task_adhoc.html +++ b/apps/ops/templates/ops/task_adhoc.html @@ -103,7 +103,7 @@ $(document).ready(function () { if (!cellData) { $(td).html("") } else { - $(td).html(cellData.user) + $(td).html(cellData) } }}, {targets: 6, createdCell: function (td, cellData) { @@ -118,8 +118,12 @@ $(document).ready(function () { }} ], ajax_url: '{% url "api-ops:adhoc-list" %}?task={{ object.pk }}', - columns: [{data: function(){return ""}}, {data: "short_id" }, {data: "hosts", orderable:false}, {data: "pattern", orderable:false}, - {data: "run_as"}, {data: "become", orderable:false}, {data: "date_created"}, {data: "id", orderable:false}] + columns: [ + {data: function(){return ""}}, {data: "short_id"}, + {data: "hosts", orderable:false}, {data: "pattern", orderable:false}, + {data: "run_as"}, {data: "become_display", orderable:false}, + {data: "date_created"}, {data: "id", orderable:false} + ] }; jumpserver.initDataTable(options); }).on('click', '.celery-task-log', function () { diff --git a/apps/ops/templates/ops/task_detail.html b/apps/ops/templates/ops/task_detail.html index 07036b58a..d2e46cc35 100644 --- a/apps/ops/templates/ops/task_detail.html +++ b/apps/ops/templates/ops/task_detail.html @@ -80,11 +80,23 @@ {% trans 'Is finished' %}: - {{ object.latest_history.is_finished|yesno:"Yes,No,Unkown" }} + + {% if object.latest_history.is_finished %} + {% trans 'Yes' %} + {% else %} + {% trans 'No' %} + {% endif %} + {% trans 'Is success ' %}: - {{ object.latest_history.is_success|yesno:"Yes,No,Unkown" }} + + {% if object.latest_history.is_success %} + {% trans 'Yes' %} + {% else %} + {% trans 'No' %} + {% endif %} + {% trans 'Contents' %}: diff --git a/apps/ops/templates/ops/task_list.html b/apps/ops/templates/ops/task_list.html index 83da62071..0d2095d78 100644 --- a/apps/ops/templates/ops/task_list.html +++ b/apps/ops/templates/ops/task_list.html @@ -10,7 +10,6 @@ {% trans 'Name' %} {% trans 'Run times' %} - {% trans 'Versions' %} {% trans 'Hosts' %} {% trans 'Success' %} {% trans 'Date' %} @@ -36,34 +35,40 @@ $(document).ready(function () { $(td).html(innerHtml); }}, {targets: 2, createdCell: function (td, cellData) { + var summary = cellData ? cellData.stat : {failed: 0, success: 0, total: 0}; var innerHtml = 'failed/success/total'; - if (cellData) { - innerHtml = innerHtml.replace('failed', cellData.failed) - .replace('success', cellData.success) - .replace('total', cellData.total); - $(td).html(innerHtml); - } else { - $(td).html('') - } + innerHtml = innerHtml.replace('failed', summary.failed) + .replace('success', summary.success) + .replace('total', summary.total); + $(td).html(innerHtml); }}, - {targets: 5, createdCell: function (td, cellData) { + {targets: 3, createdCell: function (td, cellData) { + var hostsAmount = cellData ? cellData.hosts_amount : 0; + $(td).html(hostsAmount) + }}, + {targets: 4, createdCell: function (td, cellData) { var successBtn = ''; var failedBtn = ''; - if (cellData) { + if (cellData && cellData.is_success) { $(td).html(successBtn) } else { $(td).html(failedBtn) } }}, - {targets: 6, createdCell: function (td, cellData) { - $(td).html(toSafeLocalDateStr(cellData)); + {targets: 5, createdCell: function (td, cellData) { + if (cellData) { + $(td).html(toSafeLocalDateStr(cellData.date_start)); + } else { + $(td).html(''); + } }}, - {targets: 7, createdCell: function (td, cellData) { + {targets: 6, createdCell: function (td, cellData) { + cellData = cellData ? cellData.timedelta : 0; var delta = readableSecond(cellData); $(td).html(delta); }}, { - targets: 8, + targets: 7, createdCell: function (td, cellData, rowData) { var runBtn = '{% trans "Run" %} '.replace('ID', cellData); var delBtn = '{% trans "Delete" %}'.replace('ID', cellData); @@ -73,10 +78,11 @@ $(document).ready(function () { ], ajax_url: '{% url "api-ops:task-list" %}', columns: [ - {data: "id"}, {data: "name", className: "text-left"}, {data: "history_summary", orderable: false}, - {data: "versions", orderable: false}, {data: "assets_amount", orderable: false}, - {data: "is_success", orderable: false}, {data: "date_updated"}, - {data: "timedelta", orderable:false}, {data: "id", orderable: false}, + {data: "id"}, {data: "name", className: "text-left"}, + {data: "latest_history", orderable: false}, + {data: "latest_history", orderable: false}, + {data: "latest_history", orderable: false}, {data: "latest_history"}, + {data: "latest_history", orderable:false}, {data: "id", orderable: false}, ], order: [], op_html: $('#actions').html() diff --git a/apps/ops/urls/view_urls.py b/apps/ops/urls/view_urls.py index f8428a667..13759c3f2 100644 --- a/apps/ops/urls/view_urls.py +++ b/apps/ops/urls/view_urls.py @@ -20,5 +20,5 @@ urlpatterns = [ path('celery/task//log/', views.CeleryTaskLogView.as_view(), name='celery-task-log'), path('command-execution/', views.CommandExecutionListView.as_view(), name='command-execution-list'), - path('command-execution/start/', views.CommandExecutionStartView.as_view(), name='command-execution-start'), + path('command-execution/create/', views.CommandExecutionCreateView.as_view(), name='command-execution-create'), ] diff --git a/apps/ops/views/command.py b/apps/ops/views/command.py index d9b1f61be..87e0528c6 100644 --- a/apps/ops/views/command.py +++ b/apps/ops/views/command.py @@ -15,7 +15,7 @@ from ..forms import CommandExecutionForm __all__ = [ - 'CommandExecutionListView', 'CommandExecutionStartView' + 'CommandExecutionListView', 'CommandExecutionCreateView' ] @@ -55,7 +55,7 @@ class CommandExecutionListView(PermissionsMixin, DatetimeSearchMixin, ListView): return super().get_context_data(**kwargs) -class CommandExecutionStartView(PermissionsMixin, TemplateView): +class CommandExecutionCreateView(PermissionsMixin, TemplateView): template_name = 'ops/command_execution_create.html' form_class = CommandExecutionForm permission_classes = [IsValidUser] diff --git a/apps/perms/serializers/user_permission.py b/apps/perms/serializers/user_permission.py index d83c15b6f..7ab4cf66b 100644 --- a/apps/perms/serializers/user_permission.py +++ b/apps/perms/serializers/user_permission.py @@ -46,6 +46,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer): 被授权资产的数据结构 """ protocols = ProtocolsField(label=_('Protocols'), required=False, read_only=True) + platform = serializers.ReadOnlyField(source='platform_base') class Meta: model = Asset diff --git a/apps/perms/utils/asset_permission.py b/apps/perms/utils/asset_permission.py index f71f83663..27c839d59 100644 --- a/apps/perms/utils/asset_permission.py +++ b/apps/perms/utils/asset_permission.py @@ -437,7 +437,7 @@ def sort_assets(assets, order_by='hostname', reverse=False): class ParserNode: nodes_only_fields = ("key", "value", "id") - assets_only_fields = ("platform", "hostname", "id", "ip", "protocols") + assets_only_fields = ("hostname", "id", "ip", "protocols", "org_id") system_users_only_fields = ( "id", "name", "username", "protocol", "priority", "login_mode", ) @@ -445,7 +445,6 @@ class ParserNode: @staticmethod def parse_node_to_tree_node(node): name = '{} ({})'.format(node.value, node.assets_amount) - # name = node.value data = { 'id': node.key, 'name': name, @@ -468,7 +467,7 @@ class ParserNode: @staticmethod def parse_asset_to_tree_node(node, asset): icon_skin = 'file' - platform = asset.platform.lower() + platform = asset.platform_base.lower() if platform == 'windows': icon_skin = 'windows' elif platform == 'linux': @@ -489,8 +488,8 @@ class ParserNode: 'hostname': asset.hostname, 'ip': asset.ip, 'protocols': asset.protocols_as_list, - 'platform': asset.platform, - "org_name": asset.org_name, + 'platform': asset.platform_base, + 'org_name': asset.org_name, }, } } diff --git a/apps/settings/models.py b/apps/settings/models.py index 0cbd9dd0e..75b56bb54 100644 --- a/apps/settings/models.py +++ b/apps/settings/models.py @@ -5,9 +5,7 @@ from django.db.utils import ProgrammingError, OperationalError from django.utils.translation import ugettext_lazy as _ from django.core.cache import cache -from common.utils import get_signer - -signer = get_signer() +from common.utils import signer class SettingQuerySet(models.QuerySet): diff --git a/apps/templates/_base_asset_tree_list.html b/apps/templates/_base_asset_tree_list.html index 399b8c3d0..a989a4da1 100644 --- a/apps/templates/_base_asset_tree_list.html +++ b/apps/templates/_base_asset_tree_list.html @@ -44,7 +44,6 @@ function toggleSpliter() { showTree = 1; }); } else { - console.log("hide") $("#split-right").attr("class", "col-sm-9"); $("#toggle-icon").attr("class", "fa fa-angle-left fa-x"); $("#split-left").show(500); diff --git a/apps/templates/_nav.html b/apps/templates/_nav.html index b2c00e09d..578cedfa6 100644 --- a/apps/templates/_nav.html +++ b/apps/templates/_nav.html @@ -116,7 +116,7 @@
  • {% trans 'Task list' %}
    • -
  • {% trans 'Batch command' %}
    • +
  • {% trans 'Batch command' %}
    • {% if request.user.is_superuser %}
  • {% trans 'Task monitor' %}
    • {% endif %} diff --git a/apps/templates/_nav_user.html b/apps/templates/_nav_user.html index c745c0cdc..8196a4f44 100644 --- a/apps/templates/_nav_user.html +++ b/apps/templates/_nav_user.html @@ -22,7 +22,7 @@ {% if SECURITY_COMMAND_EXECUTION %}
  • - + {% trans 'Command execution' %}
    • @@ -41,4 +41,4 @@ {% trans 'File manager' %} - \ No newline at end of file + diff --git a/apps/terminal/migrations/0017_auto_20191125_0931.py b/apps/terminal/migrations/0017_auto_20191125_0931.py index db340afd7..c555e6db6 100644 --- a/apps/terminal/migrations/0017_auto_20191125_0931.py +++ b/apps/terminal/migrations/0017_auto_20191125_0931.py @@ -4,9 +4,8 @@ from django.db import migrations def get_storage_data(s): - from common.utils import get_signer + from common.utils import signer import json - signer = get_signer() value = s.value encrypted = s.encrypted if encrypted: diff --git a/apps/users/models/user.py b/apps/users/models/user.py index af2de523b..a062f6c99 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -17,15 +17,13 @@ from django.utils import timezone from django.shortcuts import reverse from orgs.utils import current_org -from common.utils import get_signer, date_expired_default, get_logger, lazyproperty +from common.utils import signer, date_expired_default, get_logger, lazyproperty from common import fields from ..signals import post_user_change_password __all__ = ['User'] -signer = get_signer() - logger = get_logger(__file__)