From eabb41b74c7df03d97565618146209f28321e088 Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 8 Feb 2023 13:53:21 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20platform=20seriali?= =?UTF-8?q?zer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/tasks/common.py | 5 ++-- apps/assets/const/web.py | 7 ++++++ apps/assets/models/asset/web.py | 6 +---- apps/assets/serializers/platform.py | 6 +++-- apps/assets/signal_handlers/asset.py | 19 +++++++++++--- apps/assets/tasks/common.py | 11 +++++---- apps/assets/tasks/gather_facts.py | 9 +++---- apps/assets/tasks/ping.py | 37 +++++++--------------------- 8 files changed, 49 insertions(+), 51 deletions(-) diff --git a/apps/accounts/tasks/common.py b/apps/accounts/tasks/common.py index 1f422ab5f..569a487a3 100644 --- a/apps/accounts/tasks/common.py +++ b/apps/accounts/tasks/common.py @@ -1,13 +1,12 @@ - import uuid -from assets.tasks.common import generate_data +from assets.tasks.common import generate_automation_execution_data from common.const.choices import Trigger def automation_execute_start(task_name, tp, child_snapshot=None): from accounts.models import AutomationExecution - data = generate_data(task_name, tp, child_snapshot) + data = generate_automation_execution_data(task_name, tp, child_snapshot) while True: try: diff --git a/apps/assets/const/web.py b/apps/assets/const/web.py index 42ea995ac..bf9d9e3c8 100644 --- a/apps/assets/const/web.py +++ b/apps/assets/const/web.py @@ -1,3 +1,4 @@ +from django.db import models from django.utils.translation import gettext_lazy as _ from .base import BaseType @@ -52,3 +53,9 @@ class WebTypes(BaseType): return [ cls.WEBSITE, ] + + +class FillType(models.TextChoices): + no = 'no', _('Disabled') + basic = 'basic', _('Basic') + script = 'script', _('Script') diff --git a/apps/assets/models/asset/web.py b/apps/assets/models/asset/web.py index 15445b3d6..86cb87db8 100644 --- a/apps/assets/models/asset/web.py +++ b/apps/assets/models/asset/web.py @@ -1,15 +1,11 @@ from django.db import models from django.utils.translation import gettext_lazy as _ +from assets.const.web import FillType from .common import Asset class Web(Asset): - class FillType(models.TextChoices): - no = 'no', _('Disabled') - basic = 'basic', _('Basic') - script = 'script', _('Script') - autofill = models.CharField(max_length=16, choices=FillType.choices, default='basic', verbose_name=_("Autofill")) username_selector = models.CharField(max_length=128, blank=True, default='', verbose_name=_("Username selector")) password_selector = models.CharField(max_length=128, blank=True, default='', verbose_name=_("Password selector")) diff --git a/apps/assets/serializers/platform.py b/apps/assets/serializers/platform.py index c7455fbc2..b3bc86e4d 100644 --- a/apps/assets/serializers/platform.py +++ b/apps/assets/serializers/platform.py @@ -1,8 +1,9 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers -from common.serializers.fields import LabeledChoiceField +from assets.const.web import FillType from common.serializers import WritableNestedModelSerializer +from common.serializers.fields import LabeledChoiceField from ..const import Category, AllTypes from ..models import Platform, PlatformProtocol, PlatformAutomation @@ -25,7 +26,7 @@ class ProtocolSettingSerializer(serializers.Serializer): sftp_home = serializers.CharField(default="/tmp", label=_("SFTP home")) # HTTP - autofile = serializers.BooleanField(default=False, label=_("Autofill")) + autofill = serializers.ChoiceField(default='basic', choices=FillType.choices, label=_("Autofill")) username_selector = serializers.CharField( default="", allow_blank=True, label=_("Username selector") ) @@ -35,6 +36,7 @@ class ProtocolSettingSerializer(serializers.Serializer): submit_selector = serializers.CharField( default="", allow_blank=True, label=_("Submit selector") ) + script = serializers.JSONField(default=list, label=_("Script")) class PlatformAutomationSerializer(serializers.ModelSerializer): diff --git a/apps/assets/signal_handlers/asset.py b/apps/assets/signal_handlers/asset.py index f7ea06be4..0026d200e 100644 --- a/apps/assets/signal_handlers/asset.py +++ b/apps/assets/signal_handlers/asset.py @@ -4,11 +4,12 @@ from django.db.models.signals import ( post_save, m2m_changed, pre_delete, post_delete, pre_save ) from django.dispatch import receiver +from django.utils.translation import gettext_noop from assets.models import Asset, Node, Cloud, Device, Host, Web, Database -from assets.tasks.ping import test_assets_connectivity_util +from assets.tasks import test_assets_connectivity_task from common.const.signals import POST_ADD, POST_REMOVE, PRE_REMOVE -from common.decorators import on_transaction_commit +from common.decorators import on_transaction_commit, merge_delay_run from common.utils import get_logger logger = get_logger(__file__) @@ -19,6 +20,17 @@ def on_node_pre_save(sender, instance: Node, **kwargs): instance.parent_key = instance.compute_parent_key() +@merge_delay_run(ttl=10) +def test_assets_connectivity_handler(*assets): + task_name = gettext_noop("Test assets connectivity ") + test_assets_connectivity_task.delay(assets, task_name) + + +@merge_delay_run(ttl=10) +def gather_assets_facts_handler(*assets): + pass + + @receiver(post_save, sender=Asset) @on_transaction_commit def on_asset_create(sender, instance=None, created=False, **kwargs): @@ -31,7 +43,8 @@ def on_asset_create(sender, instance=None, created=False, **kwargs): logger.info("Asset create signal recv: {}".format(instance)) # 获取资产硬件信息 - test_assets_connectivity_util([instance]) + test_assets_connectivity_handler([instance]) + gather_assets_facts_handler([instance]) # 确保资产存在一个节点 has_node = instance.nodes.all().exists() diff --git a/apps/assets/tasks/common.py b/apps/assets/tasks/common.py index 9bafbcde8..a00485fe5 100644 --- a/apps/assets/tasks/common.py +++ b/apps/assets/tasks/common.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- # import uuid + from celery import current_task -from django.db.utils import IntegrityError -from orgs.utils import current_org from common.const.choices import Trigger +from orgs.utils import current_org -def generate_data(task_name, tp, child_snapshot=None): +def generate_automation_execution_data(task_name, tp, child_snapshot=None): child_snapshot = child_snapshot or {} from assets.models import BaseAutomation try: @@ -29,9 +29,9 @@ def generate_data(task_name, tp, child_snapshot=None): return {'id': eid, 'snapshot': snapshot} -def automation_execute_start(task_name, tp, child_snapshot=None): +def quickstart_automation(task_name, tp, child_snapshot=None): from assets.models import AutomationExecution - data = generate_data(task_name, tp, child_snapshot) + data = generate_automation_execution_data(task_name, tp, child_snapshot) while True: try: @@ -40,6 +40,7 @@ def automation_execute_start(task_name, tp, child_snapshot=None): data['id'] = str(uuid.uuid4()) except AutomationExecution.DoesNotExist: break + execution = AutomationExecution.objects.create( trigger=Trigger.manual, **data ) diff --git a/apps/assets/tasks/gather_facts.py b/apps/assets/tasks/gather_facts.py index ad678b92a..835621568 100644 --- a/apps/assets/tasks/gather_facts.py +++ b/apps/assets/tasks/gather_facts.py @@ -3,11 +3,10 @@ from celery import shared_task from django.utils.translation import gettext_noop, gettext_lazy as _ -from common.utils import get_logger from assets.const import AutomationTypes -from orgs.utils import org_aware_func, tmp_to_root_org - -from .common import automation_execute_start +from common.utils import get_logger +from orgs.utils import org_aware_func +from .common import quickstart_automation logger = get_logger(__file__) __all__ = [ @@ -30,7 +29,7 @@ def update_fact_util(assets=None, nodes=None, task_name=None): 'nodes': [str(node.id) for node in nodes], } tp = AutomationTypes.gather_facts - automation_execute_start(task_name, tp, child_snapshot) + quickstart_automation(task_name, tp, child_snapshot) @org_aware_func('assets') diff --git a/apps/assets/tasks/ping.py b/apps/assets/tasks/ping.py index 252390870..dd48af59d 100644 --- a/apps/assets/tasks/ping.py +++ b/apps/assets/tasks/ping.py @@ -2,10 +2,10 @@ from celery import shared_task from django.utils.translation import gettext_noop -from assets.const import AutomationTypes, GATEWAY_NAME +from assets.const import AutomationTypes from common.utils import get_logger from orgs.utils import org_aware_func -from .common import automation_execute_start +from .common import quickstart_automation logger = get_logger(__file__) @@ -16,47 +16,28 @@ __all__ = [ ] -def _test_connectivity_util(assets, tp, task_name, local_port=None): - if not assets: - return - - if local_port is None: - child_snapshot = {} - else: - child_snapshot = {'local_port': local_port} - - child_snapshot['assets'] = [str(asset.id) for asset in assets] - automation_execute_start(task_name, tp, child_snapshot) - - @shared_task @org_aware_func('assets') -def test_assets_connectivity_task(assets, task_name=None, local_port=None): +def test_assets_connectivity_task(assets, task_name=None): from assets.models import PingAutomation if task_name is None: task_name = gettext_noop("Test assets connectivity ") task_name = PingAutomation.generate_unique_name(task_name) - - gateway_assets = assets.filter(platform__name=GATEWAY_NAME) - _test_connectivity_util( - gateway_assets, AutomationTypes.ping_gateway, task_name, local_port - ) - - non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) - _test_connectivity_util(non_gateway_assets, AutomationTypes.ping, task_name) + child_snapshot = {'assets': [str(asset.id) for asset in assets]} + quickstart_automation(task_name, AutomationTypes.ping, child_snapshot) -def test_assets_connectivity_manual(asset_ids, local_port=None): +def test_assets_connectivity_manual(asset_ids): from assets.models import Asset assets = Asset.objects.filter(id__in=asset_ids) task_name = gettext_noop("Test assets connectivity ") - test_assets_connectivity_task.delay(assets, task_name, local_port) + test_assets_connectivity_task.delay(assets, task_name) -def test_node_assets_connectivity_manual(node_id, local_port=None): +def test_node_assets_connectivity_manual(node_id): from assets.models import Node node = Node.objects.get(id=node_id) task_name = gettext_noop("Test if the assets under the node are connectable ") assets = node.get_all_assets() - test_assets_connectivity_task.delay(*assets, task_name, local_port) + test_assets_connectivity_task.delay(*assets, task_name)