From 3d6609ec8c81a8a11d0b121068edafbc6ffd0541 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Wed, 23 Nov 2022 18:36:42 +0800 Subject: [PATCH 1/3] perf: gateway --- apps/assets/models/asset/host.py | 1 - apps/assets/models/domain.py | 5 +- apps/assets/serializers/domain.py | 113 ++++++++++++++---- .../0015_alter_connectiontoken_login.py | 18 +++ 4 files changed, 113 insertions(+), 24 deletions(-) create mode 100644 apps/authentication/migrations/0015_alter_connectiontoken_login.py diff --git a/apps/assets/models/asset/host.py b/apps/assets/models/asset/host.py index 46aeed4f3..a1dbb6de3 100644 --- a/apps/assets/models/asset/host.py +++ b/apps/assets/models/asset/host.py @@ -3,7 +3,6 @@ from .common import Asset class Host(Asset): - pass @classmethod def get_gateway_queryset(cls): diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index bf33caed6..248e02f20 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -13,8 +13,9 @@ from django.utils.translation import ugettext_lazy as _ from common.db import fields from common.utils import get_logger, lazyproperty from orgs.mixins.models import OrgModelMixin +from assets.models import Host from .base import BaseAccount -from ..const import SecretType, GATEWAY_NAME +from ..const import SecretType logger = get_logger(__file__) @@ -37,7 +38,7 @@ class Domain(OrgModelMixin): @lazyproperty def gateways(self): - return self.assets.filter(platform__name=GATEWAY_NAME, is_active=True) + return Host.get_gateway_queryset().filter(domain=self, is_active=True) def select_gateway(self): return self.random_gateway() diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 37d17c814..64c8c032c 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -1,28 +1,33 @@ # -*- coding: utf-8 -*- # from rest_framework import serializers +from rest_framework.generics import get_object_or_404 from django.utils.translation import ugettext_lazy as _ from orgs.mixins.serializers import BulkOrgResourceModelSerializer from common.drf.serializers import SecretReadableMixin -from ..models import Domain, Asset +from common.drf.fields import ObjectRelatedField, EncryptedField +from assets.const import SecretType +from ..models import Domain, Asset, Account +from ..serializers import HostSerializer +from .utils import validate_password_for_ansible, validate_ssh_key class DomainSerializer(BulkOrgResourceModelSerializer): asset_count = serializers.SerializerMethodField(label=_('Assets amount')) gateway_count = serializers.SerializerMethodField(label=_('Gateways count')) + assets = ObjectRelatedField( + many=True, required=False, queryset=Asset.objects, label=_('Asset') + ) class Meta: model = Domain fields_mini = ['id', 'name'] - fields_small = fields_mini + [ - 'comment', 'date_created' - ] - fields_m2m = [ - 'asset_count', 'assets', 'gateway_count', - ] - fields = fields_small + fields_m2m - read_only_fields = ('asset_count', 'gateway_count', 'date_created') + fields_small = fields_mini + ['comment'] + fields_m2m = ['assets'] + read_only_fields = ['asset_count', 'gateway_count', 'date_created'] + fields = fields_small + fields_m2m + read_only_fields + extra_kwargs = { 'assets': {'required': False, 'label': _('Assets')}, } @@ -36,20 +41,86 @@ class DomainSerializer(BulkOrgResourceModelSerializer): return obj.gateways.count() -class GatewaySerializer(BulkOrgResourceModelSerializer): - is_connective = serializers.BooleanField(required=False, label=_('Connectivity')) +class GatewaySerializer(HostSerializer): + password = EncryptedField( + label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024, + validators=[validate_password_for_ansible], write_only=True + ) + private_key = EncryptedField( + label=_('SSH private key'), required=False, allow_blank=True, allow_null=True, + max_length=16384, write_only=True + ) + passphrase = serializers.CharField( + label=_('Key password'), allow_blank=True, allow_null=True, required=False, write_only=True, + max_length=512, + ) + username = serializers.CharField( + label=_('Username'), allow_blank=True, max_length=128, required=True, + ) - class Meta: - model = Asset - fields_mini = ['id'] - fields_small = fields_mini + [ - 'address', 'port', 'protocol', - 'is_active', 'is_connective', - 'date_created', 'date_updated', - 'created_by', 'comment', + class Meta(HostSerializer.Meta): + fields = HostSerializer.Meta.fields + [ + 'username', 'password', 'private_key', 'passphrase' ] - fields_fk = ['domain'] - fields = fields_small + fields_fk + + def validate_private_key(self, secret): + if not secret: + return + passphrase = self.initial_data.get('passphrase') + passphrase = passphrase if passphrase else None + validate_ssh_key(secret, passphrase) + return secret + + @staticmethod + def clean_auth_fields(validated_data): + username = validated_data.pop('username', None) + password = validated_data.pop('password', None) + private_key = validated_data.pop('private_key', None) + validated_data.pop('passphrase', None) + return username, password, private_key + + @staticmethod + def create_accounts(instance, username, password, private_key): + account_name = f'{instance.name}-{_("Gateway")}' + account_data = { + 'privileged': True, + 'name': account_name, + 'username': username, + 'asset_id': instance.id, + 'created_by': instance.created_by + } + if password: + Account.objects.create( + **account_data, secret=password, secret_type=SecretType.PASSWORD + ) + if private_key: + Account.objects.create( + **account_data, secret=private_key, secret_type=SecretType.SSH_KEY + ) + + @staticmethod + def update_accounts(instance, username, password, private_key): + accounts = instance.accounts.filter(username=username) + if password: + account = get_object_or_404(accounts, SecretType.PASSWORD) + account.secret = password + account.save() + if private_key: + account = get_object_or_404(accounts, SecretType.SSH_KEY) + account.secret = private_key + account.save() + + def create(self, validated_data): + auth_fields = self.clean_auth_fields(validated_data) + instance = super().create(validated_data) + self.create_accounts(instance, *auth_fields) + return instance + + def update(self, instance, validated_data): + auth_fields = self.clean_auth_fields(validated_data) + instance = super().update(instance, validated_data) + self.update_accounts(instance, *auth_fields) + return instance class GatewayWithAuthSerializer(SecretReadableMixin, GatewaySerializer): diff --git a/apps/authentication/migrations/0015_alter_connectiontoken_login.py b/apps/authentication/migrations/0015_alter_connectiontoken_login.py new file mode 100644 index 000000000..f2c6abecb --- /dev/null +++ b/apps/authentication/migrations/0015_alter_connectiontoken_login.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-23 02:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('authentication', '0014_auto_20221122_2152'), + ] + + operations = [ + migrations.AlterField( + model_name='connectiontoken', + name='login', + field=models.CharField(max_length=128, verbose_name='Login account'), + ), + ] From d44d475cae1af6e90c9e937ed4f40cba82cc6d43 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 23 Nov 2022 18:39:05 +0800 Subject: [PATCH 2/3] perf: gateway (#9115) Co-authored-by: feng <1304903146@qq.com> --- apps/assets/models/asset/host.py | 1 - apps/assets/models/domain.py | 5 +- apps/assets/serializers/domain.py | 113 ++++++++++++++---- .../0015_alter_connectiontoken_login.py | 18 +++ 4 files changed, 113 insertions(+), 24 deletions(-) create mode 100644 apps/authentication/migrations/0015_alter_connectiontoken_login.py diff --git a/apps/assets/models/asset/host.py b/apps/assets/models/asset/host.py index 46aeed4f3..a1dbb6de3 100644 --- a/apps/assets/models/asset/host.py +++ b/apps/assets/models/asset/host.py @@ -3,7 +3,6 @@ from .common import Asset class Host(Asset): - pass @classmethod def get_gateway_queryset(cls): diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index bf33caed6..248e02f20 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -13,8 +13,9 @@ from django.utils.translation import ugettext_lazy as _ from common.db import fields from common.utils import get_logger, lazyproperty from orgs.mixins.models import OrgModelMixin +from assets.models import Host from .base import BaseAccount -from ..const import SecretType, GATEWAY_NAME +from ..const import SecretType logger = get_logger(__file__) @@ -37,7 +38,7 @@ class Domain(OrgModelMixin): @lazyproperty def gateways(self): - return self.assets.filter(platform__name=GATEWAY_NAME, is_active=True) + return Host.get_gateway_queryset().filter(domain=self, is_active=True) def select_gateway(self): return self.random_gateway() diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 37d17c814..64c8c032c 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -1,28 +1,33 @@ # -*- coding: utf-8 -*- # from rest_framework import serializers +from rest_framework.generics import get_object_or_404 from django.utils.translation import ugettext_lazy as _ from orgs.mixins.serializers import BulkOrgResourceModelSerializer from common.drf.serializers import SecretReadableMixin -from ..models import Domain, Asset +from common.drf.fields import ObjectRelatedField, EncryptedField +from assets.const import SecretType +from ..models import Domain, Asset, Account +from ..serializers import HostSerializer +from .utils import validate_password_for_ansible, validate_ssh_key class DomainSerializer(BulkOrgResourceModelSerializer): asset_count = serializers.SerializerMethodField(label=_('Assets amount')) gateway_count = serializers.SerializerMethodField(label=_('Gateways count')) + assets = ObjectRelatedField( + many=True, required=False, queryset=Asset.objects, label=_('Asset') + ) class Meta: model = Domain fields_mini = ['id', 'name'] - fields_small = fields_mini + [ - 'comment', 'date_created' - ] - fields_m2m = [ - 'asset_count', 'assets', 'gateway_count', - ] - fields = fields_small + fields_m2m - read_only_fields = ('asset_count', 'gateway_count', 'date_created') + fields_small = fields_mini + ['comment'] + fields_m2m = ['assets'] + read_only_fields = ['asset_count', 'gateway_count', 'date_created'] + fields = fields_small + fields_m2m + read_only_fields + extra_kwargs = { 'assets': {'required': False, 'label': _('Assets')}, } @@ -36,20 +41,86 @@ class DomainSerializer(BulkOrgResourceModelSerializer): return obj.gateways.count() -class GatewaySerializer(BulkOrgResourceModelSerializer): - is_connective = serializers.BooleanField(required=False, label=_('Connectivity')) +class GatewaySerializer(HostSerializer): + password = EncryptedField( + label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024, + validators=[validate_password_for_ansible], write_only=True + ) + private_key = EncryptedField( + label=_('SSH private key'), required=False, allow_blank=True, allow_null=True, + max_length=16384, write_only=True + ) + passphrase = serializers.CharField( + label=_('Key password'), allow_blank=True, allow_null=True, required=False, write_only=True, + max_length=512, + ) + username = serializers.CharField( + label=_('Username'), allow_blank=True, max_length=128, required=True, + ) - class Meta: - model = Asset - fields_mini = ['id'] - fields_small = fields_mini + [ - 'address', 'port', 'protocol', - 'is_active', 'is_connective', - 'date_created', 'date_updated', - 'created_by', 'comment', + class Meta(HostSerializer.Meta): + fields = HostSerializer.Meta.fields + [ + 'username', 'password', 'private_key', 'passphrase' ] - fields_fk = ['domain'] - fields = fields_small + fields_fk + + def validate_private_key(self, secret): + if not secret: + return + passphrase = self.initial_data.get('passphrase') + passphrase = passphrase if passphrase else None + validate_ssh_key(secret, passphrase) + return secret + + @staticmethod + def clean_auth_fields(validated_data): + username = validated_data.pop('username', None) + password = validated_data.pop('password', None) + private_key = validated_data.pop('private_key', None) + validated_data.pop('passphrase', None) + return username, password, private_key + + @staticmethod + def create_accounts(instance, username, password, private_key): + account_name = f'{instance.name}-{_("Gateway")}' + account_data = { + 'privileged': True, + 'name': account_name, + 'username': username, + 'asset_id': instance.id, + 'created_by': instance.created_by + } + if password: + Account.objects.create( + **account_data, secret=password, secret_type=SecretType.PASSWORD + ) + if private_key: + Account.objects.create( + **account_data, secret=private_key, secret_type=SecretType.SSH_KEY + ) + + @staticmethod + def update_accounts(instance, username, password, private_key): + accounts = instance.accounts.filter(username=username) + if password: + account = get_object_or_404(accounts, SecretType.PASSWORD) + account.secret = password + account.save() + if private_key: + account = get_object_or_404(accounts, SecretType.SSH_KEY) + account.secret = private_key + account.save() + + def create(self, validated_data): + auth_fields = self.clean_auth_fields(validated_data) + instance = super().create(validated_data) + self.create_accounts(instance, *auth_fields) + return instance + + def update(self, instance, validated_data): + auth_fields = self.clean_auth_fields(validated_data) + instance = super().update(instance, validated_data) + self.update_accounts(instance, *auth_fields) + return instance class GatewayWithAuthSerializer(SecretReadableMixin, GatewaySerializer): diff --git a/apps/authentication/migrations/0015_alter_connectiontoken_login.py b/apps/authentication/migrations/0015_alter_connectiontoken_login.py new file mode 100644 index 000000000..f2c6abecb --- /dev/null +++ b/apps/authentication/migrations/0015_alter_connectiontoken_login.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-23 02:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('authentication', '0014_auto_20221122_2152'), + ] + + operations = [ + migrations.AlterField( + model_name='connectiontoken', + name='login', + field=models.CharField(max_length=128, verbose_name='Login account'), + ), + ] From fa948f7327e817e7db11c14f88abdc1e8bac1f80 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Thu, 24 Nov 2022 00:50:37 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20job=20=E5=A2=9E=E5=8A=A0=E7=BB=84?= =?UTF-8?q?=E7=BB=87=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/api/job.py | 24 ++++++------ apps/ops/migrations/0034_job_org_id.py | 18 +++++++++ .../migrations/0035_jobexecution_org_id.py | 18 +++++++++ apps/ops/models/job.py | 8 ++-- apps/ops/serializers/job.py | 5 +-- apps/ops/tasks.py | 37 ++++++++++--------- 6 files changed, 74 insertions(+), 36 deletions(-) create mode 100644 apps/ops/migrations/0034_job_org_id.py create mode 100644 apps/ops/migrations/0035_jobexecution_org_id.py diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index eaad4514c..86a52f373 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -1,4 +1,3 @@ -from django.shortcuts import get_object_or_404 from rest_framework import viewsets from ops.models import Job, JobExecution @@ -7,14 +6,17 @@ from ops.serializers.job import JobSerializer, JobExecutionSerializer __all__ = ['JobViewSet', 'JobExecutionViewSet'] from ops.tasks import run_ops_job, run_ops_job_executions +from orgs.mixins.api import OrgBulkModelViewSet -class JobViewSet(viewsets.ModelViewSet): +class JobViewSet(OrgBulkModelViewSet): serializer_class = JobSerializer - queryset = Job.objects.all() + model = Job + permission_classes = () def get_queryset(self): - return self.queryset.filter(instant=False) + query_set = super().get_queryset() + return query_set.filter(instant=False) def perform_create(self, serializer): instance = serializer.save() @@ -22,20 +24,20 @@ class JobViewSet(viewsets.ModelViewSet): run_ops_job.delay(instance.id) -class JobExecutionViewSet(viewsets.ModelViewSet): +class JobExecutionViewSet(OrgBulkModelViewSet): serializer_class = JobExecutionSerializer - queryset = JobExecution.objects.all() http_method_names = ('get', 'post', 'head', 'options',) + # filter_fields = ('type',) + permission_classes = () + model = JobExecution def perform_create(self, serializer): instance = serializer.save() run_ops_job_executions.delay(instance.id) def get_queryset(self): + query_set = super().get_queryset() job_id = self.request.query_params.get('job_id') - job_type = self.request.query_params.get('type') if job_id: - self.queryset = self.queryset.filter(job_id=job_id) - if job_type: - self.queryset = self.queryset.filter(job__type=job_type) - return self.queryset + self.queryset = query_set.filter(job_id=job_id) + return query_set diff --git a/apps/ops/migrations/0034_job_org_id.py b/apps/ops/migrations/0034_job_org_id.py new file mode 100644 index 000000000..07926cec3 --- /dev/null +++ b/apps/ops/migrations/0034_job_org_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-23 09:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0033_auto_20221118_1431'), + ] + + operations = [ + migrations.AddField( + model_name='job', + name='org_id', + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), + ), + ] diff --git a/apps/ops/migrations/0035_jobexecution_org_id.py b/apps/ops/migrations/0035_jobexecution_org_id.py new file mode 100644 index 000000000..1161d10e3 --- /dev/null +++ b/apps/ops/migrations/0035_jobexecution_org_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-23 10:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0034_job_org_id'), + ] + + operations = [ + migrations.AddField( + model_name='jobexecution', + name='org_id', + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), + ), + ] diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py index d5542970d..f2e7eaa4b 100644 --- a/apps/ops/models/job.py +++ b/apps/ops/models/job.py @@ -9,16 +9,14 @@ from django.utils.translation import gettext_lazy as _ from django.utils import timezone from celery import current_task -from common.const.choices import Trigger -from common.db.models import BaseCreateUpdateModel - __all__ = ["Job", "JobExecution"] from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner from ops.mixin import PeriodTaskModelMixin +from orgs.mixins.models import JMSOrgBaseModel -class Job(BaseCreateUpdateModel, PeriodTaskModelMixin): +class Job(JMSOrgBaseModel, PeriodTaskModelMixin): class Types(models.TextChoices): adhoc = 'adhoc', _('Adhoc') playbook = 'playbook', _('Playbook') @@ -94,7 +92,7 @@ class Job(BaseCreateUpdateModel, PeriodTaskModelMixin): return self.executions.create() -class JobExecution(BaseCreateUpdateModel): +class JobExecution(JMSOrgBaseModel): id = models.UUIDField(default=uuid.uuid4, primary_key=True) task_id = models.UUIDField(null=True) status = models.CharField(max_length=16, verbose_name=_('Status'), default='running') diff --git a/apps/ops/serializers/job.py b/apps/ops/serializers/job.py index 389b92ce2..e5d76f85b 100644 --- a/apps/ops/serializers/job.py +++ b/apps/ops/serializers/job.py @@ -1,14 +1,13 @@ -from django.db import transaction from rest_framework import serializers - from common.drf.fields import ReadableHiddenField from ops.mixin import PeriodTaskSerializerMixin from ops.models import Job, JobExecution +from orgs.mixins.serializers import BulkOrgResourceModelSerializer _all_ = [] -class JobSerializer(serializers.ModelSerializer, PeriodTaskSerializerMixin): +class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin): owner = ReadableHiddenField(default=serializers.CurrentUserDefault()) class Meta: diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 841f759ff..51541dd32 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -10,6 +10,7 @@ from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from common.utils import get_logger, get_object_or_none, get_log_keep_day +from orgs.utils import tmp_to_org from .celery.decorator import ( register_as_period_task, after_app_shutdown_clean_periodic, after_app_ready_start @@ -27,28 +28,30 @@ logger = get_logger(__file__) @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task")) def run_ops_job(job_id): job = get_object_or_none(Job, id=job_id) - execution = job.create_execution() - try: - execution.start() - except SoftTimeLimitExceeded: - execution.set_error('Run timeout') - logger.error("Run adhoc timeout") - except Exception as e: - execution.set_error(e) - logger.error("Start adhoc execution error: {}".format(e)) + with tmp_to_org(job.org): + execution = job.create_execution() + try: + execution.start() + except SoftTimeLimitExceeded: + execution.set_error('Run timeout') + logger.error("Run adhoc timeout") + except Exception as e: + execution.set_error(e) + logger.error("Start adhoc execution error: {}".format(e)) @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task execution")) def run_ops_job_executions(execution_id, **kwargs): execution = get_object_or_none(JobExecution, id=execution_id) - try: - execution.start() - except SoftTimeLimitExceeded: - execution.set_error('Run timeout') - logger.error("Run adhoc timeout") - except Exception as e: - execution.set_error(e) - logger.error("Start adhoc execution error: {}".format(e)) + with tmp_to_org(execution.org): + try: + execution.start() + except SoftTimeLimitExceeded: + execution.set_error('Run timeout') + logger.error("Run adhoc timeout") + except Exception as e: + execution.set_error(e) + logger.error("Start adhoc execution error: {}".format(e)) @shared_task(verbose_name=_('Periodic clear celery tasks'))