From d3c67d2f04fab687e0ae24f718e3ccc6ad8468ef Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 14 Jul 2022 10:56:09 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=9A=82=E5=AD=98=E4=B8=80=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/account_history.py | 13 ++---- apps/assets/api/accounts.py | 18 +++----- apps/assets/api/system_user_relation.py | 1 - .../migrations/0092_auto_20220711_1409.py | 8 +++- .../migrations/0093_auto_20220711_1413.py | 13 +++--- .../0094_alter_systemuser_assets.py | 22 +++++++++ .../migrations/0095_auto_20220713_1746.py | 45 +++++++++++++++++++ apps/assets/models/account.py | 9 +++- apps/assets/models/user.py | 1 - apps/assets/serializers/system_user.py | 2 +- apps/assets/tasks/asset_connectivity.py | 11 ++--- apps/assets/tasks/common.py | 3 +- apps/assets/tasks/system_user_connectivity.py | 10 ++--- 13 files changed, 112 insertions(+), 44 deletions(-) create mode 100644 apps/assets/migrations/0094_alter_systemuser_assets.py create mode 100644 apps/assets/migrations/0095_auto_20220713_1746.py diff --git a/apps/assets/api/account_history.py b/apps/assets/api/account_history.py index 3258dfd48..6ca4fd349 100644 --- a/apps/assets/api/account_history.py +++ b/apps/assets/api/account_history.py @@ -1,23 +1,21 @@ -from django.db.models import F - from assets.api.accounts import ( AccountFilterSet, AccountViewSet, AccountSecretsViewSet ) from common.mixins import RecordViewLogMixin from .. import serializers -from ..models import AuthBook +from ..models import Account __all__ = ['AccountHistoryViewSet', 'AccountHistorySecretsViewSet'] class AccountHistoryFilterSet(AccountFilterSet): class Meta: - model = AuthBook.history.model + model = Account.history.model fields = AccountFilterSet.Meta.fields class AccountHistoryViewSet(AccountViewSet): - model = AuthBook.history.model + model = Account.history.model filterset_class = AccountHistoryFilterSet serializer_classes = { 'default': serializers.AccountHistorySerializer, @@ -26,13 +24,8 @@ class AccountHistoryViewSet(AccountViewSet): 'list': 'assets.view_assethistoryaccount', 'retrieve': 'assets.view_assethistoryaccount', } - http_method_names = ['get', 'options'] - def get_queryset(self): - queryset = AuthBook.get_queryset(is_history_model=True) - return queryset - class AccountHistorySecretsViewSet(RecordViewLogMixin, AccountHistoryViewSet): serializer_classes = { diff --git a/apps/assets/api/accounts.py b/apps/assets/api/accounts.py index 6ad59ed88..be6833e7b 100644 --- a/apps/assets/api/accounts.py +++ b/apps/assets/api/accounts.py @@ -12,7 +12,7 @@ from common.mixins import RecordViewLogMixin from common.permissions import UserConfirmation from authentication.const import ConfirmType from ..tasks.account_connectivity import test_accounts_connectivity_manual -from ..models import AuthBook, Node +from ..models import Node, Account from .. import serializers __all__ = ['AccountFilterSet', 'AccountViewSet', 'AccountSecretsViewSet', 'AccountTaskCreateAPI'] @@ -50,16 +50,16 @@ class AccountFilterSet(BaseFilterSet): return qs class Meta: - model = AuthBook + model = Account fields = [ - 'asset', 'systemuser', 'id', + 'asset', 'id', ] class AccountViewSet(OrgBulkModelViewSet): - model = AuthBook - filterset_fields = ("username", "asset", "systemuser", 'ip', 'hostname') - search_fields = ('username', 'ip', 'hostname', 'systemuser__username') + model = Account + filterset_fields = ("username", "asset", 'ip', 'hostname') + search_fields = ('username', 'ip', 'hostname') filterset_class = AccountFilterSet serializer_classes = { 'default': serializers.AccountSerializer, @@ -70,10 +70,6 @@ class AccountViewSet(OrgBulkModelViewSet): 'partial_update': 'assets.change_assetaccountsecret', } - def get_queryset(self): - queryset = AuthBook.get_queryset() - return queryset - @action(methods=['post'], detail=True, url_path='verify') def verify_account(self, request, *args, **kwargs): account = super().get_object() @@ -106,7 +102,7 @@ class AccountTaskCreateAPI(CreateAPIView): return request.user.has_perm('assets.test_assetconnectivity') def get_accounts(self): - queryset = AuthBook.objects.all() + queryset = Account.objects.all() queryset = self.filter_queryset(queryset) return queryset diff --git a/apps/assets/api/system_user_relation.py b/apps/assets/api/system_user_relation.py index 2d8018e4d..36c16a09b 100644 --- a/apps/assets/api/system_user_relation.py +++ b/apps/assets/api/system_user_relation.py @@ -68,7 +68,6 @@ class BaseRelationViewSet(RelationMixin, OrgBulkModelViewSet): class SystemUserAssetRelationViewSet(BaseRelationViewSet): - perm_model = models.AuthBook serializer_class = serializers.SystemUserAssetRelationSerializer model = models.SystemUser.assets.through filterset_fields = [ diff --git a/apps/assets/migrations/0092_auto_20220711_1409.py b/apps/assets/migrations/0092_auto_20220711_1409.py index 5fb8704bd..d6076b6c3 100644 --- a/apps/assets/migrations/0092_auto_20220711_1409.py +++ b/apps/assets/migrations/0092_auto_20220711_1409.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.12 on 2022-07-11 06:08 +# Generated by Django 3.2.12 on 2022-07-11 08:59 import assets.models.base import assets.models.user @@ -22,6 +22,8 @@ class Migration(migrations.Migration): name='HistoricalAccount', fields=[ ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')), + ('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')), ('id', models.UUIDField(db_index=True, default=uuid.uuid4)), ('name', models.CharField(max_length=128, verbose_name='Name')), ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')), @@ -33,6 +35,7 @@ class Migration(migrations.Migration): ('date_updated', models.DateTimeField(blank=True, editable=False, verbose_name='Date updated')), ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')), ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')), + ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')), ('version', models.IntegerField(default=1, verbose_name='Version')), ('history_id', models.AutoField(primary_key=True, serialize=False)), ('history_date', models.DateTimeField(db_index=True)), @@ -53,6 +56,8 @@ class Migration(migrations.Migration): name='Account', fields=[ ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')), + ('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')), ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), ('name', models.CharField(max_length=128, verbose_name='Name')), ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')), @@ -64,6 +69,7 @@ class Migration(migrations.Migration): ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')), ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')), + ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')), ('version', models.IntegerField(default=1, verbose_name='Version')), ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.asset', verbose_name='Asset')), ], diff --git a/apps/assets/migrations/0093_auto_20220711_1413.py b/apps/assets/migrations/0093_auto_20220711_1413.py index 2a9e1b856..ecb4b47e6 100644 --- a/apps/assets/migrations/0093_auto_20220711_1413.py +++ b/apps/assets/migrations/0093_auto_20220711_1413.py @@ -10,16 +10,18 @@ def migrate_accounts(apps, schema_editor): count = 0 bulk_size = 1000 + print("\nStart migrate accounts") while True: + start = time.time() auth_books = auth_book_model.objects \ - .prefetch_related('systemuser') \ - .all()[count:count+bulk_size] + .prefetch_related('systemuser') \ + .all()[count:count+bulk_size] + count += len(auth_books) if not auth_books: break accounts = [] - start = time.time() - # authbook 和 account 相同的属性 + # auth book 和 account 相同的属性 same_attrs = [ 'id', 'comment', 'date_created', 'date_updated', 'created_by', 'asset_id', 'org_id', @@ -47,9 +49,8 @@ def migrate_accounts(apps, schema_editor): account_model.objects.bulk_create(accounts, ignore_conflicts=True) print("Create accounts: {}-{} using: {:.2f}s".format( - count, count + len(accounts), time.time()-start + count - bulk_size, count, time.time()-start )) - count += len(auth_books) class Migration(migrations.Migration): diff --git a/apps/assets/migrations/0094_alter_systemuser_assets.py b/apps/assets/migrations/0094_alter_systemuser_assets.py new file mode 100644 index 000000000..c3f2f2cfc --- /dev/null +++ b/apps/assets/migrations/0094_alter_systemuser_assets.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.12 on 2022-07-13 09:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0093_auto_20220711_1413'), + ] + + operations = [ + migrations.RemoveField( + model_name='systemuser', + name='assets', + ), + migrations.AddField( + model_name='systemuser', + name='assets', + field=models.ManyToManyField(blank=True, related_name='system_users', to='assets.Asset', verbose_name='Assets'), + ), + ] diff --git a/apps/assets/migrations/0095_auto_20220713_1746.py b/apps/assets/migrations/0095_auto_20220713_1746.py new file mode 100644 index 000000000..b803dae51 --- /dev/null +++ b/apps/assets/migrations/0095_auto_20220713_1746.py @@ -0,0 +1,45 @@ +# Generated by Django 3.2.12 on 2022-07-13 09:46 + +import time +from django.db import migrations + + +def migrate_asset_system_user_relations(apps, schema_editor): + system_user_model = apps.get_model('assets', 'SystemUser') + old_model = apps.get_model('assets', 'AuthBook') + new_model = system_user_model.assets.through + + count = 0 + bulk_size = 1000 + print("\nStart migrate asset system user relations") + while True: + start = time.time() + auth_books = old_model.objects.only('asset_id', 'systemuser_id')[count:count+bulk_size] + auth_books = list(auth_books) + count += len(auth_books) + if not auth_books: + break + asset_system_users = [] + for auth_book in auth_books: + if not auth_book.asset_id or not auth_book.systemuser_id: + continue + asset_system_user = new_model( + asset_id=auth_book.asset_id, + systemuser_id=auth_book.systemuser_id + ) + asset_system_users.append(asset_system_user) + new_model.objects.bulk_create(asset_system_users, ignore_conflicts=True) + print("Create asset system user relations: {}-{} using: {:.2f}s".format( + count - bulk_size, count, time.time()-start + )) + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0094_alter_systemuser_assets'), + ] + + operations = [ + migrations.RunPython(migrate_asset_system_user_relations) + ] diff --git a/apps/assets/models/account.py b/apps/assets/models/account.py index 7732c50a2..d060ec40d 100644 --- a/apps/assets/models/account.py +++ b/apps/assets/models/account.py @@ -3,15 +3,20 @@ from django.utils.translation import gettext_lazy as _ from simple_history.models import HistoricalRecords from .user import ProtocolMixin -from .base import BaseUser +from .base import BaseUser, AbsConnectivity __all__ = ['Account'] -class Account(BaseUser, ProtocolMixin): +class Account(BaseUser, AbsConnectivity, ProtocolMixin): + class Type(models.TextChoices): + common = 'common', _('Common user') + admin = 'admin', _('Admin user') + protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh', verbose_name=_('Protocol')) + type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_("Type")) asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset')) version = models.IntegerField(default=1, verbose_name=_('Version')) history = HistoricalRecords() diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 0eb27e912..29f7b5972 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -242,7 +242,6 @@ class SystemUser(ProtocolMixin, AuthMixin, BaseUser): nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes")) assets = models.ManyToManyField( 'assets.Asset', blank=True, verbose_name=_("Assets"), - through='assets.AuthBook', through_fields=['systemuser', 'asset'], related_name='system_users' ) users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users")) diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index 54aff3e82..aa17b0585 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -298,7 +298,7 @@ class SystemUserAssetRelationSerializer(RelationMixin, serializers.ModelSerializ asset_display = serializers.ReadOnlyField(label=_('Asset hostname')) class Meta: - model = SystemUser.assets.through + model = SystemUser fields = [ "id", "asset", "asset_display", 'systemuser', 'systemuser_display', "connectivity", 'date_verified', 'org_id' diff --git a/apps/assets/tasks/asset_connectivity.py b/apps/assets/tasks/asset_connectivity.py index 0038c38d3..491792095 100644 --- a/apps/assets/tasks/asset_connectivity.py +++ b/apps/assets/tasks/asset_connectivity.py @@ -6,7 +6,7 @@ from django.utils.translation import gettext_noop from common.utils import get_logger from orgs.utils import org_aware_func -from ..models import Asset, Connectivity, AuthBook +from ..models import Asset, Connectivity, Account from . import const from .utils import clean_ansible_task_hosts, group_asset_by_platform @@ -18,6 +18,7 @@ __all__ = [ ] +# Todo: 这里可能有问题了 def set_assets_accounts_connectivity(assets, results_summary): asset_ids_ok = set() asset_ids_failed = set() @@ -33,11 +34,11 @@ def set_assets_accounts_connectivity(assets, results_summary): Asset.bulk_set_connectivity(asset_ids_ok, Connectivity.ok) Asset.bulk_set_connectivity(asset_ids_failed, Connectivity.failed) - accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser__type='admin') - accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser__type='admin') + accounts_ok = Account.objects.filter(asset_id__in=asset_ids_ok,) + accounts_failed = Account.objects.filter(asset_id__in=asset_ids_faile) - AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok) - AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed) + Account.bulk_set_connectivity(accounts_ok, Connectivity.ok) + Account.bulk_set_connectivity(accounts_failed, Connectivity.failed) @shared_task(queue="ansible") diff --git a/apps/assets/tasks/common.py b/apps/assets/tasks/common.py index b6c2326e9..a1bd79e11 100644 --- a/apps/assets/tasks/common.py +++ b/apps/assets/tasks/common.py @@ -17,6 +17,7 @@ def add_nodes_assets_to_system_users(nodes_keys, system_users): nodes = Node.objects.filter(key__in=nodes_keys) assets = Node.get_nodes_all_assets(*nodes) + for system_user in system_users: """ 解决资产和节点进行关联时，已经关联过的节点不会触发 authbook post_save 信号， 无法更新节点下所有资产的管理用户的问题 """ @@ -28,7 +29,7 @@ def add_nodes_assets_to_system_users(nodes_keys, system_users): ) if created: need_push_asset_ids.append(asset.id) - # # 不再自动更新资产管理用户，只允许用户手动指定。 + # 不再自动更新资产管理用户，只允许用户手动指定。 # 只要关联都需要更新资产的管理用户 # instance.update_asset_admin_user_if_need() diff --git a/apps/assets/tasks/system_user_connectivity.py b/apps/assets/tasks/system_user_connectivity.py index 893081163..2213bfa26 100644 --- a/apps/assets/tasks/system_user_connectivity.py +++ b/apps/assets/tasks/system_user_connectivity.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext as _, gettext_noop from assets.models import Asset from common.utils import get_logger from orgs.utils import tmp_to_org, org_aware_func -from ..models import SystemUser, Connectivity, AuthBook +from ..models import SystemUser, Connectivity, Account from . import const from .utils import ( clean_ansible_task_hosts, group_asset_by_platform @@ -34,11 +34,11 @@ def set_assets_accounts_connectivity(system_user, assets, results_summary): else: asset_ids_failed.add(asset.id) - accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user) - accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user) + accounts_ok = Account.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user) + accounts_failed = Account.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user) - AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok) - AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed) + Account.bulk_set_connectivity(accounts_ok, Connectivity.ok) + Account.bulk_set_connectivity(accounts_failed, Connectivity.failed) @org_aware_func("system_user")