diff --git a/apps/__init__.py b/apps/__init__.py index 6110e1346..af5b4a679 100644 --- a/apps/__init__.py +++ b/apps/__init__.py @@ -2,4 +2,4 @@ # -*- coding: utf-8 -*- # -__version__ = "1.0.0" +__version__ = "1.3.0" diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index 9520ff120..50c037df9 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -50,7 +50,9 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet): if node_id: node = get_object_or_404(Node, id=node_id) if not node.is_root(): - queryset = queryset.filter(nodes__key__startswith=node.key).distinct() + queryset = queryset.filter( + nodes__key__regex='{}(:[0-9]+)*$'.format(node.key), + ).distinct() return queryset diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 3b61ceb51..d7499760a 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -30,6 +30,7 @@ from .. import serializers logger = get_logger(__file__) __all__ = [ 'NodeViewSet', 'NodeChildrenApi', + 'NodeAssetsApi', 'NodeWithAssetsApi', 'NodeAddAssetsApi', 'NodeRemoveAssetsApi', 'NodeAddChildrenApi', 'RefreshNodeHardwareInfoApi', 'TestNodeConnectiveApi' @@ -47,6 +48,34 @@ class NodeViewSet(BulkModelViewSet): serializer.save() +class NodeWithAssetsApi(generics.ListAPIView): + permission_classes = (IsSuperUser,) + serializers = serializers.NodeSerializer + + def get_node(self): + pk = self.kwargs.get('pk') or self.request.query_params.get('node') + if not pk: + node = Node.root() + else: + node = get_object_or_404(Node, pk) + return node + + def get_queryset(self): + queryset = [] + node = self.get_node() + children = node.get_children() + assets = node.get_assets() + queryset.extend(list(children)) + + for asset in assets: + node = Node() + node.id = asset.id + node.parent = node.id + node.value = asset.hostname + queryset.append(node) + return queryset + + class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): queryset = Node.objects.all() permission_classes = (IsSuperUser,) @@ -69,14 +98,54 @@ class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): status=201, ) - def get(self, request, *args, **kwargs): - instance = self.get_object() - if self.request.query_params.get("all"): - children = instance.get_all_children() + def get_object(self): + pk = self.kwargs.get('pk') or self.request.query_params.get('id') + if not pk: + node = Node.root() else: - children = instance.get_children() - response = [{"id": node.id, "key": node.key, "value": node.value} for node in children] - return Response(response, status=200) + node = get_object_or_404(Node, pk=pk) + return node + + def get_queryset(self): + queryset = [] + query_all = self.request.query_params.get("all") + query_assets = self.request.query_params.get('assets') + node = self.get_object() + if node == Node.root(): + queryset.append(node) + if query_all: + children = node.get_all_children() + else: + children = node.get_children() + + queryset.extend(list(children)) + if query_assets: + assets = node.get_assets() + for asset in assets: + node_fake = Node() + node_fake.id = asset.id + node_fake.parent = node + node_fake.value = asset.hostname + node_fake.is_asset = True + queryset.append(node_fake) + return queryset + + def get(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + + +class NodeAssetsApi(generics.ListAPIView): + permission_classes = (IsSuperUser,) + serializer_class = serializers.AssetSerializer + + def get_queryset(self): + node_id = self.kwargs.get('pk') + query_all = self.request.query_params.get('all') + instance = get_object_or_404(Node, pk=node_id) + if query_all: + return instance.get_all_assets() + else: + return instance.get_assets() class NodeAddChildrenApi(generics.UpdateAPIView): @@ -146,4 +215,3 @@ class TestNodeConnectiveApi(APIView): task_name = _("测试节点下资产是否可连接: {}".format(node.name)) task = test_asset_connectability_util.delay(assets, task_name=task_name) return Response({"task": task.id}) - diff --git a/apps/assets/forms/asset.py b/apps/assets/forms/asset.py index a6f488761..f8f187b4d 100644 --- a/apps/assets/forms/asset.py +++ b/apps/assets/forms/asset.py @@ -27,13 +27,16 @@ class AssetCreateForm(forms.ModelForm): 'class': 'select2', 'data-placeholder': _('Admin user') }), 'labels': forms.SelectMultiple(attrs={ - 'class': 'select2', 'data-placeholder': _('Labels') + 'class': 'select2', 'data-placeholder': _('Label') }), 'port': forms.TextInput(), 'domain': forms.Select(attrs={ 'class': 'select2', 'data-placeholder': _('Domain') }), } + labels = { + 'nodes': _("Node"), + } help_texts = { 'hostname': '* required', 'ip': '* required', @@ -57,19 +60,22 @@ class AssetUpdateForm(forms.ModelForm): ] widgets = { 'nodes': forms.SelectMultiple(attrs={ - 'class': 'select2', 'data-placeholder': _('Nodes') + 'class': 'select2', 'data-placeholder': _('Node') }), 'admin_user': forms.Select(attrs={ 'class': 'select2', 'data-placeholder': _('Admin user') }), 'labels': forms.SelectMultiple(attrs={ - 'class': 'select2', 'data-placeholder': _('Labels') + 'class': 'select2', 'data-placeholder': _('Label') }), 'port': forms.TextInput(), 'domain': forms.Select(attrs={ 'class': 'select2', 'data-placeholder': _('Domain') }), } + labels = { + 'nodes': _("Node"), + } help_texts = { 'hostname': '* required', 'ip': '* required', @@ -116,10 +122,10 @@ class AssetBulkUpdateForm(forms.ModelForm): ] widgets = { 'labels': forms.SelectMultiple( - attrs={'class': 'select2', 'data-placeholder': _('Select labels')} + attrs={'class': 'select2', 'data-placeholder': _('Label')} ), 'nodes': forms.SelectMultiple( - attrs={'class': 'select2', 'data-placeholder': _('Select nodes')} + attrs={'class': 'select2', 'data-placeholder': _('Node')} ), } diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 4aaa0287d..96bd598b0 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -84,7 +84,7 @@ class Asset(models.Model): comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment')) def __str__(self): - return self.hostname + return '{0.hostname}({0.ip})'.format(self) @property def is_valid(self): @@ -101,6 +101,10 @@ class Asset(models.Model): else: return False + def get_nodes(self): + from .node import Node + return self.nodes.all() or [Node.root()] + @property def hardware_info(self): if self.cpu_count: diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 5ce195783..99236f781 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -16,6 +16,8 @@ class Node(models.Model): child_mark = models.IntegerField(default=0) date_create = models.DateTimeField(auto_now_add=True) + is_asset = False + def __str__(self): return self.value @@ -73,6 +75,9 @@ class Node(models.Model): assets = Asset.objects.filter(nodes__in=nodes) return assets + def has_assets(self): + return self.get_all_assets() + def get_all_active_assets(self): return self.get_all_assets().filter(is_active=True) diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 541ef8b6a..bf31b8491 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -3,6 +3,7 @@ # import logging +import uuid from django.core.cache import cache from django.db import models @@ -100,14 +101,15 @@ class SystemUser(AssetUser): ) nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes")) + assets = models.ManyToManyField('assets.Asset', blank=True, verbose_name=_("Assets")) priority = models.IntegerField(default=10, verbose_name=_("Priority")) protocol = models.CharField(max_length=16, choices=PROTOCOL_CHOICES, default='ssh', verbose_name=_('Protocol')) auto_push = models.BooleanField(default=True, verbose_name=_('Auto push')) - sudo = models.TextField(default='/sbin/ifconfig', verbose_name=_('Sudo')) + sudo = models.TextField(default='/bin/whoami', verbose_name=_('Sudo')) shell = models.CharField(max_length=64, default='/bin/bash', verbose_name=_('Shell')) def __str__(self): - return self.name + return '{0.name}({0.username})'.format(self) def to_json(self): return { @@ -119,11 +121,8 @@ class SystemUser(AssetUser): 'auto_push': self.auto_push, } - @property - def assets(self): - assets = set() - for node in self.nodes.all(): - assets.update(set(node.get_all_assets())) + def get_assets(self): + assets = set(self.assets.all()) return assets @property @@ -168,6 +167,3 @@ class SystemUser(AssetUser): except IntegrityError: print('Error continue') continue - - - diff --git a/apps/assets/serializers/node.py b/apps/assets/serializers/node.py index f6654aef9..736b06c7e 100644 --- a/apps/assets/serializers/node.py +++ b/apps/assets/serializers/node.py @@ -42,7 +42,7 @@ class NodeSerializer(serializers.ModelSerializer): class Meta: model = Node - fields = ['id', 'key', 'value', 'parent', 'assets_amount'] + fields = ['id', 'key', 'value', 'parent', 'assets_amount', 'is_asset'] list_serializer_class = BulkListSerializer @staticmethod diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index 1dff79422..7abd09d29 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -34,7 +34,7 @@ class SystemUserSerializer(serializers.ModelSerializer): @staticmethod def get_assets_amount(obj): - return len(obj.assets) + return len(obj.get_assets()) class SystemUserAuthSerializer(AuthSerializer): diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index fe5508720..06cd9f63e 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- # - +from collections import defaultdict from django.db.models.signals import post_save, m2m_changed from django.dispatch import receiver from common.utils import get_logger from .models import Asset, SystemUser, Node from .tasks import update_assets_hardware_info_util, \ - test_asset_connectability_util, push_system_user_to_node, \ - push_node_system_users_to_asset + test_asset_connectability_util, push_system_user_to_assets logger = get_logger(__file__) @@ -31,7 +30,6 @@ def set_asset_root_node(asset): @receiver(post_save, sender=Asset, dispatch_uid="my_unique_identifier") def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): - set_asset_root_node(instance) if created: logger.info("Asset `{}` create signal received".format(instance)) update_asset_hardware_info_on_created(instance) @@ -41,25 +39,39 @@ def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): @receiver(post_save, sender=SystemUser, dispatch_uid="my_unique_identifier") def on_system_user_update(sender, instance=None, created=True, **kwargs): if instance and not created: - for node in instance.nodes.all(): - push_system_user_to_node(instance, node) + logger.info("System user `{}` update signal received".format(instance)) + assets = instance.assets.all() + push_system_user_to_assets.delay(instance, assets) @receiver(m2m_changed, sender=SystemUser.nodes.through) -def on_system_user_node_change(sender, instance=None, **kwargs): +def on_system_user_nodes_change(sender, instance=None, **kwargs): if instance and kwargs["action"] == "post_add": - for pk in kwargs['pk_set']: - node = kwargs['model'].objects.get(pk=pk) - push_system_user_to_node(instance, node) + assets = set() + nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + for node in nodes: + assets.update(set(node.get_all_assets())) + instance.assets.add(*tuple(assets)) + + +@receiver(m2m_changed, sender=SystemUser.assets.through) +def on_system_user_assets_change(sender, instance=None, **kwargs): + if instance and kwargs["action"] == "post_add": + assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + push_system_user_to_assets(instance, assets) @receiver(m2m_changed, sender=Asset.nodes.through) def on_asset_node_changed(sender, instance=None, **kwargs): if isinstance(instance, Asset) and kwargs['action'] == 'post_add': logger.debug("Asset node change signal received") - for pk in kwargs['pk_set']: - node = kwargs['model'].objects.get(pk=pk) - push_node_system_users_to_asset(node, [instance]) + nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + system_users_assets = defaultdict(set) + system_users = SystemUser.objects.filter(nodes__in=nodes) + for system_user in system_users: + system_users_assets[system_user].update({instance}) + for system_user, assets in system_users_assets.items(): + system_user.assets.add(*tuple(assets)) @receiver(m2m_changed, sender=Asset.nodes.through) @@ -67,5 +79,6 @@ def on_node_assets_changed(sender, instance=None, **kwargs): if isinstance(instance, Node) and kwargs['action'] == 'post_add': logger.debug("Node assets change signal received") assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) - push_node_system_users_to_asset(instance, assets) - + system_users = SystemUser.objects.filter(nodes=instance) + for system_user in system_users: + system_user.assets.add(*tuple(assets)) diff --git a/apps/assets/tasks.py b/apps/assets/tasks.py index 53a54764d..381e1a4cf 100644 --- a/apps/assets/tasks.py +++ b/apps/assets/tasks.py @@ -276,7 +276,7 @@ def test_system_user_connectability_util(system_user, task_name): :return: """ from ops.utils import update_or_create_ansible_task - assets = system_user.assets + assets = system_user.get_assets() hosts = [asset.hostname for asset in assets if asset.is_active and asset.is_unixlike()] tasks = const.TEST_SYSTEM_USER_CONN_TASKS if not hosts: @@ -386,52 +386,17 @@ def push_system_user_util(system_users, assets, task_name): return task.run() -def get_node_push_system_user_task_name(system_user, node): - - # return _("Push system user to node: {} => {}").format( - return _("推送系统用户到节点资产: {} => {}").format( - system_user.name, - node.value - ) - - -@shared_task -def push_system_user_to_node(system_user, node): - logger.info("Start push system user node: {} => {}".format(system_user.name, node.value)) - assets = node.get_all_assets() - task_name = get_node_push_system_user_task_name(system_user, node) - push_system_user_util([system_user], assets, task_name) - - -@shared_task -def push_system_user_related_nodes(system_user): - if not system_user.is_need_push(): - msg = "push system user `{}` passed, may be not auto push or ssh " \ - "protocol is not ssh".format(system_user.name) - logger.info(msg) - return - - nodes = system_user.nodes.all() - for node in nodes: - push_system_user_to_node(system_user, node) - - @shared_task def push_system_user_to_assets_manual(system_user): - push_system_user_related_nodes(system_user) + assets = system_user.get_assets() + task_name = "推送系统用户到入资产: {}".format(system_user.name) + return push_system_user_util([system_user], assets, task_name=task_name) -def push_node_system_users_to_asset(node, assets): - system_users = [] - nodes = node.ancestor_with_node - # 获取该节点所有父节点有的系统用户, 然后推送 - for n in nodes: - system_users.extend(list(n.systemuser_set.all())) - - if system_users: - # task_name = _("Push system users to node: {}").format(node.value) - task_name = _("推送节点系统用户到新加入资产中: {}").format(node.value) - push_system_user_util.delay(system_users, assets, task_name) +@shared_task +def push_system_user_to_assets(system_user, assets): + task_name = _("推送系统用户到入资产: {}").format(system_user.name) + return push_system_user_util.delay([system_user], assets, task_name) # @shared_task diff --git a/apps/assets/templates/assets/asset_create.html b/apps/assets/templates/assets/asset_create.html index 7f01e0530..eca0f6a03 100644 --- a/apps/assets/templates/assets/asset_create.html +++ b/apps/assets/templates/assets/asset_create.html @@ -34,7 +34,7 @@