From 1fe18e807370da740dcb7b4957424871c573cdac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=80=81=E5=B9=BF?= Date: Fri, 23 Aug 2019 18:23:07 +0800 Subject: [PATCH] Bugfix (#3153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] 修改一部分bug * [Update] 修改一些 --- apps/assets/api/asset.py | 6 +- apps/assets/api/node.py | 13 +- apps/assets/models/asset.py | 33 +---- apps/assets/models/node.py | 132 +++++++++--------- apps/assets/models/user.py | 2 + apps/assets/signals_handler.py | 90 ++++++------ apps/assets/views/asset.py | 4 +- .../authentication/_access_key_modal.html | 4 +- apps/orgs/signals_handler.py | 2 +- apps/perms/models/asset_permission.py | 1 + apps/perms/signals_handler.py | 11 +- apps/perms/utils/asset_permission.py | 20 +-- apps/perms/views/asset_permission.py | 3 +- 13 files changed, 146 insertions(+), 175 deletions(-) diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index f24768217..a6280ff56 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -63,15 +63,15 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet): show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true') # 当前节点是顶层节点, 并且仅显示直接资产 - if node.is_root() and show_current_asset: + if node.is_org_root() and show_current_asset: queryset = queryset.filter( Q(nodes=node_id) | Q(nodes__isnull=True) ).distinct() # 当前节点是顶层节点，显示所有资产 - elif node.is_root() and not show_current_asset: + elif node.is_org_root() and not show_current_asset: return queryset # 当前节点不是鼎城节点，只显示直接资产 - elif not node.is_root() and show_current_asset: + elif not node.is_org_root() and show_current_asset: queryset = queryset.filter(nodes=node) else: children = node.get_all_children(with_self=True) diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 83b7ec2da..ae1311971 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -50,13 +50,13 @@ class NodeViewSet(OrgModelViewSet): # 仅支持根节点指直接创建，子节点下的节点需要通过children接口创建 def perform_create(self, serializer): - child_key = Node.root().get_next_child_key() + child_key = Node.org_root().get_next_child_key() serializer.validated_data["key"] = child_key serializer.save() def perform_update(self, serializer): node = self.get_object() - if node.is_root() and node.value != serializer.validated_data['value']: + if node.is_org_root() and node.value != serializer.validated_data['value']: msg = _("You can't update the root node name") raise ValidationError({"error": msg}) return super().perform_update(serializer) @@ -97,6 +97,7 @@ class NodeChildrenApi(generics.ListCreateAPIView): permission_classes = (IsOrgAdmin,) serializer_class = serializers.NodeSerializer instance = None + is_initial = False def initial(self, request, *args, **kwargs): self.instance = self.get_object() @@ -117,7 +118,8 @@ class NodeChildrenApi(generics.ListCreateAPIView): pk = self.kwargs.get('pk') or self.request.query_params.get('id') key = self.request.query_params.get("key") if not pk and not key: - node = Node.root() + node = Node.org_root() + self.is_initial = True return node if pk: node = get_object_or_404(Node, pk=pk) @@ -130,7 +132,7 @@ class NodeChildrenApi(generics.ListCreateAPIView): if not self.instance: return Node.objects.none() - if self.instance.is_root(): + if self.is_initial: with_self = True else: with_self = False @@ -234,7 +236,7 @@ class NodeRemoveAssetsApi(generics.UpdateAPIView): def perform_update(self, serializer): assets = serializer.validated_data.get('assets') instance = self.get_object() - if instance != Node.root(): + if instance != Node.org_root(): instance.assets.remove(*tuple(assets)) else: assets = [asset for asset in assets if asset.nodes.count() > 1] @@ -287,5 +289,4 @@ class RefreshAssetsAmount(APIView): model = Node def get(self, request, *args, **kwargs): - self.model.expire_nodes_assets_amount() return Response("Ok") diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 1c398e49d..06c3b18d8 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -6,8 +6,7 @@ import uuid import logging import random from functools import reduce -from collections import OrderedDict, defaultdict -from django.core.cache import cache +from collections import OrderedDict from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -32,7 +31,7 @@ def default_cluster(): def default_node(): try: from .node import Node - root = Node.root() + root = Node.org_root() return root except: return None @@ -103,33 +102,11 @@ class NodesRelationMixin: id = "" _all_nodes_keys = None - @classmethod - def get_all_nodes_keys(cls): - """ - :return: {asset.id: [node.key, ]} - """ - from .node import Node - cache_key = cls.ALL_ASSET_NODES_CACHE_KEY - cached = cache.get(cache_key) - if cached: - return cached - assets = Asset.objects.all().only('id').prefetch_related( - models.Prefetch('nodes', queryset=Node.objects.all().only('key')) - ) - assets_nodes_keys = {} - for asset in assets: - assets_nodes_keys[asset.id] = [n.key for n in asset.nodes.all()] - cache.set(cache_key, assets_nodes_keys, cls.CACHE_TIME) - return assets_nodes_keys - - @classmethod - def expire_all_nodes_keys_cache(cls): - cache_key = cls.ALL_ASSET_NODES_CACHE_KEY - cache.delete(cache_key) - def get_nodes(self): from .node import Node - nodes = self.nodes.all() or [Node.root()] + nodes = self.nodes.all() + if not nodes: + nodes = Node.objects.filter(id=Node.org_root().id) return nodes def get_all_nodes(self, flat=False): diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 3694f9f7d..889d1de1f 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -24,40 +24,34 @@ class NodeQuerySet(models.QuerySet): class TreeMixin: - time_tree_updated = None - time_tree_updated_cache_key = 'NODE_TREE_CREATED_AT' - tree_cache_time = 3600 + tree_created_time = None + tree_updated_time_cache_key = 'NODE_TREE_CREATED_AT' + tree_update_time_cache_time = 3600 _tree_service = None @classmethod def tree(cls): - # Todo: 有待优化, 因为每次刷新都会导致其他节点的tree失效 - # Todo: ungroup node + # Todo: 有待优化, 因为每次刷新都会导致其他节点的tree失效 完成 # TOdo: 游离的资产，在树上显示的数量不对 - # Todo: api key页面有bug + # Todo: ungroup node + # Todo: api key页面有bug 完成 from ..utils import TreeService - cache_updated_time = cls.get_cache_time() - if not cls.time_tree_updated or \ - cache_updated_time != cls.time_tree_updated: - t = TreeService.new() - cls.update_cache_tree(t) - return t + tree_updated_time = cache.get(cls.tree_updated_time_cache_key, 0) + if not cls.tree_created_time or \ + tree_updated_time > cls.tree_created_time: + print("New tree") + tree = TreeService.new() + cls.tree_created_time = time.time() + cls._tree_service = tree + return tree return cls._tree_service - @classmethod - def get_cache_time(cls): - return cache.get(cls.time_tree_updated_cache_key) - - @classmethod - def update_cache_tree(cls, t): - cls._tree_service = t - now = time.time() - cls.time_tree_updated = now - cache.set(cls.time_tree_updated_cache_key, now, cls.tree_cache_time) - @classmethod def expire_cache_tree(cls): - cache.delete(cls.time_tree_updated_cache_key) + key = cls.tree_updated_time_cache_key + ttl = cls.tree_update_time_cache_time + value = time.time() + cache.set(key, value, ttl) @classmethod def refresh_tree(cls): @@ -74,6 +68,20 @@ class FamilyMixin: __all_children = None is_node = True + @staticmethod + def clean_children_keys(nodes_keys): + nodes_keys = sorted(list(nodes_keys), key=lambda x: (len(x), x)) + nodes_keys_clean = [] + for key in nodes_keys[::-1]: + found = False + for k in nodes_keys: + if key.startswith(k + ':'): + found = True + break + if not found: + nodes_keys_clean.append(key) + return nodes_keys_clean + @property def children(self): return self.get_children(with_self=False) @@ -108,7 +116,7 @@ class FamilyMixin: @property def parent(self): - if self.is_root(): + if self.is_org_root(): return self parent_key = self.parent_key return Node.objects.get(key=parent_key) @@ -122,10 +130,10 @@ class FamilyMixin: old_key = self.key with transaction.atomic(): self.key = parent.get_next_child_key() + self.save() for child in children: child.key = child.key.replace(old_key, self.key, 1) child.save() - self.save() def get_siblings(self, with_self=False): key = ':'.join(self.key.split(':')[:-1]) @@ -182,21 +190,17 @@ class FullValueMixin: @property def full_value(self): - if self.is_root(): + if self.is_org_root(): return self.value if self._full_value is not None: return self._full_value - print("Get full value") value = self._tree.get_node_full_tag(self.key) return value class NodeAssetsMixin: - _assets_amount_cache_key = '_NODE_ASSETS_AMOUNT_{}' - _assets_cache_key = '_NODE_ASSETS_{}' _assets_amount = None key = '' - cache_time = 3600 * 24 * 7 id = None @property @@ -210,24 +214,19 @@ class NodeAssetsMixin: amount = self._tree.assets_amount(self.key) return amount - # TOdo: 是否依赖tree def get_all_assets(self): from .asset import Asset - if self.is_root(): + if self.is_org_root(): return Asset.objects.filter(org_id=self.org_id) - assets_ids = self._tree.all_assets(self.key) - return Asset.objects.filter(id__in=assets_ids) - - def assets_ids(self): - assets_ids = self._tree.assets(self.key) - return assets_ids + pattern = '^{0}$|^{0}:'.format(self.key) + return Asset.objects.filter(nodes__key__regex=pattern).distinct() def get_assets(self): from .asset import Asset - if self.is_default_node(): - assets = Asset.objects.filter(Q(nodes__id=self.id) | Q(nodes__isnull=True)) + if self.is_org_root(): + assets = Asset.objects.filter(Q(nodes=self) | Q(nodes__isnull=True)) else: - assets = Asset.objects.filter(id=self.assets_ids()) + assets = Asset.objects.filter(nodes=self) return assets.distinct() def get_valid_assets(self): @@ -236,6 +235,16 @@ class NodeAssetsMixin: def get_all_valid_assets(self): return self.get_all_assets().valid() + @classmethod + def get_nodes_all_assets(cls, nodes_keys): + from .asset import Asset + nodes_keys = cls.clean_children_keys(nodes_keys) + pattern = set() + for key in nodes_keys: + pattern.add(r'^{0}$|^{0}:'.format(key)) + pattern = '|'.join(list(pattern)) + return Asset.objects.filter(nodes__key__regex=pattern) + class Node(OrgModelMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) @@ -261,22 +270,13 @@ class Node(OrgModelMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixi return self.id == other.id def __gt__(self, other): - # if self.is_root() and not other.is_root(): - # return False - # elif not self.is_root() and other.is_root(): - # return True self_key = [int(k) for k in self.key.split(':')] other_key = [int(k) for k in other.key.split(':')] self_parent_key = self_key[:-1] other_parent_key = other_key[:-1] - if self_parent_key and other_parent_key and \ - self_parent_key == other_parent_key: + if self_parent_key and self_parent_key == other_parent_key: return self.value > other.value - # if len(self_parent_key) < len(other_parent_key): - # return True - # elif len(self_parent_key) > len(other_parent_key): - # return False return self_key > other_key def __lt__(self, other): @@ -310,7 +310,9 @@ class Node(OrgModelMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixi def create_child(self, value, _id=None): with transaction.atomic(): child_key = self.get_next_child_key() - child = self.__class__.objects.create(id=_id, key=child_key, value=value) + child = self.__class__.objects.create( + id=_id, key=child_key, value=value + ) return child @classmethod @@ -318,37 +320,39 @@ class Node(OrgModelMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixi cls.refresh_tree() def is_default_node(self): - return self.is_root() and self.key == '1' + return self.key == '1' - def is_root(self): + def is_org_root(self): if self.key.isdigit(): return True else: return False @classmethod - def create_root_node(cls): + def create_org_root_node(cls): # 如果使用current_org 在set_current_org时会死循环 - _current_org = get_current_org() + ori_org = get_current_org() with transaction.atomic(): - if not _current_org.is_real(): + if not ori_org.is_real(): return cls.default_node() set_current_org(Organization.root()) org_nodes_roots = cls.objects.filter(key__regex=r'^[0-9]+$') - org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True) or ['1'] + org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True) + if not org_nodes_roots_keys: + org_nodes_roots_keys = ['1'] key = max([int(k) for k in org_nodes_roots_keys]) key = str(key + 1) if key != 0 else '2' - set_current_org(_current_org) - root = cls.objects.create(key=key, value=_current_org.name) + set_current_org(ori_org) + root = cls.objects.create(key=key, value=ori_org.name) return root @classmethod - def root(cls): + def org_root(cls): root = cls.objects.filter(key__regex=r'^[0-9]+$') if root: return root[0] else: - return cls.create_root_node() + return cls.create_org_root_node() @classmethod def default_node(cls): @@ -365,7 +369,7 @@ class Node(OrgModelMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixi 'title': name, 'pId': self.parent_key, 'isParent': True, - 'open': self.is_root(), + 'open': self.is_org_root(), 'meta': { 'node': { "id": self.id, diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 61a353e67..ce2af149b 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -148,9 +148,11 @@ class SystemUser(AssetUser): return True, None def get_all_assets(self): + from .node import Node args = [Q(systemuser=self)] pattern = set() nodes_keys = self.nodes.all().values_list('key', flat=True) + nodes_keys = Node.clean_children_keys(nodes_keys) for key in nodes_keys: pattern.add(r'^{0}$|^{0}:'.format(key)) pattern = '|'.join(list(pattern)) diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 85b24e416..266fe44bf 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- # from collections import defaultdict -from django.db.models.signals import post_save, m2m_changed, pre_delete +from django.db.models.signals import ( + post_save, m2m_changed, pre_delete, pre_save, pre_init, post_init +) from django.dispatch import receiver from common.utils import get_logger @@ -30,6 +32,9 @@ def test_asset_conn_on_created(asset): @receiver(post_save, sender=Asset, dispatch_uid="my_unique_identifier") @on_transaction_commit def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): + """ + 当资产创建时，更新硬件信息，更新可连接性 + """ if created: logger.info("Asset `{}` create signal received".format(instance)) @@ -37,78 +42,81 @@ def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): update_asset_hardware_info_on_created(instance) test_asset_conn_on_created(instance) - # 过期节点资产数量 - Node.refresh_nodes() - @receiver(pre_delete, sender=Asset, dispatch_uid="my_unique_identifier") def on_asset_delete(sender, instance=None, **kwargs): - # 过期节点资产数量 + """ + 当资产删除时，刷新节点，节点中存在节点和资产的关系 + """ Node.refresh_nodes() @receiver(post_save, sender=SystemUser, dispatch_uid="my_unique_identifier") def on_system_user_update(sender, instance=None, created=True, **kwargs): + """ + 当系统用户更新时，可能更新了秘钥，用户名等，这时要自动推送系统用户到资产上, + 其实应该当 用户名，密码，秘钥 sudo等更新时再推送，这里偷个懒, + 这里直接取了 instance.assets 因为nodes和系统用户发生变化时，会自动将nodes下的资产 + 关联到上面 + """ if instance and not created: logger.info("System user `{}` update signal received".format(instance)) - assets = instance.assets.all() + assets = instance.assets.all().valid() push_system_user_to_assets.delay(instance, assets) -# @receiver(m2m_changed, sender=SystemUser.nodes.through) -# def on_system_user_nodes_change(sender, instance=None, **kwargs): -# if instance and kwargs["action"] == "post_add": -# logger.info("System user `{}` nodes update signal received".format(instance)) -# assets = set() -# nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) -# for node in nodes: -# assets.update(set(node.get_all_assets())) -# instance.assets.add(*tuple(assets)) -# - -@receiver(m2m_changed, sender=SystemUser.assets.through) +@receiver(m2m_changed, sender=SystemUser.assets.through, dispatch_uid="my_unique_identifier") def on_system_user_assets_change(sender, instance=None, **kwargs): + """ + 当系统用户和资产关系发生变化时，应该重新推送系统用户到新添加的资产中 + """ if instance and kwargs["action"] == "post_add": assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) push_system_user_to_assets.delay(instance, assets) -@receiver(m2m_changed, sender=Asset.nodes.through) -def on_asset_node_changed(sender, instance=None, **kwargs): - logger.debug("Asset nodes change signal received") - Asset.expire_all_nodes_keys_cache() +@receiver(m2m_changed, sender=SystemUser.nodes.through, dispatch_uid="my_unique_identifier") +def on_system_user_nodes_change(sender, instance=None, **kwargs): + """ + 当系统用户和节点关系发生变化时，应该将节点关联到新的系统用户上 + """ + if instance and kwargs["action"] == "post_add": + logger.info("System user `{}` nodes update signal received".format(instance)) + nodes_keys = kwargs['model'].objects.filter( + pk__in=kwargs['pk_set'] + ).values_list('key', flat=True) + assets = Node.get_nodes_all_assets(nodes_keys) + instance.assets.add(*tuple(assets)) + + +@receiver(m2m_changed, sender=Asset.nodes.through, dispatch_uid="my_unique_identifier") +def on_asset_nodes_changed(sender, instance=None, **kwargs): + """ + 当资产的节点发生变化时，或者 当节点的资产关系发生变化时， + 节点下新增的资产，添加到节点关联的系统用户中 + 并刷新节点 + """ if isinstance(instance, Asset): - # nodes = [] - # if kwargs['action'] == 'pre_remove': - # nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + logger.debug("Asset nodes change signal received: {}".format(instance)) + # 节点资产发生变化时，将资产关联到节点关联的系统用户 if kwargs['action'] == 'post_add': nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) system_users_assets = defaultdict(set) system_users = SystemUser.objects.filter(nodes__in=nodes) for system_user in system_users: - system_users_assets[system_user].update({instance}) + system_users_assets[system_user].add(instance) for system_user, assets in system_users_assets.items(): system_user.assets.add(*tuple(assets)) - Node.refresh_nodes() - - -@receiver(m2m_changed, sender=Asset.nodes.through) -def on_node_assets_changed(sender, instance=None, **kwargs): if isinstance(instance, Node): - logger.debug("Node assets change signal {} received".format(instance)) - # 当节点和资产关系发生改变时，过期资产数量缓存 - assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) - if kwargs['action'] == 'post_add': - # 重新关联系统用户和资产的关系 - system_users = SystemUser.objects.filter(nodes=instance) - for system_user in system_users: - system_user.assets.add(*tuple(assets)) + logger.debug("Node assets change signal received: {}".format(instance)) + + Node.refresh_nodes() @receiver(post_save, sender=Node) def on_node_update_or_created(sender, instance=None, created=False, **kwargs): - if instance and not created: - Node.refresh_nodes() + # 刷新节点 + Node.refresh_nodes() @receiver(post_save, sender=AuthBook) diff --git a/apps/assets/views/asset.py b/apps/assets/views/asset.py index 1fcfd553a..257691fa4 100644 --- a/apps/assets/views/asset.py +++ b/apps/assets/views/asset.py @@ -32,7 +32,7 @@ class AssetListView(PermissionsMixin, TemplateView): permission_classes = [IsOrgAdmin] def get_context_data(self, **kwargs): - Node.root() + Node.org_root() context = { 'app': _('Assets'), 'action': _('Asset list'), @@ -85,7 +85,7 @@ class AssetCreateView(PermissionsMixin, FormMixin, TemplateView): if node_id: node = get_object_or_none(Node, id=node_id) else: - node = Node.root() + node = Node.org_root() form["nodes"].initial = node return form diff --git a/apps/authentication/templates/authentication/_access_key_modal.html b/apps/authentication/templates/authentication/_access_key_modal.html index 1bfcc8315..2d46ed727 100644 --- a/apps/authentication/templates/authentication/_access_key_modal.html +++ b/apps/authentication/templates/authentication/_access_key_modal.html @@ -40,7 +40,7 @@