1
0
mirror of https://github.com/jumpserver/jumpserver.git synced 2025-05-03 13:46:33 +00:00
jumpserver/apps/assets/signals_handler/common.py
Jiangjie.Bai 7cf6e54f01
refactor tree (重构&优化资产树/用户授权树加载速度) () ()
* Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) ()

* tree: v0.1

* tree: v0.2

* tree: v0.3

* tree: v0.4

* tree: 添加并发锁未请求到时的debug日志

* 以空间换时间的方式优化资产树

* Reactor tree togther v2 ()

* Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) ()

* tree: v0.1

* tree: v0.2

* tree: v0.3

* tree: v0.4

* tree: 添加并发锁未请求到时的debug日志

* 以空间换时间的方式优化资产树

* 修改授权适配新方案

* 添加树处理工具

* 完成新的用户授权树计算以及修改一些信号

* 重构了获取资产的一些 api

* 重构了一些节点的api

* 整理了一些代码

* 完成了api 的重构

* 重构检查节点数量功能

* 完成重构授权树工具类

* api 添加强制刷新参数

* 整理一些信号

* 处理一些信号的问题

* 完成了信号的处理

* 重构了资产树相关的锁机制

* RebuildUserTreeTask 还得添加回来

* 优化下不能在root组织的检查函数

* 优化资产树变化时锁的使用

* 修改一些算法的小工具

* 资产树锁不再校验是否在具体组织里

* 整理了一些信号的位置

* 修复资产与节点关系维护的bug

* 去掉一些调试代码

* 修复资产授权过期检查刷新授权树的 bug

* 添加了可重入锁

* 添加一些计时,优化一些sql

* 增加 union 查询的支持

* 尝试用 sql 解决节点资产数量问题

* 开始优化计算授权树节点资产数量不用冗余表

* 新代码能跑起来了,修复一下bug

* 去掉 UserGrantedMappingNode 换成 UserAssetGrantedTreeNodeRelation

* 修了些bug,做了些优化

* 优化QuerySetStage 执行逻辑

* 与小白的内存结合了

* 删掉老的表,迁移新的 assets_amount 字段

* 优化用户授权页面资产列表 count 慢

* 修复批量命令数量不对

* 修改获取非直接授权节点的 children 的逻辑

* 获取整棵树的节点

* 回退锁

* 整理迁移脚本

* 改变更新树策略

* perf: 修改一波缩进

* fix: 修改handler名称

* 修复授权树获取资产sql 泛滥

* 修复授权规则有效bug

* 修复一些bug

* 修复一些bug

* 又修了一些小bug

* 去掉了老的 get_nodes_all_assets

* 修改一些写法

* Reactor tree togther b2 ()

* fix: 修改handler名称

* perf: 优化生成树

* perf: 去掉注释

* 优化了一些

* 重新生成迁移脚本

* 去掉周期检查节点资产数量的任务

* Pr@reactor tree togther guang@perf mapping ()

* fix: 修改handler名称

* perf: mapping 拆分出来

* 修改名称

* perf: 修改锁名

* perf: 去掉检查节点任务

* perf: 修改一下名称

* perf: 优化一波

Co-authored-by: Jiangjie.Bai <32935519+BaiJiangJie@users.noreply.github.com>
Co-authored-by: Bai <bugatti_it@163.com>
Co-authored-by: xinwen <coderWen@126.com>

Co-authored-by: xinwen <coderWen@126.com>
Co-authored-by: 老广 <ibuler@qq.com>
2021-02-05 13:29:29 +08:00

223 lines
7.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# -*- coding: utf-8 -*-
#
from django.db.models.signals import (
post_save, m2m_changed, pre_delete, post_delete, pre_save
)
from django.dispatch import receiver
from common.exceptions import M2MReverseNotAllowed
from common.const.signals import POST_ADD, POST_REMOVE, PRE_REMOVE
from common.utils import get_logger
from common.decorator import on_transaction_commit
from assets.models import Asset, SystemUser, Node
from users.models import User
from assets.tasks import (
update_assets_hardware_info_util,
test_asset_connectivity_util,
push_system_user_to_assets_manual,
push_system_user_to_assets,
add_nodes_assets_to_system_users
)
logger = get_logger(__file__)
def update_asset_hardware_info_on_created(asset):
logger.debug("Update asset `{}` hardware info".format(asset))
update_assets_hardware_info_util.delay([asset])
def test_asset_conn_on_created(asset):
logger.debug("Test asset `{}` connectivity".format(asset))
test_asset_connectivity_util.delay([asset])
@receiver(pre_save, sender=Node)
def on_node_pre_save(sender, instance: Node, **kwargs):
instance.parent_key = instance.compute_parent_key()
@receiver(post_save, sender=Asset)
@on_transaction_commit
def on_asset_created_or_update(sender, instance=None, created=False, **kwargs):
"""
当资产创建时,更新硬件信息,更新可连接性
确保资产必须属于一个节点
"""
if created:
logger.info("Asset create signal recv: {}".format(instance))
# 获取资产硬件信息
update_asset_hardware_info_on_created(instance)
test_asset_conn_on_created(instance)
# 确保资产存在一个节点
has_node = instance.nodes.all().exists()
if not has_node:
instance.nodes.add(Node.org_root())
@receiver(post_save, sender=SystemUser, dispatch_uid="jms")
@on_transaction_commit
def on_system_user_update(instance: SystemUser, created, **kwargs):
"""
当系统用户更新时,可能更新了秘钥,用户名等,这时要自动推送系统用户到资产上,
其实应该当 用户名,密码,秘钥 sudo等更新时再推送这里偷个懒,
这里直接取了 instance.assets 因为nodes和系统用户发生变化时会自动将nodes下的资产
关联到上面
"""
if instance and not created:
logger.info("System user update signal recv: {}".format(instance))
assets = instance.assets.all().valid()
push_system_user_to_assets.delay(instance.id, [_asset.id for _asset in assets])
@receiver(m2m_changed, sender=SystemUser.assets.through)
@on_transaction_commit
def on_system_user_assets_change(instance, action, model, pk_set, **kwargs):
"""
当系统用户和资产关系发生变化时,应该重新推送系统用户到新添加的资产中
"""
if action != POST_ADD:
return
logger.debug("System user assets change signal recv: {}".format(instance))
if model == Asset:
system_users_id = [instance.id]
assets_id = pk_set
else:
system_users_id = pk_set
assets_id = [instance.id]
for system_user_id in system_users_id:
push_system_user_to_assets.delay(system_user_id, assets_id)
@receiver(m2m_changed, sender=SystemUser.users.through)
@on_transaction_commit
def on_system_user_users_change(sender, instance: SystemUser, action, model, pk_set, reverse, **kwargs):
"""
当系统用户和用户关系发生变化时,应该重新推送系统用户资产中
"""
if action != POST_ADD:
return
if reverse:
raise M2MReverseNotAllowed
if not instance.username_same_with_user:
return
logger.debug("System user users change signal recv: {}".format(instance))
usernames = model.objects.filter(pk__in=pk_set).values_list('username', flat=True)
for username in usernames:
push_system_user_to_assets_manual.delay(instance, username)
@receiver(m2m_changed, sender=SystemUser.nodes.through)
@on_transaction_commit
def on_system_user_nodes_change(sender, instance=None, action=None, model=None, pk_set=None, **kwargs):
"""
当系统用户和节点关系发生变化时,应该将节点下资产关联到新的系统用户上
"""
if action != POST_ADD:
return
logger.info("System user nodes update signal recv: {}".format(instance))
queryset = model.objects.filter(pk__in=pk_set)
if model == Node:
nodes_keys = queryset.values_list('key', flat=True)
system_users = [instance]
else:
nodes_keys = [instance.key]
system_users = queryset
add_nodes_assets_to_system_users.delay(nodes_keys, system_users)
@receiver(m2m_changed, sender=SystemUser.groups.through)
def on_system_user_groups_change(instance, action, pk_set, reverse, **kwargs):
"""
当系统用户和用户组关系发生变化时,应该将组下用户关联到新的系统用户上
"""
if action != POST_ADD:
return
if reverse:
raise M2MReverseNotAllowed
logger.info("System user groups update signal recv: {}".format(instance))
users = User.objects.filter(groups__id__in=pk_set).distinct()
instance.users.add(*users)
@receiver(m2m_changed, sender=Asset.nodes.through)
def on_asset_nodes_add(instance, action, reverse, pk_set, **kwargs):
"""
本操作共访问 4 次数据库
当资产的节点发生变化时,或者 当节点的资产关系发生变化时,
节点下新增的资产,添加到节点关联的系统用户中
"""
if action != POST_ADD:
return
logger.debug("Assets node add signal recv: {}".format(action))
if reverse:
nodes = [instance.key]
asset_ids = pk_set
else:
nodes = Node.objects.filter(pk__in=pk_set).values_list('key', flat=True)
asset_ids = [instance.id]
# 节点资产发生变化时,将资产关联到节点及祖先节点关联的系统用户, 只关注新增的
nodes_ancestors_keys = set()
for node in nodes:
nodes_ancestors_keys.update(Node.get_node_ancestor_keys(node, with_self=True))
# 查询所有祖先节点关联的系统用户,都是要跟资产建立关系的
system_user_ids = SystemUser.objects.filter(
nodes__key__in=nodes_ancestors_keys
).distinct().values_list('id', flat=True)
# 查询所有已存在的关系
m2m_model = SystemUser.assets.through
exist = set(m2m_model.objects.filter(
systemuser_id__in=system_user_ids, asset_id__in=asset_ids
).values_list('systemuser_id', 'asset_id'))
# TODO 优化
to_create = []
for system_user_id in system_user_ids:
asset_ids_to_push = []
for asset_id in asset_ids:
if (system_user_id, asset_id) in exist:
continue
asset_ids_to_push.append(asset_id)
to_create.append(m2m_model(
systemuser_id=system_user_id,
asset_id=asset_id
))
push_system_user_to_assets.delay(system_user_id, asset_ids_to_push)
m2m_model.objects.bulk_create(to_create)
RELATED_NODE_IDS = '_related_node_ids'
@receiver(pre_delete, sender=Asset)
def on_asset_delete(instance: Asset, using, **kwargs):
node_ids = set(Node.objects.filter(
assets=instance
).distinct().values_list('id', flat=True))
setattr(instance, RELATED_NODE_IDS, node_ids)
m2m_changed.send(
sender=Asset.nodes.through, instance=instance, reverse=False,
model=Node, pk_set=node_ids, using=using, action=PRE_REMOVE
)
@receiver(post_delete, sender=Asset)
def on_asset_post_delete(instance: Asset, using, **kwargs):
node_ids = getattr(instance, RELATED_NODE_IDS, None)
if node_ids:
m2m_changed.send(
sender=Asset.nodes.through, instance=instance, reverse=False,
model=Node, pk_set=node_ids, using=using, action=POST_REMOVE
)