jumpserver/apps/assets/api/asset.py
Jiangjie.Bai 7cf6e54f01
refactor tree (重构&优化资产树/用户授权树加载速度) (#5548) (#5549)
* Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) (#5548)

* tree: v0.1

* tree: v0.2

* tree: v0.3

* tree: v0.4

* tree: 添加并发锁未请求到时的debug日志

* 以空间换时间的方式优化资产树

* Reactor tree togther v2 (#5576)

* Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) (#5548)

* tree: v0.1

* tree: v0.2

* tree: v0.3

* tree: v0.4

* tree: 添加并发锁未请求到时的debug日志

* 以空间换时间的方式优化资产树

* 修改授权适配新方案

* 添加树处理工具

* 完成新的用户授权树计算以及修改一些信号

* 重构了获取资产的一些 api

* 重构了一些节点的api

* 整理了一些代码

* 完成了api 的重构

* 重构检查节点数量功能

* 完成重构授权树工具类

* api 添加强制刷新参数

* 整理一些信号

* 处理一些信号的问题

* 完成了信号的处理

* 重构了资产树相关的锁机制

* RebuildUserTreeTask 还得添加回来

* 优化下不能在root组织的检查函数

* 优化资产树变化时锁的使用

* 修改一些算法的小工具

* 资产树锁不再校验是否在具体组织里

* 整理了一些信号的位置

* 修复资产与节点关系维护的bug

* 去掉一些调试代码

* 修复资产授权过期检查刷新授权树的 bug

* 添加了可重入锁

* 添加一些计时,优化一些sql

* 增加 union 查询的支持

* 尝试用 sql 解决节点资产数量问题

* 开始优化计算授权树节点资产数量不用冗余表

* 新代码能跑起来了,修复一下bug

* 去掉 UserGrantedMappingNode 换成 UserAssetGrantedTreeNodeRelation

* 修了些bug,做了些优化

* 优化QuerySetStage 执行逻辑

* 与小白的内存结合了

* 删掉老的表,迁移新的 assets_amount 字段

* 优化用户授权页面资产列表 count 慢

* 修复批量命令数量不对

* 修改获取非直接授权节点的 children 的逻辑

* 获取整棵树的节点

* 回退锁

* 整理迁移脚本

* 改变更新树策略

* perf: 修改一波缩进

* fix: 修改handler名称

* 修复授权树获取资产sql 泛滥

* 修复授权规则有效bug

* 修复一些bug

* 修复一些bug

* 又修了一些小bug

* 去掉了老的 get_nodes_all_assets

* 修改一些写法

* Reactor tree togther b2 (#5570)

* fix: 修改handler名称

* perf: 优化生成树

* perf: 去掉注释

* 优化了一些

* 重新生成迁移脚本

* 去掉周期检查节点资产数量的任务

* Pr@reactor tree togther guang@perf mapping (#5573)

* fix: 修改handler名称

* perf: mapping 拆分出来

* 修改名称

* perf: 修改锁名

* perf: 去掉检查节点任务

* perf: 修改一下名称

* perf: 优化一波

Co-authored-by: Jiangjie.Bai <32935519+BaiJiangJie@users.noreply.github.com>
Co-authored-by: Bai <bugatti_it@163.com>
Co-authored-by: xinwen <coderWen@126.com>

Co-authored-by: xinwen <coderWen@126.com>
Co-authored-by: 老广 <ibuler@qq.com>
2021-02-05 13:29:29 +08:00

145 lines
4.8 KiB
Python

# -*- coding: utf-8 -*-
#
from assets.api import FilterAssetByNodeMixin
from rest_framework.viewsets import ModelViewSet
from rest_framework.generics import RetrieveAPIView
from django.shortcuts import get_object_or_404
from django.utils.decorators import method_decorator
from assets.locks import NodeTreeUpdateLock
from common.utils import get_logger, get_object_or_none
from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser, IsSuperUser
from orgs.mixins.api import OrgBulkModelViewSet
from orgs.mixins import generics
from ..models import Asset, Node, Platform
from .. import serializers
from ..tasks import (
update_assets_hardware_info_manual, test_assets_connectivity_manual
)
from ..filters import FilterAssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend
logger = get_logger(__file__)
__all__ = [
'AssetViewSet', 'AssetPlatformRetrieveApi',
'AssetGatewayListApi', 'AssetPlatformViewSet',
'AssetTaskCreateApi', 'AssetsTaskCreateApi',
]
class AssetViewSet(FilterAssetByNodeMixin, OrgBulkModelViewSet):
"""
API endpoint that allows Asset to be viewed or edited.
"""
model = Asset
filterset_fields = {
'hostname': ['exact'],
'ip': ['exact'],
'systemuser__id': ['exact'],
'admin_user__id': ['exact'],
'platform__base': ['exact'],
'is_active': ['exact'],
'protocols': ['exact', 'icontains']
}
search_fields = ("hostname", "ip")
ordering_fields = ("hostname", "ip", "port", "cpu_cores")
serializer_classes = {
'default': serializers.AssetSerializer,
'display': serializers.AssetDisplaySerializer,
}
permission_classes = (IsOrgAdminOrAppUser,)
extra_filter_backends = [FilterAssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend]
def set_assets_node(self, assets):
if not isinstance(assets, list):
assets = [assets]
node_id = self.request.query_params.get('node_id')
if not node_id:
return
node = get_object_or_none(Node, pk=node_id)
if not node:
return
node.assets.add(*assets)
def perform_create(self, serializer):
assets = serializer.save()
self.set_assets_node(assets)
class AssetPlatformRetrieveApi(RetrieveAPIView):
queryset = Platform.objects.all()
permission_classes = (IsOrgAdminOrAppUser,)
serializer_class = serializers.PlatformSerializer
def get_object(self):
asset_pk = self.kwargs.get('pk')
asset = get_object_or_404(Asset, pk=asset_pk)
return asset.platform
class AssetPlatformViewSet(ModelViewSet):
queryset = Platform.objects.all()
permission_classes = (IsSuperUser,)
serializer_class = serializers.PlatformSerializer
filterset_fields = ['name', 'base']
search_fields = ['name']
def get_permissions(self):
if self.request.method.lower() in ['get', 'options']:
self.permission_classes = (IsOrgAdmin,)
return super().get_permissions()
def check_object_permissions(self, request, obj):
if request.method.lower() in ['delete', 'put', 'patch'] and obj.internal:
self.permission_denied(
request, message={"detail": "Internal platform"}
)
return super().check_object_permissions(request, obj)
class AssetsTaskMixin:
def perform_assets_task(self, serializer):
data = serializer.validated_data
assets = data['assets']
action = data['action']
if action == "refresh":
task = update_assets_hardware_info_manual.delay(assets)
else:
task = test_assets_connectivity_manual.delay(assets)
data = getattr(serializer, '_data', {})
data["task"] = task.id
setattr(serializer, '_data', data)
def perform_create(self, serializer):
self.perform_assets_task(serializer)
class AssetTaskCreateApi(AssetsTaskMixin, generics.CreateAPIView):
model = Asset
serializer_class = serializers.AssetTaskSerializer
permission_classes = (IsOrgAdmin,)
def create(self, request, *args, **kwargs):
pk = self.kwargs.get('pk')
request.data['assets'] = [pk]
return super().create(request, *args, **kwargs)
class AssetsTaskCreateApi(AssetsTaskMixin, generics.CreateAPIView):
model = Asset
serializer_class = serializers.AssetTaskSerializer
permission_classes = (IsOrgAdmin,)
class AssetGatewayListApi(generics.ListAPIView):
permission_classes = (IsOrgAdminOrAppUser,)
serializer_class = serializers.GatewayWithAuthSerializer
def get_queryset(self):
asset_id = self.kwargs.get('pk')
asset = get_object_or_404(Asset, pk=asset_id)
if not asset.domain:
return []
queryset = asset.domain.gateways.filter(protocol='ssh')
return queryset