fix: 修复 celery 等日志文件的访问漏洞 (#5475)

Co-authored-by: xinwen <coderWen@126.com>

Dockerfile

@@ -11,6 +11,8 @@ RUN cd utils && bash -ixeu build.sh
 FROM registry.fit2cloud.com/public/python:v3
 ARG PIP_MIRROR=https://pypi.douban.com/simple
 ENV PIP_MIRROR=$PIP_MIRROR
+ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
+ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR
 ARG MYSQL_MIRROR=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql57-community-el6/
 ENV MYSQL_MIRROR=$MYSQL_MIRROR
 
@@ -18,13 +20,16 @@ WORKDIR /opt/jumpserver
 
 COPY ./requirements ./requirements
 RUN useradd jumpserver
+RUN wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-6.repo \
+    && sed -i 's@/centos/@/centos-vault/@g' /etc/yum.repos.d/CentOS-Base.repo \
+    && sed -i 's@$releasever@6.10@g' /etc/yum.repos.d/CentOS-Base.repo
 RUN yum -y install epel-release && \
       echo -e "[mysql]\nname=mysql\nbaseurl=${MYSQL_MIRROR}\ngpgcheck=0\nenabled=1" > /etc/yum.repos.d/mysql.repo
 RUN yum -y install $(cat requirements/rpm_requirements.txt)
-RUN pip install --upgrade pip setuptools==49.6.0 wheel -i ${PIP_MIRROR} && \
+RUN pip install --upgrade pip==20.2.4 setuptools==49.6.0 wheel -i ${PIP_MIRROR} && \
     pip config set global.index-url ${PIP_MIRROR}
-RUN pip install $(grep 'jms' requirements/requirements.txt) -i https://pypi.org/simple
-RUN pip install -r requirements/requirements.txt
+RUN pip install --no-cache-dir $(grep 'jms' requirements/requirements.txt) -i ${PIP_JMS_MIRROR}
+RUN pip install --no-cache-dir -r requirements/requirements.txt
 
 COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
 RUN mkdir -p /root/.ssh/ && echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config

apps/applications/serializers/database_app.py

@@ -12,9 +12,8 @@ from .. import models
 class DBAttrsSerializer(serializers.Serializer):
     host = serializers.CharField(max_length=128, label=_('Host'))
     port = serializers.IntegerField(label=_('Port'))
-    database = serializers.CharField(
-        max_length=128, required=False, allow_blank=True, allow_null=True, label=_('Database')
-    )
+    # 添加allow_null=True，兼容之前数据库中database字段为None的情况
+    database = serializers.CharField(max_length=128, required=True, allow_null=True, label=_('Database'))
 
 
 class MySQLAttrsSerializer(DBAttrsSerializer):

apps/assets/api/node.py

@@ -173,7 +173,7 @@ class NodeChildrenAsTreeApi(SerializeToTreeNodeMixin, NodeChildrenApi):
             return []
         assets = self.instance.get_assets().only(
             "id", "hostname", "ip", "os",
-            "org_id", "protocols",
+            "org_id", "protocols", "is_active"
         )
         return self.serialize_assets(assets, self.instance.key)
 
@@ -201,10 +201,8 @@ class NodeAddChildrenApi(generics.UpdateAPIView):
     def put(self, request, *args, **kwargs):
         instance = self.get_object()
         nodes_id = request.data.get("nodes")
-        children = [get_object_or_none(Node, id=pk) for pk in nodes_id]
+        children = Node.objects.filter(id__in=nodes_id)
         for node in children:
-            if not node:
-                continue
             node.parent = instance
         return Response("OK")
 

apps/assets/models/node.py

@@ -103,7 +103,7 @@ class FamilyMixin:
             if value is None:
                 value = child_key
             child = self.__class__.objects.create(
-                id=_id, key=child_key, value=value, parent_key=self.key,
+                id=_id, key=child_key, value=value
             )
             return child
 
@@ -354,7 +354,8 @@ class SomeNodesMixin:
     def org_root(cls):
         root = cls.objects.filter(parent_key='')\
             .filter(key__regex=r'^[0-9]+$')\
-            .exclude(key__startswith='-')
+            .exclude(key__startswith='-')\
+            .order_by('key')
         if root:
             return root[0]
         else:
@@ -411,7 +412,7 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin):
 
     class Meta:
         verbose_name = _("Node")
-        ordering = ['value']
+        ordering = ['parent_key', 'value']
 
     def __str__(self):
         return self.full_value

apps/assets/signals_handler.py

@@ -4,7 +4,7 @@ from operator import add, sub
 
 from assets.utils import is_asset_exists_in_node
 from django.db.models.signals import (
-    post_save, m2m_changed, pre_delete, post_delete
+    post_save, m2m_changed, pre_delete, post_delete, pre_save
 )
 from django.db.models import Q, F
 from django.dispatch import receiver
@@ -37,6 +37,11 @@ def test_asset_conn_on_created(asset):
     test_asset_connectivity_util.delay([asset])
 
 
+@receiver(pre_save, sender=Node)
+def on_node_pre_save(sender, instance: Node, **kwargs):
+    instance.parent_key = instance.compute_parent_key()
+
+
 @receiver(post_save, sender=Asset)
 @on_transaction_commit
 def on_asset_created_or_update(sender, instance=None, created=False, **kwargs):
@@ -73,6 +78,7 @@ def on_system_user_update(instance: SystemUser, created, **kwargs):
 
 
 @receiver(m2m_changed, sender=SystemUser.assets.through)
+@on_transaction_commit
 def on_system_user_assets_change(instance, action, model, pk_set, **kwargs):
     """
     当系统用户和资产关系发生变化时，应该重新推送系统用户到新添加的资产中
@@ -91,25 +97,29 @@ def on_system_user_assets_change(instance, action, model, pk_set, **kwargs):
 
 
 @receiver(m2m_changed, sender=SystemUser.users.through)
-def on_system_user_users_change(sender, instance=None, action='', model=None, pk_set=None, **kwargs):
+@on_transaction_commit
+def on_system_user_users_change(sender, instance: SystemUser, action, model, pk_set, reverse, **kwargs):
     """
     当系统用户和用户关系发生变化时，应该重新推送系统用户资产中
     """
     if action != POST_ADD:
         return
+
+    if reverse:
+        raise M2MReverseNotAllowed
+
     if not instance.username_same_with_user:
         return
+
     logger.debug("System user users change signal recv: {}".format(instance))
-    queryset = model.objects.filter(pk__in=pk_set)
-    if model == SystemUser:
-        system_users = queryset
-    else:
-        system_users = [instance]
-    for s in system_users:
-        push_system_user_to_assets_manual.delay(s)
+    usernames = model.objects.filter(pk__in=pk_set).values_list('username', flat=True)
+
+    for username in usernames:
+        push_system_user_to_assets_manual.delay(instance, username)
 
 
 @receiver(m2m_changed, sender=SystemUser.nodes.through)
+@on_transaction_commit
 def on_system_user_nodes_change(sender, instance=None, action=None, model=None, pk_set=None, **kwargs):
     """
     当系统用户和节点关系发生变化时，应该将节点下资产关联到新的系统用户上

apps/assets/tasks/nodes_amount.py

@@ -1,14 +1,13 @@
 from celery import shared_task
 
+from ops.celery.decorator import register_as_period_task
 from assets.utils import check_node_assets_amount
 from common.utils import get_logger
-from common.utils.timezone import now
 
 logger = get_logger(__file__)
 
 
-@shared_task()
+@register_as_period_task(crontab='0 2 * * *')
+@shared_task(queue='celery_heavy_tasks')
 def check_node_assets_amount_celery_task():
-    logger.info(f'>>> {now()} begin check_node_assets_amount_celery_task ...')
     check_node_assets_amount()
-    logger.info(f'>>> {now()} end check_node_assets_amount_celery_task ...')

apps/assets/tasks/push_system_user.py

@@ -2,13 +2,13 @@
 
 from itertools import groupby
 from celery import shared_task
-from common.db.utils import get_object_if_need, get_objects_if_need, get_objects
+from common.db.utils import get_object_if_need, get_objects
 from django.utils.translation import ugettext as _
 from django.db.models import Empty
 
 from common.utils import encrypt_password, get_logger
-from assets.models import SystemUser, Asset
-from orgs.utils import org_aware_func
+from assets.models import SystemUser, Asset, AuthBook
+from orgs.utils import org_aware_func, tmp_to_root_org
 from . import const
 from .utils import clean_ansible_task_hosts, group_asset_by_platform
 
@@ -190,15 +190,12 @@ def get_push_system_user_tasks(system_user, platform="unixlike", username=None):
 @org_aware_func("system_user")
 def push_system_user_util(system_user, assets, task_name, username=None):
     from ops.utils import update_or_create_ansible_task
-    hosts = clean_ansible_task_hosts(assets, system_user=system_user)
-    if not hosts:
+    assets = clean_ansible_task_hosts(assets, system_user=system_user)
+    if not assets:
         return {}
 
-    platform_hosts_map = {}
-    hosts_sorted = sorted(hosts, key=group_asset_by_platform)
-    platform_hosts = groupby(hosts_sorted, key=group_asset_by_platform)
-    for i in platform_hosts:
-        platform_hosts_map[i[0]] = list(i[1])
+    assets_sorted = sorted(assets, key=group_asset_by_platform)
+    platform_hosts = groupby(assets_sorted, key=group_asset_by_platform)
 
     def run_task(_tasks, _hosts):
         if not _tasks:
@@ -209,27 +206,59 @@ def push_system_user_util(system_user, assets, task_name, username=None):
         )
         task.run()
 
-    for platform, _hosts in platform_hosts_map.items():
-        if not _hosts:
+    if system_user.username_same_with_user:
+        if username is None:
+            # 动态系统用户，但是没有指定 username
+            usernames = list(system_user.users.all().values_list('username', flat=True).distinct())
+        else:
+            usernames = [username]
+    else:
+        # 非动态系统用户指定 username 无效
+        assert username is None, 'Only Dynamic user can assign `username`'
+        usernames = [system_user.username]
+
+    for platform, _assets in platform_hosts:
+        _assets = list(_assets)
+        if not _assets:
             continue
         print(_("Start push system user for platform: [{}]").format(platform))
-        print(_("Hosts count: {}").format(len(_hosts)))
+        print(_("Hosts count: {}").format(len(_assets)))
 
-        # 如果没有特殊密码设置，就不需要单独推送某台机器了
-        if not system_user.has_special_auth(username=username):
-            logger.debug("System user not has special auth")
-            tasks = get_push_system_user_tasks(system_user, platform, username=username)
-            run_task(tasks, _hosts)
-            continue
+        id_asset_map = {_asset.id: _asset for _asset in _assets}
+        assets_id = id_asset_map.keys()
+        no_special_auth = []
+        special_auth_set = set()
 
-        for _host in _hosts:
-            system_user.load_asset_special_auth(_host, username=username)
-            tasks = get_push_system_user_tasks(system_user, platform, username=username)
-            run_task(tasks, [_host])
+        auth_books = AuthBook.objects.filter(username__in=usernames, asset_id__in=assets_id)
+
+        for auth_book in auth_books:
+            special_auth_set.add((auth_book.username, auth_book.asset_id))
+
+        for _username in usernames:
+            no_special_assets = []
+            for asset_id in assets_id:
+                if (_username, asset_id) not in special_auth_set:
+                    no_special_assets.append(id_asset_map[asset_id])
+            if no_special_assets:
+                no_special_auth.append((_username, no_special_assets))
+
+        for _username, no_special_assets in no_special_auth:
+            tasks = get_push_system_user_tasks(system_user, platform, username=_username)
+            run_task(tasks, no_special_assets)
+
+        for auth_book in auth_books:
+            system_user._merge_auth(auth_book)
+            tasks = get_push_system_user_tasks(system_user, platform, username=auth_book.username)
+            asset = id_asset_map[auth_book.asset_id]
+            run_task(tasks, [asset])
 
 
 @shared_task(queue="ansible")
+@tmp_to_root_org()
 def push_system_user_to_assets_manual(system_user, username=None):
+    """
+    将系统用户推送到与它关联的所有资产上
+    """
     system_user = get_object_if_need(SystemUser, system_user)
     assets = system_user.get_related_assets()
     task_name = _("Push system users to assets: {}").format(system_user.name)
@@ -237,7 +266,11 @@ def push_system_user_to_assets_manual(system_user, username=None):
 
 
 @shared_task(queue="ansible")
+@tmp_to_root_org()
 def push_system_user_a_asset_manual(system_user, asset, username=None):
+    """
+    将系统用户推送到一个资产上
+    """
     if username is None:
         username = system_user.username
     task_name = _("Push system users to asset: {}({}) => {}").format(
@@ -247,10 +280,15 @@ def push_system_user_a_asset_manual(system_user, asset, username=None):
 
 
 @shared_task(queue="ansible")
+@tmp_to_root_org()
 def push_system_user_to_assets(system_user_id, assets_id, username=None):
+    """
+    推送系统用户到指定的若干资产上
+    """
     system_user = SystemUser.objects.get(id=system_user_id)
     assets = get_objects(Asset, assets_id)
     task_name = _("Push system users to assets: {}").format(system_user.name)
+
     return push_system_user_util(system_user, assets, task_name, username=username)
 
 # @shared_task

apps/assets/utils.py

@@ -1,5 +1,7 @@
 # ~*~ coding: utf-8 ~*~
 #
+import time
+
 from django.db.models import Q
 
 from common.utils import get_logger, dict_get_any, is_uuid, get_object_or_none
@@ -12,15 +14,18 @@ logger = get_logger(__file__)
 
 def check_node_assets_amount():
     for node in Node.objects.all():
+        logger.info(f'Check node assets amount: {node}')
         assets_amount = Asset.objects.filter(
             Q(nodes__key__istartswith=f'{node.key}:') | Q(nodes=node)
         ).distinct().count()
 
         if node.assets_amount != assets_amount:
-            print(f'>>> <Node:{node.key}> wrong assets amount '
-                  f'{node.assets_amount} right is {assets_amount}')
+            logger.warn(f'Node wrong assets amount <Node:{node.key}> '
+                        f'{node.assets_amount} right is {assets_amount}')
             node.assets_amount = assets_amount
             node.save()
+        # 防止自检程序给数据库的压力太大
+        time.sleep(0.1)
 
 
 def is_asset_exists_in_node(asset_pk, node_key):

apps/authentication/api/auth.py

@@ -54,12 +54,3 @@ class UserConnectionTokenApi(RootOrgViewMixin, APIView):
             return Response(value)
         else:
             return Response({'user': value['user']})
-
-    def get_permissions(self):
-        if self.request.query_params.get('user-only', None):
-            self.permission_classes = (AllowAny,)
-        return super().get_permissions()
-
-
-
-

apps/ops/celery/__init__.py

@@ -29,16 +29,3 @@ configs["CELERY_ROUTES"] = {
 app.namespace = 'CELERY'
 app.conf.update(configs)
 app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])
-
-app.conf.beat_schedule = {
-    'check-asset-permission-expired': {
-        'task': 'perms.tasks.check_asset_permission_expired',
-        'schedule': settings.PERM_EXPIRED_CHECK_PERIODIC,
-        'args': ()
-    },
-    'check-node-assets-amount': {
-        'task': 'assets.tasks.nodes_amount.check_node_assets_amount_celery_task',
-        'schedule': crontab(minute=0, hour=0),
-        'args': ()
-    },
-}

apps/ops/celery/utils.py

@@ -2,6 +2,7 @@
 #
 import json
 import os
+import uuid
 
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
@@ -101,6 +102,10 @@ def get_celery_periodic_task(task_name):
 
 def get_celery_task_log_path(task_id):
     task_id = str(task_id)
+    try:
+        uuid.UUID(task_id)
+    except:
+        return
     rel_path = os.path.join(task_id[0], task_id[1], task_id + '.log')
     path = os.path.join(settings.CELERY_LOG_DIR, rel_path)
     os.makedirs(os.path.dirname(path), exist_ok=True)

apps/ops/ws.py

@@ -15,7 +15,11 @@ class CeleryLogWebsocket(JsonWebsocketConsumer):
     disconnected = False
 
     def connect(self):
-        self.accept()
+        user = self.scope["user"]
+        if user.is_authenticated:
+            self.accept()
+        else:
+            self.close()
 
     def receive(self, text_data=None, bytes_data=None, **kwargs):
         data = json.loads(text_data)

apps/orgs/api.py

@@ -92,6 +92,7 @@ class OrgMemberAdminRelationBulkViewSet(JMSBulkRelationModelViewSet):
     serializer_class = OrgMemberAdminSerializer
     filterset_class = OrgMemberRelationFilterSet
     search_fields = ('user__name', 'user__username', 'org__name')
+    lookup_field = 'user_id'
 
     def get_queryset(self):
         queryset = super().get_queryset()
@@ -116,6 +117,7 @@ class OrgMemberUserRelationBulkViewSet(JMSBulkRelationModelViewSet):
     serializer_class = OrgMemberUserSerializer
     filterset_class = OrgMemberRelationFilterSet
     search_fields = ('user__name', 'user__username', 'org__name')
+    lookup_field = 'user_id'
 
     def get_queryset(self):
         queryset = super().get_queryset()

apps/orgs/middleware.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 #
 
-from .models import Organization
 from .utils import get_org_from_request, set_current_org
 
 

apps/orgs/serializers.py

@@ -74,26 +74,29 @@ class OrgMemberSerializer(BulkModelSerializer):
         ).distinct()
 
 
-class OrgMemberAdminSerializer(BulkModelSerializer):
+class OrgMemberOldBaseSerializer(BulkModelSerializer):
+    organization = serializers.PrimaryKeyRelatedField(
+        label=_('Organization'), queryset=Organization.objects.all(), required=True, source='org'
+    )
+
+    def to_internal_value(self, data):
+        view = self.context['view']
+        org_id = view.kwargs.get('org_id')
+        if org_id:
+            data['organization'] = org_id
+        return super().to_internal_value(data)
+
+    class Meta:
+        model = OrganizationMember
+        fields = ('id', 'organization', 'user', 'role')
+
+
+class OrgMemberAdminSerializer(OrgMemberOldBaseSerializer):
     role = serializers.HiddenField(default=ROLE.ADMIN)
-    organization = serializers.PrimaryKeyRelatedField(
-        label=_('Organization'), queryset=Organization.objects.all(), required=True, source='org'
-    )
-
-    class Meta:
-        model = OrganizationMember
-        fields = ('id', 'organization', 'user', 'role')
 
 
-class OrgMemberUserSerializer(BulkModelSerializer):
+class OrgMemberUserSerializer(OrgMemberOldBaseSerializer):
     role = serializers.HiddenField(default=ROLE.USER)
-    organization = serializers.PrimaryKeyRelatedField(
-        label=_('Organization'), queryset=Organization.objects.all(), required=True, source='org'
-    )
-
-    class Meta:
-        model = OrganizationMember
-        fields = ('id', 'organization', 'user', 'role')
 
 
 class OrgRetrieveSerializer(OrgReadSerializer):

apps/orgs/signals_handler.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 #
+from collections import defaultdict
+from functools import partial
 
 from django.db.models.signals import m2m_changed
 from django.db.models.signals import post_save
@@ -7,10 +9,10 @@ from django.dispatch import receiver
 
 from orgs.utils import tmp_to_org
 from .models import Organization, OrganizationMember
-from .hands import set_current_org, current_org, Node, get_current_org
-from perms.models import (AssetPermission, DatabaseAppPermission,
-                          K8sAppPermission, RemoteAppPermission)
-from users.models import UserGroup
+from .hands import set_current_org, Node, get_current_org
+from perms.models import (AssetPermission, ApplicationPermission)
+from users.models import UserGroup, User
+from common.const.signals import PRE_REMOVE, POST_REMOVE
 
 
 @receiver(post_save, sender=Organization)
@@ -34,11 +36,44 @@ def _remove_users(model, users, org):
             users = (users, )
 
         m2m_model = model.users.through
-        if model.users.reverse:
+        reverse = model.users.reverse
+        if reverse:
             m2m_field_name = model.users.field.m2m_reverse_field_name()
         else:
             m2m_field_name = model.users.field.m2m_field_name()
-        m2m_model.objects.filter(**{'user__in': users, f'{m2m_field_name}__org_id': org.id}).delete()
+        relations = m2m_model.objects.filter(**{
+            'user__in': users,
+            f'{m2m_field_name}__org_id': org.id
+        })
+
+        object_id_users_id_map = defaultdict(set)
+
+        m2m_field_attr_name = f'{m2m_field_name}_id'
+        for relation in relations:
+            object_id = getattr(relation, m2m_field_attr_name)
+            object_id_users_id_map[object_id].add(relation.user_id)
+
+        objects = model.objects.filter(id__in=object_id_users_id_map.keys())
+        send_m2m_change_signal = partial(
+            m2m_changed.send,
+            sender=m2m_model, reverse=reverse, model=User, using=model.objects.db
+        )
+
+        for obj in objects:
+            send_m2m_change_signal(
+                instance=obj,
+                pk_set=object_id_users_id_map[obj.id],
+                action=PRE_REMOVE
+            )
+
+        relations.delete()
+
+        for obj in objects:
+            send_m2m_change_signal(
+                instance=obj,
+                pk_set=object_id_users_id_map[obj.id],
+                action=POST_REMOVE
+            )
 
 
 def _clear_users_from_org(org, users):
@@ -48,8 +83,7 @@ def _clear_users_from_org(org, users):
     if not users:
         return
 
-    models = (AssetPermission, DatabaseAppPermission,
-              RemoteAppPermission, K8sAppPermission, UserGroup)
+    models = (AssetPermission, ApplicationPermission, UserGroup)
 
     for m in models:
         _remove_users(m, users, org)

apps/perms/api/asset/user_group_permission.py

@@ -32,9 +32,6 @@ class UserGroupMixin:
 
 
 class UserGroupGrantedAssetsApi(ListAPIView):
-    """
-    获取用户组直接授权的资产
-    """
     permission_classes = (IsOrgAdminOrAppUser,)
     serializer_class = serializers.AssetGrantedSerializer
     only_fields = serializers.AssetGrantedSerializer.Meta.only_fields
@@ -44,11 +41,27 @@ class UserGroupGrantedAssetsApi(ListAPIView):
     def get_queryset(self):
         user_group_id = self.kwargs.get('pk', '')
 
-        return Asset.objects.filter(
-            Q(granted_by_permissions__user_groups__id=user_group_id)
+        asset_perms_id = list(AssetPermission.objects.valid().filter(
+            user_groups__id=user_group_id
+        ).distinct().values_list('id', flat=True))
+
+        granted_node_keys = Node.objects.filter(
+            granted_by_permissions__id__in=asset_perms_id,
+        ).distinct().values_list('key', flat=True)
+
+        granted_q = Q()
+        for _key in granted_node_keys:
+            granted_q |= Q(nodes__key__startswith=f'{_key}:')
+            granted_q |= Q(nodes__key=_key)
+
+        granted_q |= Q(granted_by_permissions__id__in=asset_perms_id)
+
+        assets = Asset.objects.filter(
+            granted_q
         ).distinct().only(
             *self.only_fields
         )
+        return assets
 
 
 class UserGroupGrantedNodeAssetsApi(ListAPIView):
@@ -66,7 +79,7 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView):
         granted = AssetPermission.objects.filter(
             user_groups__id=user_group_id,
             nodes__id=node_id
-        ).exists()
+        ).valid().exists()
         if granted:
             assets = Asset.objects.filter(
                 Q(nodes__key__startswith=f'{node.key}:') |
@@ -74,8 +87,12 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView):
             )
             return assets
         else:
+            asset_perms_id = list(AssetPermission.objects.valid().filter(
+                user_groups__id=user_group_id
+            ).distinct().values_list('id', flat=True))
+
             granted_node_keys = Node.objects.filter(
-                granted_by_permissions__user_groups__id=user_group_id,
+                granted_by_permissions__id__in=asset_perms_id,
                 key__startswith=f'{node.key}:'
             ).distinct().values_list('key', flat=True)
 
@@ -85,7 +102,7 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView):
                 granted_node_q |= Q(nodes__key=_key)
 
             granted_asset_q = (
-                Q(granted_by_permissions__user_groups__id=user_group_id) &
+                Q(granted_by_permissions__id__in=asset_perms_id) &
                 (
                     Q(nodes__key__startswith=f'{node.key}:') |
                     Q(nodes__key=node.key)
@@ -129,12 +146,16 @@ class UserGroupGrantedNodeChildrenAsTreeApi(SerializeToTreeNodeMixin, ListAPIVie
         group_id = self.kwargs.get('pk')
         node_key = self.request.query_params.get('key', None)
 
+        asset_perms_id = list(AssetPermission.objects.valid().filter(
+            user_groups__id=group_id
+        ).distinct().values_list('id', flat=True))
+
         granted_keys = Node.objects.filter(
-            granted_by_permissions__user_groups__id=group_id
+            granted_by_permissions__id__in=asset_perms_id
         ).values_list('key', flat=True)
 
         asset_granted_keys = Node.objects.filter(
-            assets__granted_by_permissions__user_groups__id=group_id
+            assets__granted_by_permissions__id__in=asset_perms_id
         ).values_list('key', flat=True)
 
         if node_key is None:

apps/perms/api/asset/user_permission/user_permission_assets.py

@@ -3,6 +3,7 @@
 from perms.api.asset.user_permission.mixin import UserNodeGrantStatusDispatchMixin
 from rest_framework.generics import ListAPIView
 from rest_framework.response import Response
+from rest_framework.request import Request
 from django.conf import settings
 
 from assets.api.mixin import SerializeToTreeNodeMixin
@@ -55,8 +56,12 @@ class AssetsAsTreeMixin(SerializeToTreeNodeMixin):
     """
     将 资产 序列化成树的结构返回
     """
-    def list(self, request, *args, **kwargs):
+    def list(self, request: Request, *args, **kwargs):
         queryset = self.filter_queryset(self.get_queryset())
+        if request.query_params.get('search'):
+            # 如果用户搜索的条件不精准，会导致返回大量的无意义数据。
+            # 这里限制一下返回数据的最大条数
+            queryset = queryset[:999]
         data = self.serialize_assets(queryset, None)
         return Response(data=data)
 

apps/perms/signals_handler.py

@@ -6,7 +6,7 @@ from django.dispatch import receiver
 from perms.tasks import create_rebuild_user_tree_task, \
     create_rebuild_user_tree_task_by_related_nodes_or_assets
 from users.models import User, UserGroup
-from assets.models import Asset
+from assets.models import Asset, SystemUser
 from common.utils import get_logger
 from common.exceptions import M2MReverseNotAllowed
 from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR
@@ -16,6 +16,42 @@ from .models import AssetPermission, RemoteAppPermission
 logger = get_logger(__file__)
 
 
+def handle_rebuild_user_tree(instance, action, reverse, pk_set, **kwargs):
+    if action.startswith('post'):
+        if reverse:
+            create_rebuild_user_tree_task(pk_set)
+        else:
+            create_rebuild_user_tree_task([instance.id])
+
+
+def handle_bind_groups_systemuser(instance, action, reverse, pk_set, **kwargs):
+    """
+    UserGroup 增加 User 时，增加的 User 需要与 UserGroup 关联的动态系统用户相关联
+    """
+    user: User
+
+    if action != POST_ADD:
+        return
+
+    if not reverse:
+        # 一个用户添加了多个用户组
+        users_id = [instance.id]
+        system_users = SystemUser.objects.filter(groups__id__in=pk_set).distinct()
+    else:
+        # 一个用户组添加了多个用户
+        users_id = pk_set
+        system_users = SystemUser.objects.filter(groups__id=instance.pk).distinct()
+
+    for system_user in system_users:
+        system_user.users.add(*users_id)
+
+
+@receiver(m2m_changed, sender=User.groups.through)
+def on_user_groups_change(**kwargs):
+    handle_rebuild_user_tree(**kwargs)
+    handle_bind_groups_systemuser(**kwargs)
+
+
 @receiver([pre_save], sender=AssetPermission)
 def on_asset_perm_deactive(instance: AssetPermission, **kwargs):
     try:

apps/perms/tasks.py

@@ -5,10 +5,12 @@ from datetime import timedelta
 from django.db import transaction
 from django.db.models import Q
 from django.db.transaction import atomic
+from django.conf import settings
 from celery import shared_task
 from common.utils import get_logger
 from common.utils.timezone import now, dt_formater, dt_parser
 from users.models import User
+from ops.celery.decorator import register_as_period_task
 from assets.models import Node
 from perms.models import RebuildUserTreeTask, AssetPermission
 from perms.utils.asset.user_permission import rebuild_user_mapping_nodes_if_need_with_lock, lock
@@ -33,7 +35,8 @@ def dispatch_mapping_node_tasks():
         rebuild_user_mapping_nodes_celery_task.delay(id)
 
 
-@shared_task(queue='check_asset_perm_expired')
+@register_as_period_task(interval=settings.PERM_EXPIRED_CHECK_PERIODIC)
+@shared_task(queue='celery_check_asset_perm_expired')
 @atomic()
 def check_asset_permission_expired():
     """

apps/perms/urls/asset_permission.py

@@ -21,7 +21,7 @@ user_permission_urlpatterns = [
     # ---------------------------------------------------------
     # 以 serializer 格式返回
     path('<uuid:pk>/assets/', api.UserAllGrantedAssetsApi.as_view(), name='user-assets'),
-    path('assets/', api.MyAllAssetsAsTreeApi.as_view(), name='my-assets'),
+    path('assets/', api.MyAllGrantedAssetsApi.as_view(), name='my-assets'),
 
     # Tree Node 的数据格式返回
     path('<uuid:pk>/assets/tree/', api.UserDirectGrantedAssetsAsTreeForAdminApi.as_view(), name='user-assets-as-tree'),

apps/perms/utils/asset/user_permission.py

@@ -34,27 +34,6 @@ TMP_ASSET_GRANTED_FIELD = '_asset_granted'
 TMP_GRANTED_ASSETS_AMOUNT_FIELD = '_granted_assets_amount'
 
 
-# 使用场景
-# Asset.objects.filter(get_user_resources_q_granted_by_permissions(user))
-def get_user_resources_q_granted_by_permissions(user: User):
-    """
-    获取用户关联的 asset permission 或者 用户组关联的 asset permission 获取规则,
-    前提 AssetPermission 对象中的 related_name 为 granted_by_permissions
-    :param user:
-    :return:
-    """
-    _now = now()
-    return reduce(and_, (
-        Q(granted_by_permissions__date_start__lt=_now),
-        Q(granted_by_permissions__date_expired__gt=_now),
-        Q(granted_by_permissions__is_active=True),
-        (
-            Q(granted_by_permissions__users=user) |
-            Q(granted_by_permissions__user_groups__users=user)
-        )
-    ))
-
-
 # 使用场景
 # `Node.objects.annotate(**node_annotate_mapping_node)`
 node_annotate_mapping_node = {
@@ -215,7 +194,7 @@ def compute_tmp_mapping_node_from_perm(user: User, asset_perms_id=None):
     return [*leaf_nodes, *ancestors]
 
 
-def create_mapping_nodes(user, nodes, clear=True):
+def create_mapping_nodes(user, nodes):
     to_create = []
     for node in nodes:
         _granted = getattr(node, TMP_GRANTED_FIELD, False)
@@ -231,8 +210,6 @@ def create_mapping_nodes(user, nodes, clear=True):
             assets_amount=_granted_assets_amount,
         ))
 
-    if clear:
-        UserGrantedMappingNode.objects.filter(user=user).delete()
     UserGrantedMappingNode.objects.bulk_create(to_create)
 
 
@@ -254,6 +231,9 @@ def set_node_granted_assets_amount(user, node, asset_perms_id=None):
 @tmp_to_root_org()
 def rebuild_user_mapping_nodes(user):
     logger.info(f'>>> {dt_formater(now())} start rebuild {user} mapping nodes')
+
+    # 先删除旧的授权树🌲
+    UserGrantedMappingNode.objects.filter(user=user).delete()
     asset_perms_id = get_user_all_assetpermissions_id(user)
     if not asset_perms_id:
         # 没有授权直接返回
@@ -384,7 +364,8 @@ def get_node_all_granted_assets(user: User, key):
 
     if only_asset_granted_nodes_qs:
         only_asset_granted_nodes_q = reduce(or_, only_asset_granted_nodes_qs)
-        only_asset_granted_nodes_q &= get_user_resources_q_granted_by_permissions(user)
+        asset_perms_id = get_user_all_assetpermissions_id(user)
+        only_asset_granted_nodes_q &= Q(granted_by_permissions__id__in=list(asset_perms_id))
         q.append(only_asset_granted_nodes_q)
 
     if q:
@@ -484,6 +465,9 @@ def get_user_all_assetpermissions_id(user: User):
     asset_perms_id = AssetPermission.objects.valid().filter(
         Q(users=user) | Q(user_groups__users=user)
     ).distinct().values_list('id', flat=True)
+
+    # !!! 这个很重要，必须转换成 list，避免 Django 生成嵌套子查询
+    asset_perms_id = list(asset_perms_id)
     return asset_perms_id
 
 

apps/tickets/serializers/request_asset_perm.py

@@ -1,5 +1,3 @@
-from itertools import chain
-
 from rest_framework import serializers
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
@@ -9,7 +7,7 @@ from django.db.models import Q
 from common.utils.timezone import dt_parser, dt_formater
 from orgs.utils import tmp_to_root_org
 from orgs.models import Organization, ROLE as ORG_ROLE
-from assets.models.asset import Asset
+from assets.models import Asset, SystemUser
 from users.models.user import User
 from perms.serializers import ActionsField
 from perms.models import Action
@@ -130,12 +128,23 @@ class RequestAssetPermTicketSerializer(serializers.ModelSerializer):
             if hostname:
                 q |= Q(hostname__icontains=hostname)
 
-            data['confirmed_assets'] = list(
-                map(lambda x: str(x), chain(*Asset.objects.filter(q)[0: limit].values_list('id'))))
+            recomand_assets_id = Asset.objects.filter(q)[:limit].values_list('id', flat=True)
+            data['confirmed_assets'] = [str(i) for i in recomand_assets_id]
+
+    def _recommend_system_users(self, data, instance):
+        confirmed_system_users = data.get('confirmed_system_users')
+        if not confirmed_system_users and self._is_assignee(instance):
+            system_user = data.get('system_user')
+
+            recomand_system_users_id = SystemUser.objects.filter(
+                name__icontains=system_user
+            )[:3].values_list('id', flat=True)
+            data['confirmed_system_users'] = [str(i) for i in recomand_system_users_id]
 
     def to_representation(self, instance):
         data = super().to_representation(instance)
         self._recommend_assets(data, instance)
+        self._recommend_system_users(data, instance)
         return data
 
     def _create_body(self, validated_data):

apps/users/api/relation.py

@@ -28,3 +28,12 @@ class UserUserGroupRelationViewSet(JMSBulkRelationModelViewSet):
             return False
         else:
             return True
+
+    def perform_create(self, serializer):
+        validated_data = []
+        for item in serializer.validated_data:
+            if item['user'].role == User.ROLE.AUDITOR:
+                continue
+            validated_data.append(item)
+        serializer._validated_data = validated_data
+        return super().perform_create(serializer)

apps/users/api/user.py

@@ -6,8 +6,8 @@ from rest_framework.decorators import action
 from rest_framework import generics
 from rest_framework.response import Response
 from rest_framework_bulk import BulkModelViewSet
+from django.db.models import Prefetch
 
-from common.db.aggregates import GroupConcat
 from common.permissions import (
     IsOrgAdmin, IsOrgAdminOrAppUser,
     CanUpdateDeleteUser, IsSuperUser
@@ -44,9 +44,18 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet):
     extra_filter_backends = [OrgRoleUserFilterBackend]
 
     def get_queryset(self):
-        return super().get_queryset().annotate(
-            gc_m2m_org_members__role=GroupConcat('m2m_org_members__role'),
-        ).prefetch_related('groups')
+        queryset = super().get_queryset().prefetch_related(
+            'groups'
+        )
+        if current_org.is_real():
+            # 为在列表中计算用户在真实组织里的角色
+            queryset = queryset.prefetch_related(
+                Prefetch(
+                    'm2m_org_members',
+                    queryset=OrganizationMember.objects.filter(org__id=current_org.id)
+                )
+            )
+        return queryset
 
     def send_created_signal(self, users):
         if not isinstance(users, list):

apps/users/models/user.py

@@ -170,22 +170,18 @@ class RoleMixin:
         from orgs.models import ROLE as ORG_ROLE
 
         if not current_org.is_real():
+            # 不是真实的组织，取 User 本身的角色
             if self.is_superuser:
                 return [ORG_ROLE.ADMIN]
             else:
                 return [ORG_ROLE.USER]
 
-        if hasattr(self, 'gc_m2m_org_members__role'):
-            names = self.gc_m2m_org_members__role
-            if isinstance(names, str):
-                roles = set(self.gc_m2m_org_members__role.split(','))
-            else:
-                roles = set()
-        else:
-            roles = set(self.m2m_org_members.filter(
-                org_id=current_org.id
-            ).values_list('role', flat=True))
-        roles = list(roles)
+        # 是真实组织，取 OrganizationMember 中的角色
+        roles = [
+            org_member.role
+            for org_member in self.m2m_org_members.all()
+            if org_member.org_id == current_org.id
+        ]
         roles.sort()
         return roles
 

apps/users/signals_handler.py

@@ -2,14 +2,12 @@
 #
 
 from django.dispatch import receiver
-from django.db.models.signals import m2m_changed
 from django_auth_ldap.backend import populate_user
 from django.conf import settings
 from django_cas_ng.signals import cas_user_authenticated
 
 from jms_oidc_rp.signals import openid_create_or_update_user
 
-from perms.tasks import create_rebuild_user_tree_task
 from common.utils import get_logger
 from .signals import post_user_create
 from .models import User
@@ -27,15 +25,6 @@ def on_user_create(sender, user=None, **kwargs):
         send_user_created_mail(user)
 
 
-@receiver(m2m_changed, sender=User.groups.through)
-def on_user_groups_change(instance, action, reverse, pk_set, **kwargs):
-    if action.startswith('post'):
-        if reverse:
-            create_rebuild_user_tree_task(pk_set)
-        else:
-            create_rebuild_user_tree_task([instance.id])
-
-
 @receiver(cas_user_authenticated)
 def on_cas_user_authenticated(sender, user, created, **kwargs):
     if created:

jms

@@ -156,7 +156,10 @@ def is_running(s, unlink=True):
 
 def parse_service(s):
     web_services = ['gunicorn', 'flower', 'daphne']
-    celery_services = ["celery_ansible", "celery_default", "celery_node_tree", "check_asset_perm_expired"]
+    celery_services = [
+        "celery_ansible", "celery_default", "celery_node_tree",
+        "celery_check_asset_perm_expired", "celery_heavy_tasks"
+    ]
     task_services = celery_services + ['beat']
     all_services = web_services + task_services
     if s == 'all':
@@ -225,9 +228,14 @@ def get_start_celery_node_tree_kwargs():
     return get_start_worker_kwargs('node_tree', 2)
 
 
+def get_start_celery_heavy_tasks_kwargs():
+    print("\n- Start Celery as Distributed Task Queue: HeavyTasks")
+    return get_start_worker_kwargs('celery_heavy_tasks', 1)
+
+
 def get_start_celery_check_asset_perm_expired_kwargs():
     print("\n- Start Celery as Distributed Task Queue: CheckAseetPermissionExpired")
-    return get_start_worker_kwargs('check_asset_perm_expired', 1)
+    return get_start_worker_kwargs('celery_check_asset_perm_expired', 1)
 
 
 def get_start_worker_kwargs(queue, num):
@@ -366,7 +374,8 @@ def start_service(s):
         "celery_ansible": get_start_celery_ansible_kwargs,
         "celery_default": get_start_celery_default_kwargs,
         "celery_node_tree": get_start_celery_node_tree_kwargs,
-        "check_asset_perm_expired": get_start_celery_check_asset_perm_expired_kwargs,
+        "celery_heavy_tasks": get_start_celery_heavy_tasks_kwargs,
+        "celery_check_asset_perm_expired": get_start_celery_check_asset_perm_expired_kwargs,
         "beat": get_start_beat_kwargs,
         "flower": get_start_flower_kwargs,
         "daphne": get_start_daphne_kwargs,

requirements/requirements.txt

@@ -53,6 +53,8 @@ Pillow==7.1.0
 pyasn1==0.4.8
 pycparser==2.19
 pycrypto==2.6.1
+pycryptodome==3.9.9
+pycryptodomex==3.9.9
 pyotp==2.2.6
 PyNaCl==1.2.1
 python-dateutil==2.6.1