Merge pull request #6429 from jumpserver/dev

v2.12.0 rc3

apps/assets/api/accounts.py

@@ -3,6 +3,7 @@ from django.conf import settings
 from rest_framework.decorators import action
 from django_filters import rest_framework as filters
 from rest_framework.response import Response
+from rest_framework.generics import CreateAPIView
 
 from orgs.mixins.api import OrgBulkModelViewSet
 from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser, NeedMFAVerify
@@ -11,7 +12,7 @@ from ..tasks.account_connectivity import test_accounts_connectivity_manual
 from ..models import AuthBook
 from .. import serializers
 
-__all__ = ['AccountViewSet', 'AccountSecretsViewSet']
+__all__ = ['AccountViewSet', 'AccountSecretsViewSet', 'AccountTaskCreateAPI']
 
 
 class AccountFilterSet(BaseFilterSet):
@@ -38,8 +39,6 @@ class AccountFilterSet(BaseFilterSet):
             'asset', 'systemuser', 'id',
         ]
 
-from rest_framework.filters import  SearchFilter
-
 
 class AccountViewSet(OrgBulkModelViewSet):
     model = AuthBook
@@ -79,3 +78,29 @@ class AccountSecretsViewSet(AccountViewSet):
         if not settings.SECURITY_VIEW_AUTH_NEED_MFA:
             self.permission_classes = [IsOrgAdminOrAppUser]
         return super().get_permissions()
+
+
+class AccountTaskCreateAPI(CreateAPIView):
+    permission_classes = (IsOrgAdminOrAppUser,)
+    serializer_class = serializers.AccountTaskSerializer
+    filterset_fields = AccountViewSet.filterset_fields
+    search_fields = AccountViewSet.search_fields
+    filterset_class = AccountViewSet.filterset_class
+
+    def get_accounts(self):
+        queryset = AuthBook.objects.all()
+        queryset = self.filter_queryset(queryset)
+        return queryset
+
+    def perform_create(self, serializer):
+        accounts = self.get_accounts()
+        task = test_accounts_connectivity_manual.delay(accounts)
+        data = getattr(serializer, '_data', {})
+        data["task"] = task.id
+        setattr(serializer, '_data', data)
+        return task
+
+    def get_exception_handler(self):
+        def handler(e, context):
+            return Response({"error": str(e)}, status=400)
+        return handler

apps/assets/api/system_user_relation.py

@@ -75,7 +75,7 @@ class SystemUserAssetRelationViewSet(BaseRelationViewSet):
     ]
     search_fields = [
         "id", "asset__hostname", "asset__ip",
-        "systemuser__name", "systemuser__username"
+        "systemuser__name", "systemuser__username",
     ]
 
     def get_objects_attr(self):

apps/assets/migrations/0075_auto_20210705_1759.py

@@ -18,7 +18,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='asset',
             name='date_verified',
-            field=models.DateTimeField(null=True),
+            field=models.DateTimeField(null=True, verbose_name='Date verified'),
         ),
         migrations.AddField(
             model_name='authbook',
@@ -28,7 +28,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='authbook',
             name='date_verified',
-            field=models.DateTimeField(null=True),
+            field=models.DateTimeField(null=True, verbose_name='Date verified'),
         ),
         migrations.AddField(
             model_name='historicalauthbook',
@@ -38,7 +38,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='historicalauthbook',
             name='date_verified',
-            field=models.DateTimeField(null=True),
+            field=models.DateTimeField(null=True, verbose_name='Date verified'),
         ),
         migrations.AlterField(
             model_name='asset',

apps/assets/models/authbook.py

@@ -3,10 +3,9 @@
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-
 from simple_history.models import HistoricalRecords
 
-
+from common.utils import lazyproperty
 from .base import BaseUser, AbsConnectivity
 
 __all__ = ['AuthBook']
@@ -17,6 +16,7 @@ class AuthBook(BaseUser, AbsConnectivity):
     systemuser = models.ForeignKey('assets.SystemUser', on_delete=models.CASCADE, null=True, verbose_name=_("System user"))
     version = models.IntegerField(default=1, verbose_name=_('Version'))
     history = HistoricalRecords()
+    _systemuser_display = ''
 
     auth_attrs = ['username', 'password', 'private_key', 'public_key']
 
@@ -63,8 +63,10 @@ class AuthBook(BaseUser, AbsConnectivity):
     def username_display(self):
         return self.get_or_systemuser_attr('username') or '*'
 
-    @property
+    @lazyproperty
     def systemuser_display(self):
+        if self._systemuser_display:
+            return self._systemuser_display
         if not self.systemuser:
             return ''
         return str(self.systemuser)

apps/assets/models/base.py

@@ -37,7 +37,7 @@ class AbsConnectivity(models.Model):
         choices=Connectivity.choices, default=Connectivity.unknown,
         max_length=16, verbose_name=_('Connectivity')
     )
-    date_verified = models.DateTimeField(null=True)
+    date_verified = models.DateTimeField(null=True, verbose_name=_("Date verified"))
 
     def set_connectivity(self, val):
         self.connectivity = val

apps/assets/serializers/account.py

@@ -40,3 +40,11 @@ class AccountSecretSerializer(AccountSerializer):
             'private_key': {'write_only': False},
             'public_key': {'write_only': False},
         }
+
+
+class AccountTaskSerializer(serializers.Serializer):
+    ACTION_CHOICES = (
+        ('test', 'test'),
+    )
+    action = serializers.ChoiceField(choices=ACTION_CHOICES, write_only=True)
+    task = serializers.CharField(read_only=True)

apps/assets/serializers/asset.py

@@ -83,7 +83,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
             'hardware_info', 'connectivity', 'date_verified'
         ]
         fields_fk = [
-            'domain', 'domain_display', 'platform', 'admin_user', 'admin_user_display'
+            'domain', 'domain_display', 'platform', 'admin_user',
         ]
         fields_m2m = [
             'nodes', 'nodes_display', 'labels',
@@ -97,7 +97,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
             'protocol': {'write_only': True},
             'port': {'write_only': True},
             'hardware_info': {'label': _('Hardware info')},
-            'org_name': {'label': _('Org name')}
+            'org_name': {'label': _('Org name')},
         }
 
     def get_fields(self):
@@ -168,6 +168,9 @@ class AssetVerboseSerializer(AssetSerializer):
         queryset=SystemUser.objects, label=_('Admin user')
     )
 
+    class Meta(AssetSerializer.Meta):
+        fields = AssetSerializer.Meta.fields + ['admin_user_display']
+
 
 class PlatformSerializer(serializers.ModelSerializer):
     meta = serializers.DictField(required=False, allow_null=True, label=_('Meta'))

apps/assets/serializers/system_user.py

@@ -23,6 +23,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     """
     auto_generate_key = serializers.BooleanField(initial=True, required=False, write_only=True)
     type_display = serializers.ReadOnlyField(source='get_type_display')
+    ssh_key_fingerprint = serializers.ReadOnlyField(label=_('SSH key fingerprint'))
 
     class Meta:
         model = SystemUser
@@ -30,7 +31,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         fields_write_only = ['password', 'public_key', 'private_key']
         fields_small = fields_mini + fields_write_only + [
             'type', 'type_display', 'protocol', 'login_mode', 'login_mode_display',
-            'priority', 'sudo', 'shell', 'sftp_root', 'token',
+            'priority', 'sudo', 'shell', 'sftp_root', 'token', 'ssh_key_fingerprint',
             'home', 'system_groups', 'ad_domain',
             'username_same_with_user', 'auto_push', 'auto_generate_key',
             'date_created', 'date_updated',
@@ -51,8 +52,8 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         }
 
     def validate_auto_push(self, value):
-        login_mode = self.initial_data.get("login_mode")
+        login_mode = self.get_initial_value("login_mode")
-        protocol = self.initial_data.get("protocol")
+        protocol = self.get_initial_value("protocol")
 
         if login_mode == SystemUser.LOGIN_MANUAL:
             value = False
@@ -61,8 +62,8 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         return value
 
     def validate_auto_generate_key(self, value):
-        login_mode = self.initial_data.get("login_mode")
+        login_mode = self.get_initial_value("login_mode")
-        protocol = self.initial_data.get("protocol")
+        protocol = self.get_initial_value("protocol")
 
         if self.context["request"].method.lower() != "post":
             value = False
@@ -77,7 +78,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     def validate_username_same_with_user(self, username_same_with_user):
         if not username_same_with_user:
             return username_same_with_user
-        protocol = self.initial_data.get("protocol", "ssh")
+        protocol = self.get_initial_value("protocol", "ssh")
         queryset = SystemUser.objects.filter(
             protocol=protocol,
             username_same_with_user=True
@@ -93,9 +94,9 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     def validate_username(self, username):
         if username:
             return username
-        login_mode = self.initial_data.get("login_mode")
+        login_mode = self.get_initial_value("login_mode")
-        protocol = self.initial_data.get("protocol")
+        protocol = self.get_initial_value("protocol")
-        username_same_with_user = self.initial_data.get("username_same_with_user")
+        username_same_with_user = self.get_initial_value("username_same_with_user")
 
         if username_same_with_user:
             return ''
@@ -106,7 +107,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         return username
 
     def validate_home(self, home):
-        username_same_with_user = self.initial_data.get("username_same_with_user")
+        username_same_with_user = self.get_initial_value("username_same_with_user")
         if username_same_with_user:
             return ''
         return home
@@ -119,8 +120,10 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
             raise serializers.ValidationError(error)
         return value
 
-    @staticmethod
+    def validate_admin_user(self, attrs):
-    def validate_admin_user(attrs):
+        if self.instance:
+            tp = self.instance.type
+        else:
             tp = attrs.get('type')
         if tp != SystemUser.Type.admin:
             return attrs
@@ -132,9 +135,9 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
 
     def validate_password(self, password):
         super().validate_password(password)
-        auto_gen_key = self.initial_data.get("auto_generate_key", False)
+        auto_gen_key = self.get_initial_value("auto_generate_key", False)
-        private_key = self.initial_data.get("private_key")
+        private_key = self.get_initial_value("private_key")
-        login_mode = self.initial_data.get("login_mode")
+        login_mode = self.get_initial_value("login_mode")
 
         if not self.instance and not auto_gen_key and not password and \
                 not private_key and login_mode == SystemUser.LOGIN_AUTO:
@@ -179,12 +182,12 @@ class SystemUserListSerializer(SystemUserSerializer):
         fields_small = fields_mini + fields_write_only + [
             'protocol', 'login_mode', 'login_mode_display', 'priority',
             'sudo', 'shell', 'home', 'system_groups',
-            'ad_domain', 'sftp_root',
+            'ad_domain', 'sftp_root', 'ssh_key_fingerprint',
             "username_same_with_user", 'auto_push', 'auto_generate_key',
             'date_created', 'date_updated',
             'comment', 'created_by',
         ]
-        fields_m2m = ["assets_amount",]
+        fields_m2m = ["assets_amount"]
         fields = fields_small + fields_m2m
         extra_kwargs = {
             'password': {"write_only": True},
@@ -247,8 +250,8 @@ class SystemUserAssetRelationSerializer(RelationMixin, serializers.ModelSerializ
     class Meta:
         model = SystemUser.assets.through
         fields = [
-            "id", "asset", "asset_display",
+            "id", "asset", "asset_display", 'systemuser', 'systemuser_display',
-            'systemuser', 'systemuser_display'
+            "connectivity", 'date_verified', 'org_id'
         ]
         use_model_bulk_create = True
         model_bulk_create_kwargs = {

apps/assets/signals_handler/authbook.py

@@ -18,7 +18,11 @@ def pre_create_historical_record_callback(sender, instance=None, history_instanc
     for attr in attrs_to_copy:
         if getattr(history_instance, attr):
             continue
-        if not history_instance.systemuser:
+        try:
+            system_user = history_instance.systemuser
+        except SystemUser.DoesNotExist:
+            continue
+        if not system_user:
             continue
         system_user_attr_value = getattr(history_instance.systemuser, attr)
         if system_user_attr_value:

apps/assets/tasks/account_connectivity.py

@@ -105,3 +105,4 @@ def test_accounts_connectivity_manual(accounts):
     for account in accounts:
         task_name = _("Test account connectivity: {}").format(account)
         test_account_connectivity_util(account, task_name)
+        print(".\n")

apps/assets/tasks/system_user_connectivity.py

@@ -8,7 +8,7 @@ from django.utils.translation import ugettext as _
 from assets.models import Asset
 from common.utils import get_logger
 from orgs.utils import tmp_to_org, org_aware_func
-from ..models import SystemUser
+from ..models import SystemUser, Connectivity, AuthBook
 from . import const
 from .utils import (
     clean_ansible_task_hosts, group_asset_by_platform
@@ -21,6 +21,25 @@ __all__ = [
 ]
 
 
+def set_assets_accounts_connectivity(system_user, assets, results_summary):
+    asset_ids_ok = set()
+    asset_ids_failed = set()
+
+    asset_hostnames_ok = results_summary.get('contacted', {}).keys()
+
+    for asset in assets:
+        if asset.hostname in asset_hostnames_ok:
+            asset_ids_ok.add(asset.id)
+        else:
+            asset_ids_failed.add(asset.id)
+
+    accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user)
+    accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user)
+
+    AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok)
+    AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed)
+
+
 @org_aware_func("system_user")
 def test_system_user_connectivity_util(system_user, assets, task_name):
     """
@@ -32,6 +51,10 @@ def test_system_user_connectivity_util(system_user, assets, task_name):
     """
     from ops.utils import update_or_create_ansible_task
 
+    if system_user.username_same_with_user:
+        logger.error(_("Dynamic system user not support test"))
+        return
+
     # hosts = clean_ansible_task_hosts(assets, system_user=system_user)
     # TODO: 这里不传递系统用户，因为clean_ansible_task_hosts会通过system_user来判断是否可以推送，
     # 不符合测试可连接性逻辑， 后面需要优化此逻辑
@@ -81,17 +104,10 @@ def test_system_user_connectivity_util(system_user, assets, task_name):
         print(_("Start test system user connectivity for platform: [{}]").format(platform))
         print(_("Hosts count: {}").format(len(_hosts)))
         # 用户名不是动态的，用户名则是一个
-        if not system_user.username_same_with_user:
         logger.debug("System user not has special auth")
         run_task(tasks, _hosts, system_user.username)
-        # 否则需要多个任务
-        else:
-            users = system_user.users.all().values_list('username', flat=True)
-            print(_("System user is dynamic: {}").format(list(users)))
-            for username in users:
-                run_task(tasks, _hosts, username)
 
-    system_user.set_connectivity(results_summary)
+    set_assets_accounts_connectivity(system_user, hosts, results_summary)
     return results_summary
 
 

apps/assets/urls/api_urls.py

@@ -45,6 +45,8 @@ urlpatterns = [
     path('system-users/<uuid:pk>/tasks/', api.SystemUserTaskApi.as_view(), name='system-user-task-create'),
     path('system-users/<uuid:pk>/cmd-filter-rules/', api.SystemUserCommandFilterRuleListApi.as_view(), name='system-user-cmd-filter-rule-list'),
 
+    path('accounts/tasks/', api.AccountTaskCreateAPI.as_view(), name='account-task-create'),
+
     path('nodes/tree/', api.NodeListAsTreeApi.as_view(), name='node-tree'),
     path('nodes/children/tree/', api.NodeChildrenAsTreeApi.as_view(), name='node-children-tree'),
     path('nodes/<uuid:pk>/children/', api.NodeChildrenApi.as_view(), name='node-children'),

apps/audits/api.py

@@ -76,6 +76,7 @@ class PasswordChangeLogViewSet(ListModelMixin, CommonGenericViewSet):
         ('datetime', ('date_from', 'date_to'))
     ]
     filterset_fields = ['user', 'change_by', 'remote_addr']
+    search_fields = filterset_fields
     ordering = ['-datetime']
 
     def get_queryset(self):

apps/authentication/mixins.py

@@ -236,12 +236,13 @@ class AuthMixin:
         ip = self.get_request_ip()
         request = self.request
 
+        self._set_partial_credential_error(user.username, ip, request)
+
         if user.is_expired:
             self.raise_credential_error(errors.reason_user_expired)
         elif not user.is_active:
             self.raise_credential_error(errors.reason_user_inactive)
 
-        self._set_partial_credential_error(user.username, ip, request)
         self._check_is_local_user(user)
         self._check_is_block(user.username)
         self._check_login_acl(user, ip)

apps/common/mixins/serializers.py

@@ -293,7 +293,14 @@ class EagerLoadQuerySetFields:
 
 
 class CommonSerializerMixin(DynamicFieldsMixin, DefaultValueFieldsMixin):
-    pass
+    instance: None
+    initial_data: dict
+
+    def get_initial_value(self, attr, default=None):
+        if self.instance:
+            return getattr(self.instance, attr, default)
+        else:
+            return self.initial_data.get(attr)
 
 
 class CommonBulkSerializerMixin(BulkSerializerMixin, CommonSerializerMixin):

apps/terminal/filters.py

@@ -1,7 +1,7 @@
 from django_filters import rest_framework as filters
 from django.db.models import QuerySet
 
-from orgs.utils import current_org
+from orgs.utils import current_org, filter_org_queryset
 from terminal.models import Command, CommandStorage
 
 
@@ -26,7 +26,7 @@ class CommandFilter(filters.FilterSet):
     @property
     def qs(self):
         qs = super().qs
-        qs = qs.filter(org_id=self.get_org_id())
+        qs = filter_org_queryset(qs)
         qs = self.filter_by_timestamp(qs)
         return qs
 
@@ -46,11 +46,6 @@ class CommandFilter(filters.FilterSet):
         qs = qs.filter(**filters)
         return qs
 
-    @staticmethod
-    def get_org_id():
-        org_id = current_org.id
-        return org_id
-
 
 class CommandFilterForStorageTree(CommandFilter):
     asset = filters.CharFilter(method='do_nothing')

apps/users/models/user.py

@@ -197,7 +197,8 @@ class RoleMixin:
         else:
             # 是真实组织, 取 OrganizationMember 中的角色
             roles = [
-                org_member.role for org_member in self.m2m_org_members.all()
+                getattr(ORG_ROLE, org_member.role.upper())
+                for org_member in self.m2m_org_members.all()
                 if org_member.org_id == current_org.id
             ]
             roles.sort()
@@ -206,7 +207,7 @@ class RoleMixin:
     @lazyproperty
     def org_roles_label_list(self):
         from orgs.models import ROLE as ORG_ROLE
-        return [str(ORG_ROLE[role]) for role in self.org_roles if role in ORG_ROLE]
+        return [str(role.label) for role in self.org_roles if role in ORG_ROLE]
 
     @lazyproperty
     def org_role_display(self):