perf: tag top

perf: 优化命令过滤规则操作日志显示问题

GITSHA

@@ -0,0 +1 @@
+c7ea62488dff0daee514bbcfe6ca31afa7dc3a73

apps/accounts/automations/change_secret/host/aix/main.yml

@@ -9,6 +9,7 @@
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('des') }}"
         update_password: always
+      ignore_errors: true
       when: account.secret_type == "password"
 
     - name: create user If it already exists, no operation will be performed

apps/accounts/automations/change_secret/host/posix/main.yml

@@ -9,6 +9,7 @@
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('sha512') }}"
         update_password: always
+      ignore_errors: true
       when: account.secret_type == "password"
 
     - name: create user If it already exists, no operation will be performed

apps/accounts/automations/change_secret/host/windows/main.yml

@@ -21,6 +21,7 @@
         groups: "{{ user_info.groups[0].name }}"
         groups_action: add
         update_password: always
+      ignore_errors: true
       when: account.secret_type == "password"
 
     - name: Refresh connection

apps/accounts/automations/gather_accounts/filter.py

@@ -1,3 +1,5 @@
+import re
+
 from django.utils import timezone
 
 __all__ = ['GatherAccountsFilter']
@@ -27,18 +29,25 @@ class GatherAccountsFilter:
 
     @staticmethod
     def posix_filter(info):
+        username_pattern = re.compile(r'^(\S+)')
+        ip_pattern = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})')
+        login_time_pattern = re.compile(r'\w{3} \d{2} \d{2}:\d{2}:\d{2} \d{4}')
         result = {}
         for line in info:
-            data = line.split('@')
+            usernames = username_pattern.findall(line)
-            if len(data) == 1:
+            username = ''.join(usernames)
-                result[line] = {}
+            if username:
+                result[username] = {}
+            else:
                 continue
-
+            ip_addrs = ip_pattern.findall(line)
-            if len(data) != 3:
+            ip_addr = ''.join(ip_addrs)
-                continue
+            if ip_addr:
-            username, address, dt = data
+                result[username].update({'address': ip_addr})
-            date = timezone.datetime.strptime(f'{dt} +0800', '%b %d %H:%M:%S %Y %z')
+            login_times = login_time_pattern.findall(line)
-            result[username] = {'address': address, 'date': date}
+            if login_times:
+                date = timezone.datetime.strptime(f'{login_times[0]} +0800', '%b %d %H:%M:%S %Y %z')
+                result[username].update({'date': date})
         return result
 
     @staticmethod

apps/accounts/automations/gather_accounts/host/posix/main.yml

@@ -5,7 +5,7 @@
       ansible.builtin.shell:
         cmd: >
           users=$(getent passwd | grep -v nologin | grep -v shutdown | awk -F":" '{ print $1 }');for i in $users;
-          do k=$(last -w -F $i -1 | head -1 | grep -v ^$ | awk '{ print $1"@"$3"@"$5,$6,$7,$8 }')
+          do k=$(last -w -F $i -1 | head -1 | grep -v ^$ | awk '{ print $0 }')
             if [ -n "$k" ]; then
               echo $k
             else

apps/accounts/automations/push_account/host/aix/main.yml

@@ -43,6 +43,7 @@
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('sha512') }}"
         update_password: always
+      ignore_errors: true
       when: account.secret_type == "password"
 
     - name: remove jumpserver ssh key

apps/accounts/automations/push_account/host/posix/main.yml

@@ -43,6 +43,7 @@
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('sha512') }}"
         update_password: always
+      ignore_errors: true
       when: account.secret_type == "password"
 
     - name: remove jumpserver ssh key

apps/accounts/automations/push_account/host/windows/main.yml

@@ -17,6 +17,7 @@
         groups: "{{ params.groups }}"
         groups_action: add
         update_password: always
+      ignore_errors: true
       when: account.secret_type == "password"
 
     - name: Refresh connection

apps/accounts/automations/verify_account/custom/main.yml

@@ -10,5 +10,5 @@
         login_port: "{{ jms_asset.port }}"
         login_user: "{{ account.username }}"
         login_password: "{{ account.secret }}"
-        login_secret_type: "{{ jms_account.secret_type }}"
+        login_secret_type: "{{ account.secret_type }}"
-        login_private_key_path: "{{ jms_account.private_key_path }}"
+        login_private_key_path: "{{ account.private_key_path }}"

apps/accounts/serializers/account/account.py

@@ -22,8 +22,8 @@ logger = get_logger(__name__)
 
 class AccountCreateUpdateSerializerMixin(serializers.Serializer):
     template = serializers.PrimaryKeyRelatedField(
-        queryset=AccountTemplate.objects,
+        queryset=AccountTemplate.objects, required=False,
-        required=False, label=_("Template"), write_only=True
+        label=_("Template"), write_only=True, allow_null=True
     )
     push_now = serializers.BooleanField(
         default=False, label=_("Push now"), write_only=True
@@ -33,7 +33,7 @@ class AccountCreateUpdateSerializerMixin(serializers.Serializer):
     )
     on_invalid = LabeledChoiceField(
         choices=AccountInvalidPolicy.choices, default=AccountInvalidPolicy.ERROR,
-        write_only=True, label=_('Exist policy')
+        write_only=True, allow_null=True, label=_('Exist policy'),
     )
     _template = None
     clean_auth_fields: callable

apps/acls/api/login_acl.py

@@ -1,6 +1,6 @@
 from common.api import JMSBulkModelViewSet
-from ..models import LoginACL
 from .. import serializers
+from ..models import LoginACL
 from ..filters import LoginAclFilter
 
 __all__ = ['LoginACLViewSet']

apps/audits/handler.py

@@ -46,10 +46,8 @@ class OperatorLogHandler(metaclass=Singleton):
             pre_value, value = self._consistent_type_to_str(pre_value, value)
             if sorted(str(value)) == sorted(str(pre_value)):
                 continue
-            if pre_value:
+            before[key] = pre_value
-                before[key] = pre_value
+            after[key] = value
-            if value:
-                after[key] = value
         return before, after
 
     def cache_instance_before_data(self, instance_dict):

apps/audits/utils.py

@@ -70,8 +70,10 @@ def _get_instance_field_value(
 
             if getattr(f, 'primary_key', False):
                 f.verbose_name = 'id'
-            elif isinstance(value, (list, dict)):
+            elif isinstance(value, list):
                 value = copy.deepcopy(value)
+            elif isinstance(value, dict):
+                value = dict(copy.deepcopy(value))
             elif isinstance(value, datetime):
                 value = as_current_tz(value).strftime('%Y-%m-%d %H:%M:%S')
             elif isinstance(f, models.OneToOneField) and isinstance(value, models.Model):

apps/authentication/api/connection_token.py

@@ -3,6 +3,7 @@ import json
 import os
 import urllib.parse
 
+from django.conf import settings
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
@@ -379,19 +380,18 @@ class SuperConnectionTokenViewSet(ConnectionTokenViewSet):
 
         expire_now = request.data.get('expire_now', None)
         asset_type = token.asset.type
-        asset_category = token.asset.category
         # 设置默认值
         if expire_now is None:
             # TODO 暂时特殊处理 k8s 不过期
             if asset_type in ['k8s', 'kubernetes']:
                 expire_now = False
-            elif asset_category in ['database', 'db']:
-                expire_now = False
             else:
-                expire_now = True
+                expire_now = not settings.CONNECTION_TOKEN_REUSABLE
 
-        if is_false(expire_now) or token.is_reusable:
+        if is_false(expire_now):
-            logger.debug('Token is reusable or specify, not expire')
+            logger.debug('Api specified, now expire now')
+        elif token.is_reusable and settings.CONNECTION_TOKEN_REUSABLE:
+            logger.debug('Token is reusable, not expire now')
         else:
             token.expire()
 

apps/authentication/backends/custom.py

@@ -1,5 +1,6 @@
 from django.conf import settings
 from django.utils.module_loading import import_string
+from django.utils.translation import ugettext_lazy as _
 from common.utils import get_logger
 from django.contrib.auth import get_user_model
 from authentication.signals import user_auth_failed, user_auth_success

apps/authentication/errors/failed.py

@@ -89,8 +89,6 @@ class MFAFailedError(AuthFailedNeedLogMixin, AuthFailedError):
     msg: str
 
     def __init__(self, username, request, ip, mfa_type, error):
-        super().__init__(username=username, request=request)
-
         util = MFABlockUtils(username, ip)
         times_remainder = util.incr_failed_count()
         block_time = settings.SECURITY_LOGIN_LIMIT_TIME
@@ -101,6 +99,7 @@ class MFAFailedError(AuthFailedNeedLogMixin, AuthFailedError):
             )
         else:
             self.msg = const.block_mfa_msg.format(settings.SECURITY_LOGIN_LIMIT_TIME)
+        super().__init__(username=username, request=request)
 
 
 class BlockMFAError(AuthFailedNeedLogMixin, AuthFailedError):

apps/authentication/mixins.py

@@ -221,7 +221,8 @@ class MFAMixin:
         self._do_check_user_mfa(code, mfa_type, user=user)
 
     def check_user_mfa_if_need(self, user):
-        if self.request.session.get('auth_mfa'):
+        if self.request.session.get('auth_mfa') and \
+                self.request.session.get('auth_mfa_username') == user.username:
             return
         if not user.mfa_enabled:
             return
@@ -229,15 +230,16 @@ class MFAMixin:
         active_mfa_names = user.active_mfa_backends_mapper.keys()
         raise errors.MFARequiredError(mfa_types=tuple(active_mfa_names))
 
-    def mark_mfa_ok(self, mfa_type):
+    def mark_mfa_ok(self, mfa_type, user):
         self.request.session['auth_mfa'] = 1
+        self.request.session['auth_mfa_username'] = user.username
         self.request.session['auth_mfa_time'] = time.time()
         self.request.session['auth_mfa_required'] = 0
         self.request.session['auth_mfa_type'] = mfa_type
-        MFABlockUtils(self.request.user.username, self.get_request_ip()).clean_failed_count()
+        MFABlockUtils(user.username, self.get_request_ip()).clean_failed_count()
 
     def clean_mfa_mark(self):
-        keys = ['auth_mfa', 'auth_mfa_time', 'auth_mfa_required', 'auth_mfa_type']
+        keys = ['auth_mfa', 'auth_mfa_time', 'auth_mfa_required', 'auth_mfa_type', 'auth_mfa_username']
         for k in keys:
             self.request.session.pop(k, '')
 
@@ -272,7 +274,7 @@ class MFAMixin:
             ok, msg = mfa_backend.check_code(code)
 
         if ok:
-            self.mark_mfa_ok(mfa_type)
+            self.mark_mfa_ok(mfa_type, user)
             return
 
         raise errors.MFAFailedError(

apps/ops/ansible/callback.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from functools import reduce
 
 
 class DefaultCallback:
@@ -18,6 +19,7 @@ class DefaultCallback:
             failures=defaultdict(dict),
             dark=defaultdict(dict),
             skipped=defaultdict(dict),
+            ignored=defaultdict(dict),
         )
         self.summary = dict(
             ok=[],
@@ -59,6 +61,14 @@ class DefaultCallback:
         }
         self.result['ok'][host][task] = detail
 
+    def runner_on_skipped(self, event_data, host=None, task=None, **kwargs):
+        detail = {
+            'action': event_data.get('task_action', ''),
+            'res': {},
+            'rc': 0,
+        }
+        self.result['skipped'][host][task] = detail
+
     def runner_on_failed(self, event_data, host=None, task=None, res=None, **kwargs):
         detail = {
             'action': event_data.get('task_action', ''),
@@ -67,15 +77,9 @@ class DefaultCallback:
             'stdout': res.get('stdout', ''),
             'stderr': ';'.join([res.get('stderr', ''), res.get('msg', '')]).strip(';')
         }
-        self.result['failures'][host][task] = detail
+        ignore_errors = event_data.get('ignore_errors', False)
-
+        error_key = 'ignored' if ignore_errors else 'failures'
-    def runner_on_skipped(self, event_data, host=None, task=None, **kwargs):
+        self.result[error_key][host][task] = detail
-        detail = {
-            'action': event_data.get('task_action', ''),
-            'res': {},
-            'rc': 0,
-        }
-        self.result['skipped'][host][task] = detail
 
     def runner_on_unreachable(self, event_data, host=None, task=None, res=None, **kwargs):
         detail = {
@@ -106,13 +110,18 @@ class DefaultCallback:
 
     def playbook_on_stats(self, event_data, **kwargs):
         failed = []
-        for i in ['dark', 'failures']:
+        error_func = lambda err, task_detail: err + f"{task_detail[0]}: {task_detail[1]['stderr']};"
-            for host, tasks in self.result[i].items():
+        for tp in ['dark', 'failures']:
+            for host, tasks in self.result[tp].items():
                 failed.append(host)
-                error = ''
+                error = reduce(error_func, tasks.items(), '').strip(';')
-                for task, detail in tasks.items():
+                self.summary[tp][host] = error
-                    error += f'{task}: {detail["stderr"]};'
+
-                self.summary[i][host] = error.strip(';')
+        for host, tasks in self.result.get('ignored', {}).items():
+            ignore_errors = reduce(error_func, tasks.items(), '').strip(';')
+            if host in failed:
+                self.summary['failures'][host] += {ignore_errors}
+
         self.summary['ok'] = list(set(self.result['ok'].keys()) - set(failed))
         self.summary['skipped'] = list(set(self.result['skipped'].keys()) - set(failed))
 

apps/ops/api/job.py

@@ -166,6 +166,7 @@ class UsernameHintsAPI(APIView):
 
         top_accounts = Account.objects \
                            .exclude(username__startswith='jms_') \
+                           .exclude(username__startswith='js_') \
                            .filter(username__icontains=query) \
                            .filter(asset__in=assets) \
                            .values('username') \

apps/settings/serializers/auth/sms.py

@@ -38,25 +38,25 @@ class BaseSMSSettingSerializer(serializers.Serializer):
 
 
 class AlibabaSMSSettingSerializer(BaseSMSSettingSerializer):
-    ALIBABA_ACCESS_KEY_ID = serializers.CharField(max_length=256, required=True, label='AccessKeyId')
+    ALIBABA_ACCESS_KEY_ID = serializers.CharField(max_length=256, required=True, label='Access Key ID')
     ALIBABA_ACCESS_KEY_SECRET = EncryptedField(
-        max_length=256, required=False, label='access_key_secret',
+        max_length=256, required=False, label='Access Key Secret',
     )
     ALIBABA_VERIFY_SIGN_NAME = serializers.CharField(max_length=256, required=True, label=_('Signature'))
     ALIBABA_VERIFY_TEMPLATE_CODE = serializers.CharField(max_length=256, required=True, label=_('Template code'))
 
 
 class TencentSMSSettingSerializer(BaseSMSSettingSerializer):
-    TENCENT_SECRET_ID = serializers.CharField(max_length=256, required=True, label='Secret id')
+    TENCENT_SECRET_ID = serializers.CharField(max_length=256, required=True, label='Secret ID')
-    TENCENT_SECRET_KEY = EncryptedField(max_length=256, required=False, label='Secret key')
+    TENCENT_SECRET_KEY = EncryptedField(max_length=256, required=False, label='Secret Key')
-    TENCENT_SDKAPPID = serializers.CharField(max_length=256, required=True, label='SDK app id')
+    TENCENT_SDKAPPID = serializers.CharField(max_length=256, required=True, label='SDK APP ID')
     TENCENT_VERIFY_SIGN_NAME = serializers.CharField(max_length=256, required=True, label=_('Signature'))
     TENCENT_VERIFY_TEMPLATE_CODE = serializers.CharField(max_length=256, required=True, label=_('Template code'))
 
 
 class HuaweiSMSSettingSerializer(BaseSMSSettingSerializer):
-    HUAWEI_APP_KEY = serializers.CharField(max_length=256, required=True, label='App key')
+    HUAWEI_APP_KEY = serializers.CharField(max_length=256, required=True, label='App Key')
-    HUAWEI_APP_SECRET = EncryptedField(max_length=256, required=False, label='App secret')
+    HUAWEI_APP_SECRET = EncryptedField(max_length=256, required=False, label='App Secret')
     HUAWEI_SMS_ENDPOINT = serializers.CharField(max_length=1024, required=True, label=_('App Access Address'))
     HUAWEI_SIGN_CHANNEL_NUM = serializers.CharField(max_length=1024, required=True, label=_('Signature channel number'))
     HUAWEI_VERIFY_SIGN_NAME = serializers.CharField(max_length=256, required=True, label=_('Signature'))

apps/terminal/automations/deploy_applet_host/playbook.yml

@@ -182,7 +182,7 @@
     - name: Set chromium and driver on the global system path (Chromium)
       ansible.windows.win_path:
         elements:
-          - 'C:\Program Files\Chrome\chrome-win'
+          - 'C:\Program Files\Chrome\chrome-win32'
           - 'C:\Program Files\JumpServer\drivers\chromedriver_win32'
 
     - name: Set Chromium variables disable Google Api (Chromium)

apps/terminal/models/applet/applet.py

@@ -94,6 +94,7 @@ class Applet(JMSBaseModel):
     @classmethod
     def load_platform_if_need(cls, d):
         from assets.serializers import PlatformSerializer
+        from assets.const import CustomTypes
 
         if not os.path.exists(os.path.join(d, 'platform.yml')):
             return
@@ -111,6 +112,9 @@ class Applet(JMSBaseModel):
         except KeyError:
             raise ValidationError({'error': _('Missing type in platform.yml')})
 
+        if not data.get('automation'):
+            data['automation'] = CustomTypes._get_automation_constrains()['*']
+
         s = PlatformSerializer(data=data)
         s.add_type_choices(tp, tp)
         s.is_valid(raise_exception=True)
@@ -193,7 +197,7 @@ class Applet(JMSBaseModel):
             if private_account and private_account.username not in accounts_username_used:
                 account = private_account
             else:
-                accounts = accounts.exclude(username__in=accounts_username_used)
+                accounts = accounts.exclude(username__in=accounts_username_used).filter(username__startswith='jms_')
                 account = self.random_select_prefer_account(user, host, accounts)
                 if not account:
                     return

apps/terminal/models/component/endpoint.py

@@ -82,7 +82,7 @@ class Endpoint(JMSBaseModel):
             return None
         endpoints = cls.objects.filter(name__in=values).order_by('-date_updated')
         for endpoint in endpoints:
-            if endpoint.is_valid_for(protocol):
+            if endpoint.is_valid_for(instance, protocol):
                 return endpoint