feat: Update v2.28.2

fix: codacy ci
fix: password use bytes

GITSHA

@@ -0,0 +1 @@
+d3bfc0384977a6910caa7f9e8f12482427842f7d

apps/assets/models/base.py

@@ -6,23 +6,21 @@ import uuid
 from hashlib import md5
 
 import sshpubkeys
+from django.conf import settings
 from django.core.cache import cache
 from django.db import models
+from django.db.models import QuerySet
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from django.conf import settings
-from django.db.models import QuerySet
 
-from common.utils import random_string, signer
+from common.db import fields
+from common.utils import random_string
 from common.utils import (
     ssh_key_string_to_obj, ssh_key_gen, get_logger, lazyproperty
 )
-from common.utils.encode import ssh_pubkey_gen
+from common.utils.encode import parse_ssh_public_key_str
-from common.validators import alphanumeric
-from common.db import fields
 from orgs.mixins.models import OrgModelMixin
 
-
 logger = get_logger(__file__)
 
 
@@ -68,7 +66,7 @@ class AuthMixin:
             public_key = self.public_key
         elif self.private_key:
             try:
-                public_key = ssh_pubkey_gen(private_key=self.private_key, password=self.password)
+                public_key = parse_ssh_public_key_str(private_key=self.private_key, password=self.password)
             except IOError as e:
                 return str(e)
         else:
@@ -234,4 +232,3 @@ class BaseUser(OrgModelMixin, AuthMixin):
 
     class Meta:
         abstract = True
-

apps/assets/serializers/base.py

@@ -1,24 +1,24 @@
 # -*- coding: utf-8 -*-
 #
-from io import StringIO
 
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
-from common.utils import ssh_pubkey_gen, ssh_private_key_gen, validate_ssh_private_key
-from common.drf.fields import EncryptedField
 from assets.models import Type
+from common.drf.fields import EncryptedField
+from common.utils import validate_ssh_private_key, parse_ssh_private_key_str, parse_ssh_public_key_str
 from .utils import validate_password_for_ansible
 
 
 class AuthSerializer(serializers.ModelSerializer):
     password = EncryptedField(required=False, allow_blank=True, allow_null=True, max_length=1024, label=_('Password'))
-    private_key = EncryptedField(required=False, allow_blank=True, allow_null=True, max_length=16384, label=_('Private key'))
+    private_key = EncryptedField(required=False, allow_blank=True, allow_null=True, max_length=16384,
+                                 label=_('Private key'))
 
     def gen_keys(self, private_key=None, password=None):
         if private_key is None:
             return None, None
-        public_key = ssh_pubkey_gen(private_key=private_key, password=password)
+        public_key = parse_ssh_public_key_str(text=private_key, password=password)
         return private_key, public_key
 
     def save(self, **kwargs):
@@ -57,10 +57,7 @@ class AuthSerializerMixin(serializers.ModelSerializer):
         if not valid:
             raise serializers.ValidationError(_("private key invalid or passphrase error"))
 
-        private_key = ssh_private_key_gen(private_key, password=passphrase)
+        private_key = parse_ssh_private_key_str(private_key, password=passphrase)
-        string_io = StringIO()
-        private_key.write_private_key(string_io)
-        private_key = string_io.getvalue()
         return private_key
 
     def validate_public_key(self, public_key):

apps/assets/serializers/system_user.py

@@ -1,16 +1,16 @@
-from rest_framework import serializers
-from django.utils.translation import ugettext_lazy as _
 from django.db.models import Count
+from django.utils.translation import ugettext_lazy as _
+from rest_framework import serializers
 
-from common.mixins.serializers import BulkSerializerMixin
-from common.utils import ssh_pubkey_gen
 from common.drf.fields import EncryptedField
 from common.drf.serializers import SecretReadableMixin
+from common.mixins.serializers import BulkSerializerMixin
+from common.utils import parse_ssh_public_key_str
 from common.validators import alphanumeric_re, alphanumeric_cn_re, alphanumeric_win_re
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
-from ..models import SystemUser, Asset
-from .utils import validate_password_for_ansible
 from .base import AuthSerializerMixin
+from .utils import validate_password_for_ansible
+from ..models import SystemUser, Asset
 
 __all__ = [
     'SystemUserSerializer', 'MiniSystemUserSerializer',
@@ -214,7 +214,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         elif attrs.get('private_key'):
             private_key = attrs['private_key']
             password = attrs.get('password')
-            public_key = ssh_pubkey_gen(private_key, password=password, username=username)
+            public_key = parse_ssh_public_key_str(private_key, password=password)
             attrs['public_key'] = public_key
         return attrs
 

apps/assets/serializers/utils.py

@@ -1,6 +1,8 @@
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
+from common.utils import validate_ssh_private_key, parse_ssh_private_key_str
+
 
 def validate_password_for_ansible(password):
     """ 校验 Ansible 不支持的特殊字符 """
@@ -15,3 +17,9 @@ def validate_password_for_ansible(password):
     if '"' in password:
         raise serializers.ValidationError(_('Password can not contains `"` '))
 
+
+def validate_ssh_key(ssh_key, passphrase=None):
+    valid = validate_ssh_private_key(ssh_key, password=passphrase)
+    if not valid:
+        raise serializers.ValidationError(_("private key invalid or passphrase error"))
+    return parse_ssh_private_key_str(ssh_key, passphrase)

apps/common/management/commands/services/services/flower.py

@@ -9,6 +9,10 @@ class FlowerService(BaseService):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    @property
+    def db_file(self):
+        return os.path.join(BASE_DIR, 'data', 'flower')
+
     @property
     def cmd(self):
         print("\n- Start Flower as Task Monitor")
@@ -20,11 +24,11 @@ class FlowerService(BaseService):
             '-A', 'ops',
             'flower',
             '-logging=info',
+            '-db={}'.format(self.db_file),
             '--url_prefix=/core/flower',
             '--auto_refresh=False',
             '--max_tasks=1000',
             '--persistent=True',
-            '-db=/opt/jumpserver/data/flower.db',
             '--state_save_interval=600000'
         ]
         return cmd

apps/common/utils/encode.py

@@ -1,24 +1,23 @@
 # -*- coding: utf-8 -*-
 #
-import re
-import json
-from six import string_types
 import base64
-import os
-import time
 import hashlib
+import json
+import os
+import re
+import time
 from io import StringIO
-from itertools import chain
 
 import paramiko
 import sshpubkeys
+from cryptography.hazmat.primitives import serialization
+from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
 from itsdangerous import (
     TimedJSONWebSignatureSerializer, JSONWebSignatureSerializer,
     BadSignature, SignatureExpired
 )
-from django.conf import settings
+from six import string_types
-from django.core.serializers.json import DjangoJSONEncoder
-from django.db.models.fields.files import FileField
 
 from .http import http_date
 
@@ -69,22 +68,25 @@ class Signer(metaclass=Singleton):
             return None
 
 
+_supported_paramiko_ssh_key_types = (
+    paramiko.RSAKey,
+    paramiko.DSSKey,
+    paramiko.Ed25519Key,
+    paramiko.ECDSAKey,
+)
+
+
 def ssh_key_string_to_obj(text, password=None):
     key = None
+    for ssh_key_type in _supported_paramiko_ssh_key_types:
+        if not isinstance(ssh_key_type, paramiko.PKey):
+            continue
         try:
-        key = paramiko.RSAKey.from_private_key(StringIO(text), password=password)
+            key = ssh_key_type.from_private_key(StringIO(text), password=password)
         except paramiko.SSHException:
             pass
         else:
             return key
-
-    try:
-        key = paramiko.DSSKey.from_private_key(StringIO(text), password=password)
-    except paramiko.SSHException:
-        pass
-    else:
-        return key
-
     return key
 
 
@@ -98,7 +100,7 @@ def ssh_private_key_gen(private_key, password=None):
 
 def ssh_pubkey_gen(private_key=None, username='jumpserver', hostname='localhost', password=None):
     private_key = ssh_private_key_gen(private_key, password=password)
-    if not isinstance(private_key, (paramiko.RSAKey, paramiko.DSSKey)):
+    if not isinstance(private_key, _supported_paramiko_ssh_key_types):
         raise IOError('Invalid private key')
 
     public_key = "%(key_type)s %(key_content)s %(username)s@%(hostname)s" % {
@@ -137,17 +139,63 @@ def ssh_key_gen(length=2048, type='rsa', password=None, username='jumpserver', h
 
 
 def validate_ssh_private_key(text, password=None):
-    if isinstance(text, bytes):
+    key = parse_ssh_private_key_str(text, password=password)
-        try:
+    return bool(key)
-            text = text.decode("utf-8")
-        except UnicodeDecodeError:
-            return False
 
-    key = ssh_key_string_to_obj(text, password=password)
+
-    if key is None:
+def parse_ssh_private_key_str(text: bytes, password=None) -> str:
-        return False
+    private_key = _parse_ssh_private_key(text, password=password)
-    else:
+    if private_key is None:
-        return True
+        return ""
+    private_key_bytes = private_key.private_bytes(serialization.Encoding.PEM,
+                                                  serialization.PrivateFormat.OpenSSH,
+                                                  serialization.NoEncryption())
+    return private_key_bytes.decode('utf-8')
+
+
+def parse_ssh_public_key_str(text: bytes = "", password=None) -> str:
+    private_key = _parse_ssh_private_key(text, password=password)
+    if private_key is None:
+        return ""
+    public_key_bytes = private_key.public_key().public_bytes(
+        serialization.Encoding.OpenSSH,
+        serialization.PublicFormat.OpenSSH)
+    return public_key_bytes.decode('utf-8')
+
+
+def _parse_ssh_private_key(text, password=None):
+    """
+    text: bytes
+    password: str
+    return:private key types:
+                ec.EllipticCurvePrivateKey,
+                rsa.RSAPrivateKey,
+                dsa.DSAPrivateKey,
+                ed25519.Ed25519PrivateKey,
+    """
+    if isinstance(text, str):
+        try:
+            text = text.encode("utf-8")
+        except UnicodeDecodeError:
+            return None
+    if password is not None:
+        if isinstance(password, str):
+            try:
+                password = password.encode("utf-8")
+            except UnicodeDecodeError:
+                return None
+
+    try:
+        if is_openssh_format_key(text):
+            return serialization.load_ssh_private_key(text, password=password)
+        return serialization.load_pem_private_key(text, password=password)
+    except (ValueError, TypeError):
+        pass
+    return None
+
+
+def is_openssh_format_key(text: bytes):
+    return text.startswith(b"-----BEGIN OPENSSH PRIVATE KEY-----")
 
 
 def validate_ssh_public_key(text):

apps/common/utils/random.py

@@ -18,25 +18,35 @@ def random_ip():
     return socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
 
 
-def random_string(length, lower=True, upper=True, digit=True, special_char=False):
+def random_string(length: int, lower=True, upper=True, digit=True, special_char=False):
-    chars = string.ascii_letters
+    args_names = ['lower', 'upper', 'digit', 'special_char']
-    if digit:
+    args_values = [lower, upper, digit, special_char]
-        chars += string.digits
+    args_string = [string.ascii_lowercase, string.ascii_uppercase, string.digits, string_punctuation]
+    args_string_map = dict(zip(args_names, args_string))
+    kwargs = dict(zip(args_names, args_values))
+    kwargs_keys = list(kwargs.keys())
+    kwargs_values = list(kwargs.values())
+    args_true_count = len([i for i in kwargs_values if i])
+    assert any(kwargs_values), f'Parameters {kwargs_keys} must have at least one `True`'
+    assert length >= args_true_count, f'Expected length >= {args_true_count}, bug got {length}'
+
+    can_startswith_special_char = args_true_count == 1 and special_char
+
+    chars = ''.join([args_string_map[k] for k, v in kwargs.items() if v])
 
     while True:
         password = list(random.choice(chars) for i in range(length))
-        if upper and not any(c.upper() for c in password):
+        for k, v in kwargs.items():
-            continue
+            if v and not (set(password) & set(args_string_map[k])):
-        if lower and not any(c.lower() for c in password):
+                # 没有包含指定的字符, retry
-            continue
+                break
-        if digit and not any(c.isdigit() for c in password):
+        else:
-            continue
+            if not can_startswith_special_char and password[0] in args_string_map['special_char']:
+                # 首位不能为特殊字符, retry
+                continue
+            else:
+                # 满足要求终止 while 循环
                 break
-
-    if special_char:
-        spc = random.choice(string_punctuation)
-        i = random.choice(range(1, len(password)))
-        password[i] = spc
 
     password = ''.join(password)
     return password

apps/jumpserver/conf.py

@@ -202,6 +202,7 @@ class Config(dict):
         'REDIS_SSL_KEY': None,
         'REDIS_SSL_CERT': None,
         'REDIS_SSL_CA': None,
+        'REDIS_SSL_REQUIRED': 'none',
         # Redis Sentinel
         'REDIS_SENTINEL_HOSTS': '',
         'REDIS_SENTINEL_PASSWORD': '',

apps/jumpserver/settings/base.py

@@ -1,6 +1,9 @@
 import os
 import platform
 
+from redis.sentinel import SentinelManagedSSLConnection
+
+
 if platform.system() == 'Darwin' and platform.machine() == 'arm64':
     import pymysql
 
@@ -195,7 +198,7 @@ DATABASES = {
     }
 }
 
-DB_CA_PATH = os.path.join(PROJECT_DIR, 'data', 'certs', 'db_ca.pem')
+DB_CA_PATH = os.path.join(CERTS_DIR, 'db_ca.pem')
 DB_USE_SSL = False
 if CONFIG.DB_ENGINE.lower() == 'mysql':
     DB_OPTIONS['init_command'] = "SET sql_mode='STRICT_TRANS_TABLES'"
@@ -317,10 +320,19 @@ if REDIS_SENTINEL_SERVICE_NAME and REDIS_SENTINELS:
         'CLIENT_CLASS': 'django_redis.client.SentinelClient',
         'SENTINELS': REDIS_SENTINELS, 'PASSWORD': CONFIG.REDIS_PASSWORD,
         'SENTINEL_KWARGS': {
+            'ssl': REDIS_USE_SSL,
+            'ssl_cert_reqs': REDIS_SSL_REQUIRED,
+            "ssl_keyfile": REDIS_SSL_KEY,
+            "ssl_certfile": REDIS_SSL_CERT,
+            "ssl_ca_certs": REDIS_SSL_CA,
             'password': REDIS_SENTINEL_PASSWORD,
             'socket_timeout': REDIS_SENTINEL_SOCKET_TIMEOUT
         }
     })
+    if REDIS_USE_SSL:
+        REDIS_OPTIONS['CONNECTION_POOL_KWARGS'].update({
+            'connection_class': SentinelManagedSSLConnection
+        })
     DJANGO_REDIS_CONNECTION_FACTORY = 'django_redis.pool.SentinelConnectionFactory'
 else:
     REDIS_LOCATION_NO_DB = '%(protocol)s://:%(password)s@%(host)s:%(port)s/{}' % {

apps/jumpserver/settings/libs.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 #
 import os
-import ssl
+from urllib.parse import urlencode
 
 from .base import (
     REDIS_SSL_CA, REDIS_SSL_CERT, REDIS_SSL_KEY, REDIS_SSL_REQUIRED, REDIS_USE_SSL,
-    REDIS_SENTINEL_SERVICE_NAME, REDIS_SENTINELS, REDIS_SENTINEL_PASSWORD,
+    REDIS_PROTOCOL, REDIS_SENTINEL_SERVICE_NAME, REDIS_SENTINELS, REDIS_SENTINEL_PASSWORD,
     REDIS_SENTINEL_SOCKET_TIMEOUT
 )
 from ..const import CONFIG, PROJECT_DIR
@@ -81,41 +81,54 @@ BOOTSTRAP3 = {
 }
 
 # Django channels support websocket
-if not REDIS_USE_SSL:
+REDIS_LAYERS_HOST = {
-    redis_ssl = None
-else:
-    redis_ssl = ssl.SSLContext()
-    redis_ssl.check_hostname = bool(CONFIG.REDIS_SSL_REQUIRED)
-    if REDIS_SSL_CA:
-        redis_ssl.load_verify_locations(REDIS_SSL_CA)
-    if REDIS_SSL_CERT and REDIS_SSL_KEY:
-        redis_ssl.load_cert_chain(REDIS_SSL_CERT, REDIS_SSL_KEY)
-
-REDIS_HOST = {
     'db': CONFIG.REDIS_DB_WS,
     'password': CONFIG.REDIS_PASSWORD or None,
-    'ssl': redis_ssl,
 }
 
+REDIS_LAYERS_SSL_PARAMS = {}
+if REDIS_USE_SSL:
+    REDIS_LAYERS_SSL_PARAMS.update({
+        'ssl': REDIS_USE_SSL,
+        'ssl_cert_reqs': REDIS_SSL_REQUIRED,
+        "ssl_keyfile": REDIS_SSL_KEY,
+        "ssl_certfile": REDIS_SSL_CERT,
+        "ssl_ca_certs": REDIS_SSL_CA
+    })
+    REDIS_LAYERS_HOST.update(REDIS_LAYERS_SSL_PARAMS)
+
 if REDIS_SENTINEL_SERVICE_NAME and REDIS_SENTINELS:
-    REDIS_HOST['sentinels'] = REDIS_SENTINELS
+    REDIS_LAYERS_HOST['sentinels'] = REDIS_SENTINELS
-    REDIS_HOST['master_name'] = REDIS_SENTINEL_SERVICE_NAME
+    REDIS_LAYERS_HOST['master_name'] = REDIS_SENTINEL_SERVICE_NAME
-    REDIS_HOST['sentinel_kwargs'] = {
+    REDIS_LAYERS_HOST['sentinel_kwargs'] = {
         'password': REDIS_SENTINEL_PASSWORD,
-        'socket_timeout': REDIS_SENTINEL_SOCKET_TIMEOUT
+        'socket_timeout': REDIS_SENTINEL_SOCKET_TIMEOUT,
+        'ssl': REDIS_USE_SSL,
+        'ssl_cert_reqs': REDIS_SSL_REQUIRED,
+        "ssl_keyfile": REDIS_SSL_KEY,
+        "ssl_certfile": REDIS_SSL_CERT,
+        "ssl_ca_certs": REDIS_SSL_CA
     }
 else:
-    REDIS_HOST['address'] = (CONFIG.REDIS_HOST, CONFIG.REDIS_PORT)
+    # More info see: https://github.com/django/channels_redis/issues/334
+    # REDIS_LAYERS_HOST['address'] = (CONFIG.REDIS_HOST, CONFIG.REDIS_PORT)
+    REDIS_LAYERS_ADDRESS = '{protocol}://:{password}@{host}:{port}/{db}'.format(
+        protocol=REDIS_PROTOCOL, password=CONFIG.REDIS_PASSWORD,
+        host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, db=CONFIG.REDIS_DB_WS
+    )
+    REDIS_LAYERS_SSL_PARAMS.pop('ssl', None)
+    REDIS_LAYERS_HOST['address'] = '{}?{}'.format(REDIS_LAYERS_ADDRESS, urlencode(REDIS_LAYERS_SSL_PARAMS))
 
 
 CHANNEL_LAYERS = {
     'default': {
         'BACKEND': 'common.cache.RedisChannelLayer',
         'CONFIG': {
-            "hosts": [REDIS_HOST],
+            "hosts": [REDIS_LAYERS_HOST],
         },
     },
 }
+
 ASGI_APPLICATION = 'jumpserver.routing.application'
 
 # Dump all celery log to here
@@ -132,13 +145,18 @@ if REDIS_SENTINEL_SERVICE_NAME and REDIS_SENTINELS:
         'master_name': REDIS_SENTINEL_SERVICE_NAME,
         'sentinel_kwargs': {
             'password': REDIS_SENTINEL_PASSWORD,
-            'socket_timeout': REDIS_SENTINEL_SOCKET_TIMEOUT
+            'socket_timeout': REDIS_SENTINEL_SOCKET_TIMEOUT,
+            'ssl': REDIS_USE_SSL,
+            'ssl_cert_reqs': REDIS_SSL_REQUIRED,
+            "ssl_keyfile": REDIS_SSL_KEY,
+            "ssl_certfile": REDIS_SSL_CERT,
+            "ssl_ca_certs": REDIS_SSL_CA
         }
     }
     CELERY_BROKER_TRANSPORT_OPTIONS = CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = SENTINEL_OPTIONS
 else:
     CELERY_BROKER_URL = CELERY_BROKER_URL_FORMAT % {
-        'protocol': 'rediss' if REDIS_USE_SSL else 'redis',
+        'protocol': REDIS_PROTOCOL,
         'password': CONFIG.REDIS_PASSWORD,
         'host': CONFIG.REDIS_HOST,
         'port': CONFIG.REDIS_PORT,

apps/terminal/models/storage.py

@@ -19,7 +19,6 @@ from .terminal import Terminal
 from .command import Command
 from .. import const
 
-
 logger = get_logger(__file__)
 
 
@@ -37,10 +36,10 @@ class CommonStorageModelMixin(models.Model):
 
     def set_to_default(self):
         self.is_default = True
-        self.save()
+        self.save(update_fields=['is_default'])
-        self.__class__.objects.select_for_update()\
+        self.__class__.objects.select_for_update() \
-            .filter(is_default=True)\
+            .filter(is_default=True) \
-            .exclude(id=self.id)\
+            .exclude(id=self.id) \
             .update(is_default=False)
 
     @classmethod
@@ -128,7 +127,10 @@ class CommandStorage(CommonStorageModelMixin, CommonModelMixin):
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
-        super().save()
+        super().save(
+            force_insert=force_insert, force_update=force_update,
+            using=using, update_fields=update_fields
+        )
 
         if self.type in TYPE_ENGINE_MAPPING:
             engine_mod = import_module(TYPE_ENGINE_MAPPING[self.type])

requirements/requirements.txt

@@ -63,7 +63,7 @@ jsonfield2==4.0.0.post0
 geoip2==4.5.0
 ipip-ipdb==1.6.1
 # Django environment
-Django==3.2.16
+Django==3.2.15
 django-bootstrap3==14.2.0
 django-filter==2.4.0
 django-formtools==2.2

utils/start_celery_beat.py

@@ -39,6 +39,11 @@ if REDIS_SENTINEL_SERVICE_NAME and REDIS_SENTINELS:
     connection_params['sentinels'] = REDIS_SENTINELS
     sentinel_client = Sentinel(
         **connection_params, sentinel_kwargs={
+            'ssl': settings.REDIS_USE_SSL,
+            'ssl_cert_reqs': settings.REDIS_SSL_REQUIRED,
+            'ssl_keyfile': settings.REDIS_SSL_KEY,
+            'ssl_certfile': settings.REDIS_SSL_CERT,
+            'ssl_ca_certs': settings.REDIS_SSL_CA,
             'password': REDIS_SENTINEL_PASSWORD,
             'socket_timeout': REDIS_SENTINEL_SOCKET_TIMEOUT
         }