perf: 优化 redis (#8484)

* perf: 优化 redis

* perf: 优化 redis 时间

* perf: 优化时间

* perf: 修改 ssl

* perf: 修改 ssl

* perf: 修改 ssl name

* perf: 修改名称

Co-authored-by: ibuler <ibuler@qq.com>

apps/common/utils/connection.py

@@ -1,11 +1,9 @@
 import json
 import threading
+import redis
 
-from redis import Redis
+from django.core.cache import cache
-from django.conf import settings
 
-from jumpserver.const import CONFIG
-from common.http import is_true
 from common.db.utils import safe_db_connection
 from common.utils import get_logger
 
@@ -13,18 +11,9 @@ logger = get_logger(__name__)
 
 
 def get_redis_client(db=0):
-    params = {
+    client = cache.client.get_client()
-        'host': CONFIG.REDIS_HOST,
+    assert isinstance(client, redis.Redis)
-        'port': CONFIG.REDIS_PORT,
+    return client
-        'password': CONFIG.REDIS_PASSWORD,
-        'db': db,
-        "ssl": is_true(CONFIG.REDIS_USE_SSL),
-        'ssl_cert_reqs': getattr(settings, 'REDIS_SSL_REQUIRED'),
-        'ssl_keyfile': getattr(settings, 'REDIS_SSL_KEYFILE'),
-        'ssl_certfile': getattr(settings, 'REDIS_SSL_CERTFILE'),
-        'ssl_ca_certs': getattr(settings, 'REDIS_SSL_CA_CERTS'),
-    }
-    return Redis(**params)
 
 
 class Subscription:

apps/jumpserver/conf.py

@@ -139,6 +139,7 @@ class Config(dict):
         'REDIS_HOST': '127.0.0.1',
         'REDIS_PORT': 6379,
         'REDIS_PASSWORD': '',
+        'REDIS_USE_SSL': False,
         # Default value
         'REDIS_DB_CELERY': 3,
         'REDIS_DB_CACHE': 4,

apps/jumpserver/rewriting/session.py

@@ -1,50 +0,0 @@
-from redis_sessions.session import (
-    force_unicode, SessionStore as RedisSessionStore,
-    RedisServer as _RedisServer, settings as redis_setting
-)
-from redis import exceptions, Redis
-from django.conf import settings
-
-from jumpserver.const import CONFIG
-
-
-class RedisServer(_RedisServer):
-    __redis = {}
-
-    def get(self):
-        if self.connection_key in self.__redis:
-            return self.__redis[self.connection_key]
-
-        ssl_params = {}
-        if CONFIG.REDIS_USE_SSL:
-            ssl_params = {
-                'ssl_cert_reqs': getattr(settings, 'REDIS_SSL_REQUIRED'),
-                'ssl_keyfile': getattr(settings, 'REDIS_SSL_KEYFILE'),
-                'ssl_certfile': getattr(settings, 'REDIS_SSL_CERTFILE'),
-                'ssl_ca_certs': getattr(settings, 'REDIS_SSL_CA_CERTS'),
-            }
-        # 只根据 redis_url 方式连接
-        self.__redis[self.connection_key] = Redis.from_url(
-            redis_setting.SESSION_REDIS_URL, **ssl_params
-        )
-
-        return self.__redis[self.connection_key]
-
-
-class SessionStore(RedisSessionStore):
-    def __init__(self, session_key=None):
-        super(SessionStore, self).__init__(session_key)
-        self.server = RedisServer(session_key).get()
-
-    def load(self):
-        try:
-            session_data = self.server.get(
-                self.get_real_stored_key(self._get_or_create_session_key())
-            )
-            return self.decode(force_unicode(session_data))
-        except exceptions.ConnectionError as e:
-            # 解决redis服务异常(如: 主从切换时)，用户session立即过期的问题
-            raise
-        except:
-            self._session_key = None
-            return {}

apps/jumpserver/settings/base.py

@@ -178,10 +178,12 @@ DATABASES = {
 }
 
 DB_CA_PATH = os.path.join(PROJECT_DIR, 'data', 'certs', 'db_ca.pem')
+DB_USE_SSL = False
 if CONFIG.DB_ENGINE.lower() == 'mysql':
     DB_OPTIONS['init_command'] = "SET sql_mode='STRICT_TRANS_TABLES'"
     if os.path.isfile(DB_CA_PATH):
         DB_OPTIONS['ssl'] = {'ca': DB_CA_PATH}
+        DB_USE_SSL = True
 
 # Password validation
 # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
@@ -264,9 +266,11 @@ REDIS_SSL_KEYFILE = exist_or_default(os.path.join(CERTS_DIR, 'redis_client.key')
 REDIS_SSL_CERTFILE = exist_or_default(os.path.join(CERTS_DIR, 'redis_client.crt'), None)
 REDIS_SSL_CA_CERTS = exist_or_default(os.path.join(CERTS_DIR, 'redis_ca.pem'), None)
 REDIS_SSL_CA_CERTS = exist_or_default(os.path.join(CERTS_DIR, 'redis_ca.crt'), REDIS_SSL_CA_CERTS)
-REDIS_SSL_REQUIRED = CONFIG.REDIS_SSL_REQUIRED or 'none'
+REDIS_SSL_REQUIRED = 'none'
+REDIS_USE_SSL = CONFIG.REDIS_USE_SSL
+
 REDIS_LOCATION_NO_DB = '%(protocol)s://:%(password)s@%(host)s:%(port)s/{}' % {
-    'protocol': 'rediss' if CONFIG.REDIS_USE_SSL else 'redis',
+    'protocol': 'rediss' if REDIS_USE_SSL else 'redis',
     'password': CONFIG.REDIS_PASSWORD,
     'host': CONFIG.REDIS_HOST,
     'port': CONFIG.REDIS_PORT,
@@ -282,7 +286,7 @@ REDIS_CACHE_DEFAULT = {
             "ssl_keyfile": REDIS_SSL_KEYFILE,
             "ssl_certfile": REDIS_SSL_CERTFILE,
             "ssl_ca_certs": REDIS_SSL_CA_CERTS
-        } if CONFIG.REDIS_USE_SSL else {}
+        } if REDIS_USE_SSL else {}
     }
 }
 REDIS_CACHE_SESSION = dict(REDIS_CACHE_DEFAULT)
@@ -292,7 +296,6 @@ CACHES = {
     'default': REDIS_CACHE_DEFAULT,
     'session': REDIS_CACHE_SESSION
 }
-
 SESSION_CACHE_ALIAS = "session"
 
 FORCE_SCRIPT_NAME = CONFIG.FORCE_SCRIPT_NAME

apps/jumpserver/settings/libs.py

@@ -3,7 +3,10 @@
 import os
 import ssl
 
-from .base import REDIS_SSL_CA_CERTS, REDIS_SSL_CERTFILE, REDIS_SSL_KEYFILE, REDIS_SSL_REQUIRED
+from .base import (
+    REDIS_SSL_CA_CERTS, REDIS_SSL_CERTFILE, REDIS_SSL_KEYFILE,
+    REDIS_SSL_REQUIRED, REDIS_USE_SSL
+)
 from ..const import CONFIG, PROJECT_DIR
 
 
@@ -72,7 +75,6 @@ CAPTCHA_CHALLENGE_FUNCT = 'captcha.helpers.math_challenge'
 # Django bootstrap3 setting, more see http://django-bootstrap3.readthedocs.io/en/latest/settings.html
 BOOTSTRAP3 = {
     'horizontal_label_class': 'col-md-2',
-    # Field class to use in horizontal forms
     'horizontal_field_class': 'col-md-9',
     # Set placeholder attributes to label if no placeholder is provided
     'set_placeholder': False,
@@ -82,15 +84,15 @@ BOOTSTRAP3 = {
 
 
 # Django channels support websocket
-if not CONFIG.REDIS_USE_SSL:
+if not REDIS_USE_SSL:
-    context = None
+    redis_ssl = None
 else:
-    context = ssl.SSLContext()
+    redis_ssl = ssl.SSLContext()
-    context.check_hostname = bool(CONFIG.REDIS_SSL_REQUIRED)
+    redis_ssl.check_hostname = bool(CONFIG.REDIS_SSL_REQUIRED)
     if REDIS_SSL_CA_CERTS:
-        context.load_verify_locations(REDIS_SSL_CA_CERTS)
+        redis_ssl.load_verify_locations(REDIS_SSL_CA_CERTS)
     if REDIS_SSL_CERTFILE and REDIS_SSL_KEYFILE:
-        context.load_cert_chain(REDIS_SSL_CERTFILE, REDIS_SSL_KEYFILE)
+        redis_ssl.load_cert_chain(REDIS_SSL_CERTFILE, REDIS_SSL_KEYFILE)
 
 CHANNEL_LAYERS = {
     'default': {
@@ -100,7 +102,7 @@ CHANNEL_LAYERS = {
                 'address': (CONFIG.REDIS_HOST, CONFIG.REDIS_PORT),
                 'db': CONFIG.REDIS_DB_WS,
                 'password': CONFIG.REDIS_PASSWORD or None,
-                'ssl':  context
+                'ssl':  redis_ssl
             }],
         },
     },
@@ -113,7 +115,7 @@ CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'celery')
 
 # Celery using redis as broker
 CELERY_BROKER_URL = '%(protocol)s://:%(password)s@%(host)s:%(port)s/%(db)s' % {
-    'protocol': 'rediss' if CONFIG.REDIS_USE_SSL else 'redis',
+    'protocol': 'rediss' if REDIS_USE_SSL else 'redis',
     'password': CONFIG.REDIS_PASSWORD,
     'host': CONFIG.REDIS_HOST,
     'port': CONFIG.REDIS_PORT,
@@ -131,7 +133,7 @@ CELERY_WORKER_REDIRECT_STDOUTS = True
 CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
 CELERY_TASK_SOFT_TIME_LIMIT = 3600
 
-if CONFIG.REDIS_USE_SSL:
+if REDIS_USE_SSL:
     CELERY_BROKER_USE_SSL = CELERY_REDIS_BACKEND_USE_SSL = {
         'ssl_cert_reqs': REDIS_SSL_REQUIRED,
         'ssl_ca_certs': REDIS_SSL_CA_CERTS,

apps/users/models/user.py

@@ -721,6 +721,8 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser):
     dingtalk_id = models.CharField(null=True, default=None, unique=True, max_length=128, verbose_name=_('DingTalk'))
     feishu_id = models.CharField(null=True, default=None, unique=True, max_length=128, verbose_name=_('FeiShu'))
 
+    DATE_EXPIRED_WARNING_DAYS = 5
+
     def __str__(self):
         return '{0.name}({0.username})'.format(self)
 
@@ -776,7 +778,7 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser):
 
     @property
     def will_expired(self):
-        if 0 <= self.expired_remain_days < 5:
+        if 0 <= self.expired_remain_days <= self.DATE_EXPIRED_WARNING_DAYS:
             return True
         else:
             return False

apps/users/tasks.py

@@ -3,6 +3,7 @@
 
 from celery import shared_task
 from django.conf import settings
+from django.utils import timezone
 
 from users.notifications import PasswordExpirationReminderMsg
 from ops.celery.utils import (
@@ -49,7 +50,11 @@ def check_password_expired_periodic():
 
 @shared_task
 def check_user_expired():
-    users = User.get_nature_users().filter(source=User.Source.local)
+    date_expired_lt = timezone.now() + timezone.timedelta(days=User.DATE_EXPIRED_WARNING_DAYS)
+    users = User.get_nature_users()\
+        .filter(source=User.Source.local)\
+        .filter(date_expired__lt=date_expired_lt)
+
     for user in users:
         if not user.is_valid:
             continue
@@ -57,7 +62,6 @@ def check_user_expired():
             continue
         msg = "The user {} will expires in {} days"
         logger.info(msg.format(user, user.expired_remain_days))
-
         UserExpirationReminderMsg(user).publish_async()
 
 

utils/start_celery_beat.py

@@ -10,39 +10,29 @@ from redis import Redis
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 APPS_DIR = os.path.join(BASE_DIR, 'apps')
+CERTS_DIR = os.path.join(BASE_DIR, 'data', 'certs')
 
-sys.path.insert(0, BASE_DIR)
+sys.path.insert(0, APPS_DIR)
-from apps.jumpserver.const import CONFIG
+from jumpserver import settings
 
 os.environ.setdefault('PYTHONOPTIMIZE', '1')
 if os.getuid() == 0:
     os.environ.setdefault('C_FORCE_ROOT', '1')
 
-REDIS_SSL_KEYFILE = os.path.join(BASE_DIR, 'data', 'certs', 'redis_client.key')
-if not os.path.exists(REDIS_SSL_KEYFILE):
-    REDIS_SSL_KEYFILE = None
-
-REDIS_SSL_CERTFILE = os.path.join(BASE_DIR, 'data', 'certs', 'redis_client.crt')
-if not os.path.exists(REDIS_SSL_CERTFILE):
-    REDIS_SSL_CERTFILE = None
-
-REDIS_SSL_CA_CERTS = os.path.join(BASE_DIR, 'data', 'certs', 'redis_ca.crt')
-if not os.path.exists(REDIS_SSL_CA_CERTS):
-    REDIS_SSL_CA_CERTS = os.path.join(BASE_DIR, 'data', 'certs', 'redis_ca.pem')
-
 params = {
-    'host': CONFIG.REDIS_HOST,
+    'host': settings.REDIS_HOST,
-    'port': CONFIG.REDIS_PORT,
+    'port': settings.REDIS_PORT,
-    'password': CONFIG.REDIS_PASSWORD,
+    'password': settings.REDIS_PASSWORD,
-    "ssl": CONFIG.REDIS_USE_SSL,
+    'ssl': settings.REDIS_USE_SSL,
-    'ssl_cert_reqs': CONFIG.REDIS_SSL_REQUIRED,
+    'ssl_cert_reqs': settings.REDIS_SSL_REQUIRED,
-    "ssl_keyfile": REDIS_SSL_KEYFILE,
+    'ssl_keyfile': settings.REDIS_SSL_KEYFILE,
-    "ssl_certfile": REDIS_SSL_CERTFILE,
+    'ssl_certfile': settings.REDIS_SSL_CERTFILE,
-    "ssl_ca_certs": REDIS_SSL_CA_CERTS
+    'ssl_ca_certs': settings.REDIS_SSL_CA_CERTS
 }
+print("Pamras: ", params)
 redis = Redis(**params)
 scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
-
+processes = []
 cmd = [
     'celery',
     '-A', 'ops',
@@ -52,8 +42,6 @@ cmd = [
     '--max-interval', '60'
 ]
 
-processes = []
-
 
 def stop_beat_process(sig, frame):
     for p in processes: