feat: Update v2.28.20

* fix: 增加 pip 参数 --use-deprecated=legacy-resolver

* fix: 升级依赖 pymssql==2.2.7 oracledb==1.2.2

* fix: 升级依赖 git+https://github.com/marcwimmer/pymssql@fix_cython_compiler_version

* fix: 锁定依赖 Cython==0.29.22

* fix: 锁定依赖 Cython==0.29.35 --no-build-isolation pymssql==2.1.5

* fix: 锁定依赖 Cython==0.29.35 --no-build-isolation pymssql==2.1.5

* fix: 锁定依赖

* fix: 锁定依赖

* fix: 锁定依赖

* fix: 锁定依赖

* fix: 锁定依赖

---------

Co-authored-by: Bai <baijiangjie@gmail.com>

.github/workflows/release-drafter.yml

@@ -21,7 +21,7 @@ jobs:
           TAG=$(basename ${GITHUB_REF})
           VERSION=${TAG/v/}
           wget https://raw.githubusercontent.com/jumpserver/installer/master/quick_start.sh
-          sed -i "s@Version=.*@Version=v${VERSION}@g" quick_start.sh
+          sed -i "s@VERSION=dev@VERSION=v${VERSION}@g" quick_start.sh
           echo "::set-output name=TAG::$TAG"
           echo "::set-output name=VERSION::$VERSION"
       - name: Create Release

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim as stage-build
+FROM python:3.8-slim-bullseye as stage-build
 ARG TARGETARCH
 
 ARG VERSION
@@ -8,7 +8,7 @@ WORKDIR /opt/jumpserver
 ADD . .
 RUN cd utils && bash -ixeu build.sh
 
-FROM python:3.8-slim
+FROM python:3.8-slim-bullseye
 ARG TARGETARCH
 MAINTAINER JumpServer Team <ibuler@qq.com>
 
@@ -38,6 +38,7 @@ ARG TOOLS="                           \
         default-mysql-client          \
         iputils-ping                  \
         locales                       \
+        patch                         \
         procps                        \
         redis-tools                   \
         telnet                        \
@@ -87,8 +88,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     && pip config set global.index-url ${PIP_MIRROR} \
     && pip install --upgrade pip \
     && pip install --upgrade setuptools wheel \
+    && pip install Cython==0.29.35 \
+    && pip install --no-build-isolation pymssql \
     && pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
-    && pip install -r requirements/requirements.txt
+    && pip install -r requirements/requirements.txt --use-deprecated=legacy-resolver
 
 COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
 RUN echo > /opt/jumpserver/config.yml \

Dockerfile-ee

@@ -1,4 +0,0 @@
-ARG VERSION
-FROM registry.fit2cloud.com/jumpserver/xpack:${VERSION} as build-xpack
-FROM jumpserver/core:${VERSION}
-COPY --from=build-xpack /opt/xpack /opt/jumpserver/apps/xpack

GITSHA

@@ -1 +1 @@
-4e4e58480fc061c2e487c64f0f70667e22eaef27
+29ab6bbee7f1ac2dee1826e42b65039f260f6fd0

apps/authentication/api/connection_token.py

@@ -125,6 +125,7 @@ class ConnectionTokenMixin:
             'bookmarktype:i': '3',
             'use redirection server name:i': '0',
             'smart sizing:i': '1',
+            'disableconnectionsharing:i': '1',
             # 'drivestoredirect:s': '*',
             # 'domain:s': ''
             # 'alternate shell:s:': '||MySQLWorkbench',

apps/authentication/backends/pubkey.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 #
-from django.contrib.auth import get_user_model
 from django.conf import settings
+from django.contrib.auth import get_user_model
 
+from common.permissions import ServiceAccountSignaturePermission
 from .base import JMSBaseAuthBackend
 
 UserModel = get_user_model()
@@ -18,6 +19,10 @@ class PublicKeyAuthBackend(JMSBaseAuthBackend):
     def authenticate(self, request, username=None, public_key=None, **kwargs):
         if not public_key:
             return None
+
+        permission = ServiceAccountSignaturePermission()
+        if not permission.has_permission(request, None):
+            return None
         if username is None:
             username = kwargs.get(UserModel.USERNAME_FIELD)
         try:
@@ -26,7 +31,7 @@ class PublicKeyAuthBackend(JMSBaseAuthBackend):
             return None
         else:
             if user.check_public_key(public_key) and \
-                  self.user_can_authenticate(user):
+                    self.user_can_authenticate(user):
                 return user
 
     def get_user(self, user_id):

apps/common/permissions.py

@@ -81,3 +81,38 @@ class UserConfirmation(permissions.BasePermission):
         min_level = ConfirmType.values.index(confirm_type) + 1
         name = 'UserConfirmationLevel{}TTL{}'.format(min_level, ttl)
         return type(name, (cls,), {'min_level': min_level, 'ttl': ttl, 'confirm_type': confirm_type})
+
+
+class ServiceAccountSignaturePermission(permissions.BasePermission):
+    def has_permission(self, request, view):
+        from authentication.models import AccessKey
+        from common.utils.crypto import get_aes_crypto
+        signature = request.META.get('HTTP_X_JMS_SVC', '')
+        if not signature or not signature.startswith('Sign'):
+            return False
+        data = signature[4:].strip()
+        if not data or ':' not in data:
+            return False
+        ak_id, time_sign = data.split(':', 1)
+        if not ak_id or not time_sign:
+            return False
+        ak = AccessKey.objects.filter(id=ak_id).first()
+        if not ak or not ak.is_active:
+            return False
+        if not ak.user or not ak.user.is_active or not ak.user.is_service_account:
+            return False
+        aes = get_aes_crypto(str(ak.secret).replace('-', ''), mode='ECB')
+        try:
+            timestamp = aes.decrypt(time_sign)
+            if not timestamp or not timestamp.isdigit():
+                return False
+            timestamp = int(timestamp)
+            interval = abs(int(time.time()) - timestamp)
+            if interval > 30:
+                return False
+            return True
+        except Exception:
+            return False
+
+    def has_object_permission(self, request, view, obj):
+        return False

apps/common/utils/random.py

@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 #
-import struct
 import random
 import socket
 import string
-
+import struct
 
 string_punctuation = '!#$%&()*+,-.:;<=>?@[]^_~'
 
@@ -19,6 +18,7 @@ def random_ip():
 
 
 def random_string(length: int, lower=True, upper=True, digit=True, special_char=False):
+    random.seed()
     args_names = ['lower', 'upper', 'digit', 'special_char']
     args_values = [lower, upper, digit, special_char]
     args_string = [string.ascii_lowercase, string.ascii_uppercase, string.digits, string_punctuation]

apps/common/utils/verify_code.py

@@ -1,14 +1,13 @@
-from django.core.cache import cache
-from django.conf import settings
-from django.core.mail import send_mail
 from celery import shared_task
+from django.conf import settings
+from django.core.cache import cache
+from django.core.mail import send_mail
 
-from common.sdk.sms.exceptions import CodeError, CodeExpired, CodeSendTooFrequently
-from common.sdk.sms.endpoint import SMS
 from common.exceptions import JMSException
-from common.utils.random import random_string
+from common.sdk.sms.endpoint import SMS
+from common.sdk.sms.exceptions import CodeError, CodeExpired, CodeSendTooFrequently
 from common.utils import get_logger
-
+from common.utils.random import random_string
 
 logger = get_logger(__file__)
 
@@ -27,6 +26,7 @@ class SendAndVerifyCodeUtil(object):
         self.timeout = timeout
         self.backend = backend
         self.key = key or self.KEY_TMPL.format(target)
+        self.verify_key = self.key + '_verify'
         self.other_args = kwargs
 
     def gen_and_send_async(self):
@@ -47,6 +47,11 @@ class SendAndVerifyCodeUtil(object):
             raise
 
     def verify(self, code):
+        times = cache.get(self.verify_key, 0)
+        if times >= 3:
+            self.__clear()
+            raise CodeExpired
+        cache.set(self.verify_key, times + 1, timeout=self.timeout)
         right = cache.get(self.key)
         if not right:
             raise CodeExpired
@@ -59,6 +64,7 @@ class SendAndVerifyCodeUtil(object):
 
     def __clear(self):
         cache.delete(self.key)
+        cache.delete(self.verify_key)
 
     def __ttl(self):
         return cache.ttl(self.key)

apps/ops/api/adhoc.py

@@ -24,7 +24,7 @@ __all__ = [
 
 class TaskViewSet(OrgBulkModelViewSet):
     model = Task
-    filterset_fields = ("name",)
+    filterset_fields = ("name", "adhoc__execution__celery_task_id")
     search_fields = filterset_fields
     serializer_class = TaskSerializer
 
@@ -54,6 +54,7 @@ class TaskRun(generics.RetrieveAPIView):
 
 class AdHocViewSet(viewsets.ModelViewSet):
     queryset = AdHoc.objects.all()
+    filterset_fields = ('execution__celery_task_id', )
     serializer_class = AdHocSerializer
 
     def get_serializer_class(self):

apps/ops/celery/beat/schedulers.py

@@ -0,0 +1,80 @@
+import logging
+from celery.utils.log import get_logger
+from django.db import close_old_connections
+from django.core.exceptions import ObjectDoesNotExist
+from django.db.utils import DatabaseError, InterfaceError
+
+from django_celery_beat.schedulers import DatabaseScheduler as DJDatabaseScheduler
+
+logger = get_logger(__name__)
+debug, info, warning = logger.debug, logger.info, logger.warning
+
+
+__all__ = ['DatabaseScheduler']
+
+
+class DatabaseScheduler(DJDatabaseScheduler):
+
+    def sync(self):
+        if logger.isEnabledFor(logging.DEBUG):
+            debug('Writing entries...')
+        _tried = set()
+        _failed = set()
+        try:
+            close_old_connections()
+
+            while self._dirty:
+                name = self._dirty.pop()
+                try:
+                    # 源码
+                    # self.schedule[name].save()
+                    # _tried.add(name)
+
+                    """
+                    ::Debug Description (2023.07.10)::
+                    
+                    如果调用 self.schedule 可能会导致 self.save() 方法之前重新获取数据库中的数据, 而不是临时设置的 last_run_at 数据
+                    
+                    如果这里调用 self.schedule
+                    那么可能会导致调用 save 的 self.schedule[name] 的 last_run_at 是从数据库中获取回来的老数据
+                    而不是任务执行后临时设置的 last_run_at (在 __next__() 方法中设置的)
+                    当 `max_interval` 间隔之后, 下一个任务检测周期还是会再次执行任务
+                    
+                    ::Demo::
+                    任务信息:
+                        beat config: max_interval = 60s
+                        
+                        任务名称: cap
+                        任务执行周期: 每 3 分钟执行一次
+                        任务最后执行时间: 18:00
+                        
+                    任务第一次执行: 18:03 (执行时设置 last_run_at = 18:03, 此时在内存中)
+                    
+                    任务执行完成后, 
+                    检测到需要 sync, sync 中调用了 self.schedule,
+                    self.schedule 中发现 schedule_changed() 为 True, 需要调用 all_as_schedule()
+                    此时，sync 中调用的 self.schedule[name] 的 last_run_at 是 18:00
+                    这时候在 self.sync() 进行 self.save()
+                    
+                    
+                    beat: Waking up 60s ...
+                    
+                    任务第二次执行: 18:04 (因为获取回来的 last_run_at 是 18:00, entry.is_due() == True)
+                    
+                    ::解决方法::
+                    所以这里为了避免从数据库中获取，直接使用 _schedule #
+                    """
+                    self._schedule[name].save()
+                    _tried.add(name)
+                except (KeyError, TypeError, ObjectDoesNotExist):
+                    _failed.add(name)
+        except DatabaseError as exc:
+            logger.exception('Database error while sync: %r', exc)
+        except InterfaceError:
+            warning(
+                'DatabaseScheduler: InterfaceError in sync(), '
+                'waiting to retry in next call...'
+            )
+        finally:
+            # retry later, only for the failed ones
+            self._dirty |= _failed

apps/ops/serializers/adhoc.py

@@ -17,7 +17,7 @@ class AdHocExecutionSerializer(serializers.ModelSerializer):
         fields_mini = ['id']
         fields_small = fields_mini + [
             'hosts_amount', 'timedelta', 'result', 'summary', 'short_id',
-            'is_finished', 'is_success',
+            'is_finished', 'is_success', 'celery_task_id',
             'date_start', 'date_finished',
         ]
         fields_fk = ['task', 'task_display', 'adhoc', 'adhoc_short_id',]

apps/rbac/builtin.py

@@ -149,7 +149,7 @@ class BuiltinRole:
                 'User': cls.system_user.get_role(),
                 'Auditor': cls.system_auditor.get_role()
             }
-        return cls.system_role_mapper[name]
+        return cls.system_role_mapper.get(name, cls.system_role_mapper['User'])
 
     @classmethod
     def get_org_role_by_old_name(cls, name):
@@ -159,7 +159,7 @@ class BuiltinRole:
                 'User': cls.org_user.get_role(),
                 'Auditor': cls.org_auditor.get_role(),
             }
-        return cls.org_role_mapper[name]
+        return cls.org_role_mapper.get(name, cls.org_role_mapper['User'])
 
     @classmethod
     def sync_to_db(cls, show_msg=False):

apps/tickets/serializers/ticket/common.py

@@ -1,5 +1,5 @@
-from django.db.transaction import atomic
 from django.db.models import Model
+from django.db.transaction import atomic
 from django.utils.translation import ugettext as _
 from rest_framework import serializers
 
@@ -69,8 +69,6 @@ class BaseApplyAssetApplicationSerializer(serializers.Serializer):
             error = _('The expiration date should be greater than the start date')
             raise serializers.ValidationError({'apply_date_expired': error})
 
-        attrs['apply_date_start'] = apply_date_start
-        attrs['apply_date_expired'] = apply_date_expired
         return attrs
 
     @atomic

apps/users/models/user.py

@@ -1,30 +1,27 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-import uuid
 import base64
-import string
-import random
 import datetime
+import uuid
 from typing import Callable
 
-from django.db import models
 from django.conf import settings
-from django.utils import timezone
-from django.core.cache import cache
-from django.contrib.auth.models import AbstractUser
 from django.contrib.auth.hashers import check_password
-from django.utils.translation import ugettext_lazy as _
+from django.contrib.auth.models import AbstractUser
+from django.core.cache import cache
+from django.db import models
 from django.shortcuts import reverse
+from django.utils import timezone
 from django.utils.module_loading import import_string
+from django.utils.translation import ugettext_lazy as _
 
-from orgs.utils import current_org
-from orgs.models import Organization
-from rbac.const import Scope
 from common.db import fields, models as jms_models
 from common.utils import (
     date_expired_default, get_logger, lazyproperty, random_string, bulk_create_with_signal
 )
+from orgs.utils import current_org
+from rbac.const import Scope
 from ..signals import post_user_change_password, post_user_leave_org, pre_user_leave_org
 
 __all__ = ['User', 'UserPasswordHistory']
@@ -518,8 +515,7 @@ class TokenMixin:
         return self.access_keys.first()
 
     def generate_reset_token(self):
-        letter = string.ascii_letters + string.digits
-        token = ''.join([random.choice(letter) for _ in range(50)])
+        token = random_string(50)
         self.set_cache(token)
         return token
 

apps/users/tasks.py

@@ -81,7 +81,6 @@ def check_user_expired_periodic():
 
 
 @shared_task
-@transaction.atomic
 def import_ldap_user():
     logger.info("Start import ldap user task")
     util_server = LDAPServerUtil()

requirements/requirements.txt

@@ -128,9 +128,9 @@ kubernetes==21.7.0
 # DB requirements
 mysqlclient==2.1.0
 PyMySQL==1.0.2
-oracledb==1.0.1
+oracledb==1.2.2
 psycopg2-binary==2.9.1
-pymssql==2.1.5
+# pymssql==2.2.7
 django-mysql==3.9.0
 django-redis==5.2.0
 python-redis-lock==3.7.0

utils/start_celery_beat.py

@@ -54,7 +54,7 @@ else:
     connection_params['port'] = settings.REDIS_PORT
     redis_client = Redis(**connection_params)
 
-scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
+scheduler = "ops.celery.beat.schedulers:DatabaseScheduler"
 processes = []
 cmd = [
     'celery',