perf: LDAP strict sync

apps/jumpserver/conf.py

@@ -9,16 +9,15 @@
 """
 import base64
 import copy
-import errno
 import json
 import logging
 import os
 import re
-import sys
 import types
 from importlib import import_module
 from urllib.parse import urljoin, urlparse, quote
 
+import sys
 import yaml
 from django.urls import reverse_lazy
 from django.utils.translation import gettext_lazy as _
@@ -290,6 +289,7 @@ class Config(dict):
         'AUTH_LDAP_START_TLS': False,
         'AUTH_LDAP_USER_ATTR_MAP': {"username": "cn", "name": "sn", "email": "mail"},
         'AUTH_LDAP_CONNECT_TIMEOUT': 10,
+        'AUTH_LDAP_STRICT_SYNC': False,
         'AUTH_LDAP_CACHE_TIMEOUT': 0,
         'AUTH_LDAP_SEARCH_PAGED_SIZE': 1000,
         'AUTH_LDAP_SYNC_IS_PERIODIC': False,
@@ -310,6 +310,7 @@ class Config(dict):
         'AUTH_LDAP_HA_START_TLS': False,
         'AUTH_LDAP_HA_USER_ATTR_MAP': {"username": "cn", "name": "sn", "email": "mail"},
         'AUTH_LDAP_HA_CONNECT_TIMEOUT': 10,
+        'AUTH_LDAP_HA_STRICT_SYNC': False,
         'AUTH_LDAP_HA_CACHE_TIMEOUT': 0,
         'AUTH_LDAP_HA_SEARCH_PAGED_SIZE': 1000,
         'AUTH_LDAP_HA_SYNC_IS_PERIODIC': False,

apps/jumpserver/settings/auth.py

@@ -42,6 +42,7 @@ AUTH_LDAP_CONNECTION_OPTIONS = {
     ldap.OPT_TIMEOUT: CONFIG.AUTH_LDAP_CONNECT_TIMEOUT,
     ldap.OPT_NETWORK_TIMEOUT: CONFIG.AUTH_LDAP_CONNECT_TIMEOUT
 }
+AUTH_LDAP_STRICT_SYNC = CONFIG.AUTH_LDAP_STRICT_SYNC
 AUTH_LDAP_CACHE_TIMEOUT = CONFIG.AUTH_LDAP_CACHE_TIMEOUT
 AUTH_LDAP_ALWAYS_UPDATE_USER = True
 
@@ -80,6 +81,7 @@ AUTH_LDAP_HA_CONNECTION_OPTIONS = {
     ldap.OPT_TIMEOUT: CONFIG.AUTH_LDAP_HA_CONNECT_TIMEOUT,
     ldap.OPT_NETWORK_TIMEOUT: CONFIG.AUTH_LDAP_HA_CONNECT_TIMEOUT
 }
+AUTH_LDAP_HA_STRICT_SYNC = CONFIG.AUTH_LDAP_HA_STRICT_SYNC
 AUTH_LDAP_HA_CACHE_TIMEOUT = CONFIG.AUTH_LDAP_HA_CACHE_TIMEOUT
 AUTH_LDAP_HA_ALWAYS_UPDATE_USER = True
 

apps/settings/api/settings.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 #
-import re
 
 from django.conf import settings
 from django.core.cache import cache
@@ -14,6 +13,7 @@ from rest_framework.views import APIView
 from common.utils import get_logger
 from jumpserver.conf import Config
 from rbac.permissions import RBACPermission
+from users.models import User
 from .. import serializers
 from ..models import Setting
 from ..signals import category_setting_updated
@@ -182,7 +182,7 @@ class SettingsApi(generics.RetrieveUpdateAPIView):
         if hasattr(serializer, 'post_save'):
             serializer.post_save()
         self.send_signal(serializer)
-        if self.request.query_params.get('category') == 'ldap':
+        if self.request.query_params.get('category') == User.Source.ldap.value:
             self.clean_ldap_user_dn_cache()
 
     @staticmethod

apps/settings/serializers/auth/ldap.py

@@ -84,6 +84,10 @@ class LDAPSettingSerializer(LDAPSerializerMixin, serializers.Serializer):
         min_value=1, max_value=300,
         required=False, label=_('Connect timeout (s)'),
     )
+    AUTH_LDAP_STRICT_SYNC = serializers.BooleanField(
+        required=False, label=_('Strict sync'),
+        help_text=_('In strict mode, users not found in LDAP will be disabled during full or automatic sync')
+    )
     AUTH_LDAP_CACHE_TIMEOUT = serializers.IntegerField(
         min_value=0, max_value=3600 * 24 * 30 * 12,
         default=0,

apps/settings/serializers/auth/ldap_ha.py

@@ -4,7 +4,6 @@ from rest_framework import serializers
 from common.serializers.fields import EncryptedField
 from .base import OrgListField
 from .mixin import LDAPSerializerMixin
-from ops.mixin import PeriodTaskSerializerMixin
 
 __all__ = ['LDAPHATestConfigSerializer', 'LDAPHASettingSerializer']
 
@@ -67,6 +66,10 @@ class LDAPHASettingSerializer(LDAPSerializerMixin, serializers.Serializer):
         min_value=1, max_value=300,
         required=False, label=_('Connect timeout (s)'),
     )
+    AUTH_LDAP_HA_STRICT_SYNC = serializers.BooleanField(
+        required=False, label=_('Strict sync'),
+        help_text=_('In strict mode, users not found in LDAP will be disabled during full or automatic sync')
+    )
     AUTH_LDAP_HA_CACHE_TIMEOUT = serializers.IntegerField(
         min_value=0, max_value=3600 * 24 * 30 * 12,
         default=0,

apps/settings/tasks/ldap.py

@@ -1,6 +1,5 @@
 # coding: utf-8
 import time
-
 from celery import shared_task
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
@@ -24,7 +23,7 @@ __all__ = [
 logger = get_logger(__file__)
 
 
-def sync_ldap_user(category='ldap'):
+def sync_ldap_user(category=User.Source.ldap.value):
     LDAPSyncUtil(category=category).perform_sync()
 
 
@@ -33,7 +32,7 @@ def perform_import(category, util_server):
     time_start_display = local_now_display()
     logger.info(f"Start import {category} ldap user task")
 
-    util_import = LDAPImportUtil()
+    util_import = LDAPImportUtil(category)
     users = util_server.search()
 
     if settings.XPACK_ENABLED:
@@ -44,12 +43,7 @@ def perform_import(category, util_server):
         default_org = Organization.default()
 
     orgs = list(set([Organization.get_instance(org_id, default=default_org) for org_id in org_ids]))
-    new_users, errors = util_import.perform_import(users, orgs)
-
-    if errors:
-        logger.error(f"Imported {category} LDAP users errors: {errors}")
-    else:
-        logger.info(f"Imported {len(users)} {category} users successfully")
+    new_users, errors, disable_users = util_import.perform_import(users, orgs)
 
     receivers_setting = f"AUTH_{category.upper()}_SYNC_RECEIVERS"
     if getattr(settings, receivers_setting, None):
@@ -76,7 +70,7 @@ def perform_import(category, util_server):
     )
 )
 def import_ldap_user():
-    perform_import('ldap', LDAPServerUtil())
+    perform_import(User.Source.ldap.value, LDAPServerUtil())
 
 
 @shared_task(
@@ -86,7 +80,8 @@ def import_ldap_user():
     )
 )
 def import_ldap_ha_user():
-    perform_import('ldap_ha', LDAPServerUtil(category='ldap_ha'))
+    category = User.Source.ldap_ha.value
+    perform_import(category, LDAPServerUtil(category=category))
 
 
 def register_periodic_task(task_name, task_func, interval_key, enabled_key, crontab_key, **kwargs):

apps/settings/utils/ldap.py

@@ -47,7 +47,7 @@ LDAP_USE_CACHE_FLAGS = [1, '1', 'true', 'True', True]
 
 class LDAPConfig(object):
 
-    def __init__(self, config=None, category='ldap'):
+    def __init__(self, config=None, category=User.Source.ldap.value):
         self.server_uri = None
         self.bind_dn = None
         self.password = None
@@ -73,7 +73,7 @@ class LDAPConfig(object):
         self.auth_ldap = config.get('auth_ldap')
 
     def load_from_settings(self):
-        prefix = 'AUTH_LDAP' if self.category == 'ldap' else 'AUTH_LDAP_HA'
+        prefix = 'AUTH_LDAP' if self.category == User.Source.ldap.value else 'AUTH_LDAP_HA'
         self.server_uri = getattr(settings, f"{prefix}_SERVER_URI")
         self.bind_dn = getattr(settings, f"{prefix}_BIND_DN")
         self.password = getattr(settings, f"{prefix}_BIND_PASSWORD")
@@ -86,7 +86,7 @@ class LDAPConfig(object):
 
 class LDAPServerUtil(object):
 
-    def __init__(self, config=None, category='ldap'):
+    def __init__(self, config=None, category=User.Source.ldap.value):
         if isinstance(config, dict):
             self.config = LDAPConfig(config=config)
         elif isinstance(config, LDAPConfig):
@@ -234,14 +234,11 @@ class LDAPServerUtil(object):
 
 class LDAPCacheUtil(object):
 
-    def __init__(self, category='ldap'):
+    def __init__(self, category=User.Source.ldap.value):
         self.search_users = None
         self.search_value = None
         self.category = category
-        if self.category == 'ldap':
-            self.cache_key_users = 'CACHE_KEY_LDAP_USERS'
-        else:
-            self.cache_key_users = 'CACHE_KEY_LDAP_HA_USERS'
+        self.cache_key_users = 'CACHE_KEY_{}_USERS'.format(self.category.upper())
 
     def set_users(self, users):
         logger.info('Set ldap users to cache, count: {}'.format(len(users)))
@@ -295,7 +292,7 @@ class LDAPSyncUtil(object):
     TASK_STATUS_IS_RUNNING = 'RUNNING'
     TASK_STATUS_IS_OVER = 'OVER'
 
-    def __init__(self, category='ldap'):
+    def __init__(self, category=User.Source.ldap.value):
         self.server_util = LDAPServerUtil(category=category)
         self.cache_util = LDAPCacheUtil(category=category)
         self.task_error_msg = None
@@ -371,8 +368,9 @@ class LDAPSyncUtil(object):
 class LDAPImportUtil(object):
     user_group_name_prefix = 'AD '
 
-    def __init__(self):
-        pass
+    def __init__(self, category=User.Source.ldap.value, is_sync_all=True):
+        self.category = category
+        self.is_sync_all = is_sync_all
 
     @staticmethod
     def get_user_email(user):
@@ -384,7 +382,7 @@ class LDAPImportUtil(object):
     def update_or_create(self, user):
         user['email'] = self.get_user_email(user)
         if user['username'] not in ['admin']:
-            user['source'] = User.Source.ldap.value
+            user['source'] = self.category
         user.pop('status', None)
         obj, created = User.objects.update_or_create(
             username=user['username'], defaults=user
@@ -435,7 +433,29 @@ class LDAPImportUtil(object):
         for org in orgs:
             self.bind_org(org, objs, group_users_mapper)
         logger.info('End perform import ldap users')
-        return new_users, errors
+        # 禁止ldap 不存在的用户的
+        disable_usernames = []
+        if self.strict_sync_enabled and self.is_sync_all:
+            disable_usernames = self.disable_not_exist_users(users)
+
+        if errors:
+            logger.error(f"Imported {self.category.upper()} users errors: {errors}")
+        else:
+            logger.info(f"Imported {len(users)} {self.category.upper()} users successfully")
+        return new_users, errors, disable_usernames
+
+    @property
+    def strict_sync_enabled(self):
+        return getattr(settings, 'AUTH_{}_STRICT_SYNC'.format(self.category.upper()), False)
+
+    def disable_not_exist_users(self, users):
+        ldap_users = [user['username'] for user in users]
+        disable_users = User.objects.filter(source=self.category, is_active=True).exclude(username__in=ldap_users).all()
+        disable_usernames = disable_users.values_list('username', flat=True)
+        disable_usernames = list(map(str, disable_usernames))
+        disable_users.update(is_active=False)
+        logger.info(f"Disable {len(disable_usernames)} {self.category.upper()} users successfully")
+        return disable_usernames
 
     def exit_user_group(self, user_groups_mapper):
         # 通过对比查询本次导入用户需要移除的用户组
@@ -485,10 +505,10 @@ class LDAPTestUtil(object):
     class LDAPBeforeLoginCheckError(LDAPExceptionError):
         pass
 
-    def __init__(self, config=None, category='ldap'):
+    def __init__(self, config=None, category=User.Source.ldap.value):
         self.config = LDAPConfig(config, category)
         self.user_entries = []
-        if category == 'ldap':
+        if category == User.Source.ldap.value:
             self.backend = LDAPAuthorizationBackend()
         else:
             self.backend = LDAPHAAuthorizationBackend()
@@ -665,7 +685,7 @@ class LDAPTestUtil(object):
     # test login
 
     def _test_before_login_check(self, username, password):
-        from settings.ws import CACHE_KEY_LDAP_TEST_CONFIG_TASK_STATUS, TASK_STATUS_IS_OVER
+        from settings.ws import CACHE_KEY_LDAP_TEST_CONFIG_TASK_STATUS
         if not cache.get(CACHE_KEY_LDAP_TEST_CONFIG_TASK_STATUS):
             self.test_config()
 

apps/settings/ws.py

@@ -1,30 +1,31 @@
 # -*- coding: utf-8 -*-
 #
-import json
 import asyncio
+import json
+from urllib.parse import parse_qs
 
 from asgiref.sync import sync_to_async
 from channels.generic.websocket import AsyncJsonWebsocketConsumer
-from django.core.cache import cache
 from django.conf import settings
-from django.utils.translation import gettext_lazy as _
+from django.core.cache import cache
 from django.utils import translation
-from urllib.parse import parse_qs
+from django.utils.translation import gettext_lazy as _
 
 from common.db.utils import close_old_connections
 from common.utils import get_logger
+from orgs.models import Organization
+from orgs.utils import current_org
 from settings.serializers import (
     LDAPHATestConfigSerializer,
     LDAPTestConfigSerializer,
     LDAPTestLoginSerializer
 )
-from orgs.models import Organization
-from orgs.utils import current_org
 from settings.tasks import sync_ldap_user
 from settings.utils import (
     LDAPServerUtil, LDAPCacheUtil, LDAPImportUtil, LDAPSyncUtil,
     LDAP_USE_CACHE_FLAGS, LDAPTestUtil
 )
+from users.models import User
 from .const import ImportStatus
 from .tools import (
     verbose_ping, verbose_telnet, verbose_nmap,
@@ -130,7 +131,7 @@ class LdapWebsocket(AsyncJsonWebsocketConsumer):
     async def connect(self):
         user = self.scope["user"]
         query = parse_qs(self.scope['query_string'].decode())
-        self.category = query.get('category', ['ldap'])[0]
+        self.category = query.get('category', [User.Source.ldap.value])[0]
         if user.is_authenticated:
             await self.accept()
         else:
@@ -157,7 +158,7 @@ class LdapWebsocket(AsyncJsonWebsocketConsumer):
         close_old_connections()
 
     def get_ldap_config(self, serializer):
-        prefix = 'AUTH_LDAP_' if self.category == 'ldap' else 'AUTH_LDAP_HA_'
+        prefix = 'AUTH_{}_'.format(self.category.upper())
 
         config = {
             'server_uri': serializer.validated_data.get(f"{prefix}SERVER_URI"),
@@ -182,7 +183,7 @@ class LdapWebsocket(AsyncJsonWebsocketConsumer):
         cache.set(task_key, TASK_STATUS_IS_OVER, ttl)
 
     def run_testing_config(self, data):
-        if self.category == 'ldap':
+        if self.category == User.Source.ldap.value:
             serializer = LDAPTestConfigSerializer(data=data)
         else:
             serializer = LDAPHATestConfigSerializer(data=data)
@@ -222,12 +223,17 @@ class LdapWebsocket(AsyncJsonWebsocketConsumer):
                 msg = _('No LDAP user was found')
             else:
                 orgs = self.get_orgs(org_ids)
-                new_users, error_msg = LDAPImportUtil().perform_import(users, orgs)
+                is_sync_all = '*' in username_list
+                new_users, error_msg, disable_users = LDAPImportUtil(
+                    self.category, is_sync_all
+                ).perform_import(users, orgs)
                 ok = True
                 success_count = len(users) - len(error_msg)
                 msg = _('Total {}, success {}, failure {}').format(
                     len(users), success_count, len(error_msg)
                 )
+                if disable_users:
+                    msg += _(', disabled {}').format(len(disable_users))
                 self.set_users_status(users, error_msg)
         except Exception as e:
             msg = str(e)