From 24520f1e8b49f4c704d938cb6e752baa0eef083d Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:48:54 +0800 Subject: [PATCH] perf: add ANSIBLE_DOCKER_ENABLED configuration (#16920) * perf: test ansible executor github ci * perf: for test ci * fix: update Dockerfile to include docker-cli and change ansible container network to jms_net * fix: Removed cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES deprecation warning * perf: update ansible and ansible-runner versions in execution environment and requirements * fix: local python interpreter error * perf: add ANSIBLE_DOCKER_ENABLED configuration * perf: add latest tag for ansible-executor image in build workflow --------- Co-authored-by: wangruidong <940853815@qq.com> --- .github/workflows/build-ansible-executor.yml | 4 +- Dockerfile | 3 +- apps/jumpserver/conf.py | 1 + apps/jumpserver/settings/custom.py | 1 + apps/libs/ansible/ansible.cfg | 2 +- apps/ops/ansible/docker.py | 76 +++++++++++++++++++ apps/ops/ansible/exception.py | 6 +- apps/ops/ansible/runner.py | 56 +++++--------- apps/settings/serializers/feature.py | 14 ++++ .../execution-environment.yml | 4 +- .../ansible_executor/requirements-python.txt | 8 +- 11 files changed, 129 insertions(+), 46 deletions(-) create mode 100644 apps/ops/ansible/docker.py diff --git a/.github/workflows/build-ansible-executor.yml b/.github/workflows/build-ansible-executor.yml index 5e9d456ad..3ccb58f4c 100644 --- a/.github/workflows/build-ansible-executor.yml +++ b/.github/workflows/build-ansible-executor.yml @@ -70,4 +70,6 @@ jobs: push: true context: utils/ansible_executor/context file: utils/ansible_executor/context/Containerfile - tags: jumpserver/ansible-executor:${{ env.IMAGE_TAG }} + tags: | + jumpserver/ansible-executor:${{ env.IMAGE_TAG }} + jumpserver/ansible-executor:latest diff --git a/Dockerfile b/Dockerfile index 08f43dd09..7cb33fae0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,8 @@ ARG TOOLS=" \ postgresql-client \ openssh-client \ sshpass \ - bubblewrap" + bubblewrap \ + docker-cli" ARG APT_MIRROR=http://deb.debian.org diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index d3f065d85..89de0c1b0 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -602,6 +602,7 @@ class Config(dict): 'SECURITY_MFA_AUTH_ENABLED_FOR_THIRD_PARTY': True, 'SECURITY_MFA_BY_EMAIL': False, 'SECURITY_COMMAND_EXECUTION': False, + 'ANSIBLE_DOCKER_ENABLED': True, 'SECURITY_COMMAND_BLACKLIST': [ 'reboot', 'shutdown', 'poweroff', 'halt', 'dd', 'half', 'top' ], diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index 5048c2ca1..14030e248 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -43,6 +43,7 @@ SECURITY_MFA_BY_EMAIL = CONFIG.SECURITY_MFA_BY_EMAIL SECURITY_MAX_IDLE_TIME = CONFIG.SECURITY_MAX_IDLE_TIME # Unit: minute SECURITY_MAX_SESSION_TIME = CONFIG.SECURITY_MAX_SESSION_TIME # Unit: hour SECURITY_COMMAND_EXECUTION = CONFIG.SECURITY_COMMAND_EXECUTION +ANSIBLE_DOCKER_ENABLED = CONFIG.ANSIBLE_DOCKER_ENABLED SECURITY_COMMAND_BLACKLIST = CONFIG.SECURITY_COMMAND_BLACKLIST SECURITY_PASSWORD_EXPIRATION_TIME_ADMIN = CONFIG.SECURITY_PASSWORD_EXPIRATION_TIME_ADMIN # Unit: day SECURITY_PASSWORD_EXPIRATION_TIME = CONFIG.SECURITY_PASSWORD_EXPIRATION_TIME # Unit: day diff --git a/apps/libs/ansible/ansible.cfg b/apps/libs/ansible/ansible.cfg index 73bd7089d..26360f5ec 100644 --- a/apps/libs/ansible/ansible.cfg +++ b/apps/libs/ansible/ansible.cfg @@ -7,7 +7,7 @@ timeout = 65 [privilege_escalation] [paramiko_connection] [ssh_connection] -# Docker 隔离下 ControlMaster 在 bind mount 上无法创建 mux socket（macOS 尤其明显） +# for test ssh_args = -o ControlMaster=no -o ControlPersist=no pipelining = True diff --git a/apps/ops/ansible/docker.py b/apps/ops/ansible/docker.py new file mode 100644 index 000000000..1e4a7d43a --- /dev/null +++ b/apps/ops/ansible/docker.py @@ -0,0 +1,76 @@ +import os +import shutil + +from django.conf import settings +from django.utils.translation import gettext_lazy as _ + +from common.utils.safe import safe_run_cmd +from .exception import AnsibleDockerImageNotFound + +ANSIBLE_EE_IMAGE = 'jumpserver/ansible-executor:latest' +ANSIBLE_EE_PYTHON_INTERPRETER = '/usr/bin/python3.11' + +__all__ = [ + 'ANSIBLE_EE_IMAGE', + 'ANSIBLE_EE_PYTHON_INTERPRETER', + 'use_ansible_docker_isolation', + 'docker_extravars', + 'docker_isolation_kwargs', + 'prepare_isolated_ansible_cfg', + 'stage_inventory_for_docker', + 'ensure_ansible_docker_image', +] + + +def use_ansible_docker_isolation(): + return settings.ANSIBLE_DOCKER_ENABLED + + +def docker_extravars(extra_vars): + extravars = dict(extra_vars or {}) + if use_ansible_docker_isolation(): + extravars.setdefault('local_python_interpreter', ANSIBLE_EE_PYTHON_INTERPRETER) + return extravars + + +def docker_isolation_kwargs(): + return { + 'process_isolation': True, + 'process_isolation_executable': 'docker', + 'container_image': ANSIBLE_EE_IMAGE, + 'container_options': ['--network=jms_net'], + } + + +def prepare_isolated_ansible_cfg(project_dir): + if not use_ansible_docker_isolation(): + return + src = os.path.join(settings.APPS_DIR, 'libs', 'ansible', 'ansible.cfg') + dst = os.path.join(project_dir, 'ansible.cfg') + shutil.copyfile(src, dst) + + +def stage_inventory_for_docker(project_dir, inventory_path): + if not use_ansible_docker_isolation(): + return inventory_path + standard_dir = os.path.join(project_dir, 'inventory') + standard_path = os.path.join(standard_dir, 'hosts') + if os.path.realpath(inventory_path) == os.path.realpath(standard_path): + return standard_path + os.makedirs(standard_dir, mode=0o700, exist_ok=True) + shutil.copy2(inventory_path, standard_path) + return standard_path + + +def ensure_ansible_docker_image(): + if not use_ansible_docker_isolation(): + return + result = safe_run_cmd(['docker', 'image', 'inspect', ANSIBLE_EE_IMAGE]) + if not result or result.returncode != 0: + raise AnsibleDockerImageNotFound( + _('Ansible Docker image "%(image)s" not found. ' + 'You can disable this option in System Settings - Feature Settings - Job Center - ' + 'Ansible Docker isolation to run locally. ' + 'Please run: docker pull %(image)s') + % {'image': ANSIBLE_EE_IMAGE} + ) diff --git a/apps/ops/ansible/exception.py b/apps/ops/ansible/exception.py index dc5926f05..64cc34be1 100644 --- a/apps/ops/ansible/exception.py +++ b/apps/ops/ansible/exception.py @@ -1,5 +1,9 @@ -__all__ = ['CommandInBlackListException'] +__all__ = ['CommandInBlackListException', 'AnsibleDockerImageNotFound'] class CommandInBlackListException(Exception): pass + + +class AnsibleDockerImageNotFound(Exception): + pass diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index ccb4bd260..757972be0 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -10,43 +10,25 @@ from common.utils.yml import sanitize_ansible_inventory_json, sanitize_ansible_p from ..utils import get_ansible_log_verbosity from .callback import DefaultCallback +from .docker import ( + docker_extravars, + docker_isolation_kwargs, + ensure_ansible_docker_image, + prepare_isolated_ansible_cfg, + stage_inventory_for_docker, + use_ansible_docker_isolation, +) from .exception import CommandInBlackListException from .interface import interface __all__ = ['AdHocRunner', 'PlaybookRunner', 'SuperPlaybookRunner', 'UploadFileRunner'] -ANSIBLE_EE_IMAGE = 'jumpserver/ansible-executor:latest' - - -def use_ansible_docker_isolation(): - """Production runs ansible in EE container; dev runs in celery worker.""" - return not settings.DEBUG_DEV - - -def docker_isolation_kwargs(): - return { - 'process_isolation': True, - 'process_isolation_executable': 'docker', - 'container_image': ANSIBLE_EE_IMAGE, - 'container_options': ['--network=host'], - } - - -def prepare_isolated_ansible_cfg(project_dir): - """Copy ansible.cfg into job dir so the EE container picks up SSH settings.""" - if not use_ansible_docker_isolation(): - return - src = os.path.join(settings.APPS_DIR, 'libs', 'ansible', 'ansible.cfg') - dst = os.path.join(project_dir, 'ansible.cfg') - shutil.copyfile(src, dst) - class AdHocRunner: cmd_modules_choices = ('shell', 'raw', 'command', 'script', 'win_shell') need_local_connection_modules_choices = ("mysql", "postgresql", "sqlserver", "huawei") - def __init__(self, inventory, job_module, module, module_args='', pattern='*', project_dir='/tmp/', - extra_vars=None, + def __init__(self, inventory, job_module, module, module_args='', pattern='*', project_dir='/tmp/', extra_vars=None, dry_run=False, timeout=-1): if extra_vars is None: extra_vars = {} @@ -82,7 +64,7 @@ class AdHocRunner: verbosity = get_ansible_log_verbosity(verbosity) if not os.path.exists(self.project_dir): - os.mkdir(self.project_dir, 0o755) + os.makedirs(self.project_dir, 0o755, exist_ok=True) private_env = safe_join(self.project_dir, 'env') if os.path.exists(private_env): shutil.rmtree(private_env) @@ -91,7 +73,7 @@ class AdHocRunner: run_kwargs = { 'timeout': self.timeout if self.timeout > 0 else None, - 'extravars': self.extra_vars, + 'extravars': docker_extravars(self.extra_vars), 'envvars': self.envs, 'host_pattern': self.pattern, 'private_data_dir': self.project_dir, @@ -105,7 +87,7 @@ class AdHocRunner: } if use_ansible_docker_isolation(): run_kwargs.update(docker_isolation_kwargs()) - + ensure_ansible_docker_image() interface.run(**run_kwargs) return self.cb @@ -135,7 +117,8 @@ class PlaybookRunner: entry = os.path.basename(self.playbook) playbook_dir = os.path.dirname(self.playbook) project_playbook_dir = os.path.join(self.project_dir, "project") - shutil.copytree(playbook_dir, project_playbook_dir, dirs_exist_ok=True) + if os.path.realpath(playbook_dir) != os.path.realpath(project_playbook_dir): + shutil.copytree(playbook_dir, project_playbook_dir, dirs_exist_ok=True) self.playbook = entry def prepare_safe_inputs(self): @@ -169,17 +152,19 @@ class PlaybookRunner: shutil.rmtree(private_env) prepare_isolated_ansible_cfg(self.project_dir) + inventory = stage_inventory_for_docker(self.project_dir, self.inventory) kwargs = dict(kwargs) if use_ansible_docker_isolation(): kwargs.update(docker_isolation_kwargs()) + ensure_ansible_docker_image() elif self.isolate and not is_macos(): kwargs['process_isolation'] = True kwargs['process_isolation_executable'] = 'bwrap' interface.run( private_data_dir=self.project_dir, - inventory=self.inventory, + inventory=inventory, playbook=self.playbook, verbosity=verbosity, event_handler=self.cb.event_handler, @@ -187,7 +172,7 @@ class PlaybookRunner: # Docker EE workdir must be the staged playbook dir (not private_data_dir root). host_cwd=self.playbook_project_dir, envvars=self.envs, - extravars=self.extra_vars, + extravars=docker_extravars(self.extra_vars), **kwargs ) return self.cb @@ -224,7 +209,7 @@ class UploadFileRunner: def run(self, verbosity=0, **kwargs): if not os.path.exists(self.project_dir): - os.makedirs(self.project_dir, mode=0o755) + os.makedirs(self.project_dir, mode=0o755, exist_ok=True) prepare_isolated_ansible_cfg(self.project_dir) src_path = self.stage_upload_files() @@ -244,11 +229,10 @@ class UploadFileRunner: } if use_ansible_docker_isolation(): run_kwargs.update(docker_isolation_kwargs()) - + ensure_ansible_docker_image() interface.run(**run_kwargs) try: shutil.rmtree(self.share_src_dir) except OSError as e: print(f"del upload tmp dir {self.share_src_dir} failed! {e}") return self.cb - return self.cb diff --git a/apps/settings/serializers/feature.py b/apps/settings/serializers/feature.py index f1253af17..4a4080cf9 100644 --- a/apps/settings/serializers/feature.py +++ b/apps/settings/serializers/feature.py @@ -6,6 +6,7 @@ from rest_framework import serializers from common.serializers.fields import EncryptedField from common.utils import date_expired_default +from ops.ansible.docker import ANSIBLE_EE_IMAGE __all__ = [ 'AnnouncementSettingSerializer', 'OpsSettingSerializer', 'VaultSettingSerializer', @@ -17,6 +18,14 @@ from settings.const import ( ChatAITypeChoices, GPTModelChoices, DeepSeekModelChoices, ChatAIMethodChoices ) +ANSIBLE_DOCKER_HELP_TEXT = _( + 'Run Ansible jobs in Docker execution environment (%(image)s). ' + 'You can disable this option in System Settings - Feature Settings - Job Center - ' + 'Ansible Docker isolation to run locally. ' + 'If the image is missing, pull it on the ansible worker: ' + 'docker pull %(image)s' +) % {'image': ANSIBLE_EE_IMAGE} + class AnnouncementSerializer(serializers.Serializer): ID = serializers.CharField(required=False, allow_blank=True, allow_null=True) @@ -204,6 +213,11 @@ class OpsSettingSerializer(serializers.Serializer): required=False, label=_('Adhoc command'), help_text=_('Allow users to execute batch commands in the Workbench - Job Center - Adhoc') ) + ANSIBLE_DOCKER_ENABLED = serializers.BooleanField( + required=False, + label=_('Ansible Docker isolation'), + help_text=ANSIBLE_DOCKER_HELP_TEXT, + ) SECURITY_COMMAND_BLACKLIST = serializers.ListField( child=serializers.CharField(max_length=1024), label=_('Command blacklist'), diff --git a/utils/ansible_executor/execution-environment.yml b/utils/ansible_executor/execution-environment.yml index 06169dfbd..27416eb1e 100644 --- a/utils/ansible_executor/execution-environment.yml +++ b/utils/ansible_executor/execution-environment.yml @@ -15,9 +15,9 @@ dependencies: # 与 pyproject.toml [tool.uv.sources] 保持一致，不要用 PyPI 官方包 ansible_core: - package_pip: https://github.com/jumpserver-dev/ansible/archive/refs/tags/v2.14.1.7.zip + package_pip: https://github.com/jumpserver-dev/ansible/archive/refs/tags/v2.16.18.3.zip ansible_runner: - package_pip: https://github.com/jumpserver-dev/ansible-runner/archive/refs/tags/2.4.0.1.zip + package_pip: https://github.com/jumpserver-dev/ansible-runner/archive/refs/tags/v2.4.0.2.zip galaxy: collections: diff --git a/utils/ansible_executor/requirements-python.txt b/utils/ansible_executor/requirements-python.txt index 05978f43a..f3061bf5c 100644 --- a/utils/ansible_executor/requirements-python.txt +++ b/utils/ansible_executor/requirements-python.txt @@ -1,10 +1,10 @@ # Aligned with JumpServer pyproject.toml ansible-related Python deps. -paramiko==3.2.0 +paramiko==3.5.1 sshtunnel==0.4.0 pywinrm==0.4.3 telnetlib3==4.0.2 mysqlclient==2.2.4 -pymssql==2.3.4 +pymssql==2.3.10 pymongo==4.6.3 -oracledb==1.4.0 -pyfreerdp==0.0.2 +oracledb==3.4.2 +pyfreerdp==0.0.4