perf: 修改 playbook 任务执行

apps/assets/automations/__init__.py

@@ -1 +1 @@
-from .methods import platform_automation_methods
+from .methods import platform_automation_methods, filter_platform_methods

apps/assets/automations/base/manager.py

@@ -1,37 +1,65 @@
 import os
+import shutil
+import yaml
+from copy import deepcopy
+from collections import defaultdict
 
 from django.conf import settings
 from django.utils import timezone
+from django.utils.translation import gettext as _
 
-from ops.ansible import JMSInventory
+from common.utils import get_logger
+from assets.automations.methods import platform_automation_methods
+from ops.ansible import JMSInventory, PlaybookRunner, DefaultCallback
+
+logger = get_logger(__name__)
+
+
+class PlaybookCallback(DefaultCallback):
+    def playbook_on_stats(self, event_data, **kwargs):
+        print("\n*** 分任务结果")
+        super().playbook_on_stats(event_data, **kwargs)
 
 
 class BasePlaybookManager:
+    bulk_size = 100
     ansible_account_policy = 'privileged_first'
 
     def __init__(self, execution):
         self.execution = execution
         self.automation = execution.automation
+        self.method_id_meta_mapper = {
+            method['id']: method
+            for method in platform_automation_methods
+            if method['method'] == self.__class__.method_type()
+        }
+        # 根据执行方式就行分组, 不同资产的改密、推送等操作可能会使用不同的执行方式
+        # 然后根据执行方式分组, 再根据 bulk_size 分组, 生成不同的 playbook
+        # 避免一个 playbook 中包含太多的主机
+        self.method_hosts_mapper = defaultdict(list)
+        self.playbooks = []
 
-    def get_grouped_assets(self):
+    @classmethod
-        return self.automation.all_assets_group_by_platform()
+    def method_type(cls):
+        raise NotImplementedError
 
     @property
-    def playbook_dir_path(self):
+    def runtime_dir(self):
         ansible_dir = settings.ANSIBLE_DIR
         path = os.path.join(
-            ansible_dir, self.automation.type, self.automation.name.replace(' ', '_'),
+            ansible_dir, self.automation.type,
+            self.automation.name.replace(' ', '_'),
             timezone.now().strftime('%Y%m%d_%H%M%S')
         )
         return path
 
     @property
     def inventory_path(self):
-        return os.path.join(self.playbook_dir_path, 'inventory', 'hosts.json')
+        return os.path.join(self.runtime_dir, 'inventory', 'hosts.json')
 
     @property
     def playbook_path(self):
-        return os.path.join(self.playbook_dir_path, 'project', 'main.yml')
+        return os.path.join(self.runtime_dir, 'project', 'main.yml')
 
     def generate(self):
         self.prepare_playbook_dir()
@@ -41,31 +69,108 @@ class BasePlaybookManager:
     def prepare_playbook_dir(self):
         inventory_dir = os.path.dirname(self.inventory_path)
         playbook_dir = os.path.dirname(self.playbook_path)
-        for d in [inventory_dir, playbook_dir, self.playbook_dir_path]:
+        for d in [inventory_dir, playbook_dir]:
-            print("Create dir: {}".format(d))
             if not os.path.exists(d):
                 os.makedirs(d, exist_ok=True, mode=0o755)
 
-    def inventory_kwargs(self):
+    def host_callback(self, host, automation=None, **kwargs):
-        raise NotImplementedError
+        enabled_attr = '{}_enabled'.format(self.__class__.method_type())
+        method_attr = '{}_method'.format(self.__class__.method_type())
+
+        method_enabled = automation and \
+            getattr(automation, enabled_attr) and \
+            getattr(automation, method_attr) and \
+            getattr(automation, method_attr) in self.method_id_meta_mapper
+
+        if not method_enabled:
+            host['error'] = _('Change password disabled')
+            return host
+
+        self.method_hosts_mapper[getattr(automation, method_attr)].append(host['name'])
+        return host
 
     def generate_inventory(self):
         inventory = JMSInventory(
             assets=self.automation.get_all_assets(),
             account_policy=self.ansible_account_policy,
-            **self.inventory_kwargs()
+            host_callback=self.host_callback
         )
         inventory.write_to_file(self.inventory_path)
-        print("Generate inventory done: {}".format(self.inventory_path))
+        logger.debug("Generate inventory done: {}".format(self.inventory_path))
 
     def generate_playbook(self):
+        main_playbook = []
+        for method_id, host_names in self.method_hosts_mapper.items():
+            method = self.method_id_meta_mapper.get(method_id)
+            if not method:
+                logger.error("Method not found: {}".format(method_id))
+                continue
+            method_playbook_dir_path = method['dir']
+            method_playbook_dir_name = os.path.basename(method_playbook_dir_path)
+            sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name)
+            sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml')
+            shutil.copytree(method_playbook_dir_path, sub_playbook_dir)
+
+            with open(sub_playbook_path, 'r') as f:
+                host_playbook_play = yaml.safe_load(f)
+
+            if isinstance(host_playbook_play, list):
+                host_playbook_play = host_playbook_play[0]
+
+            hosts_bulked = [host_names[i:i+self.bulk_size] for i in range(0, len(host_names), self.bulk_size)]
+            for i, hosts in enumerate(hosts_bulked):
+                plays = []
+                play = deepcopy(host_playbook_play)
+                play['hosts'] = ':'.join(hosts)
+                plays.append(play)
+
+                playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i))
+                with open(playbook_path, 'w') as f:
+                    yaml.safe_dump(plays, f)
+                self.playbooks.append(playbook_path)
+
+                main_playbook.append({
+                    'name': method['name'] + ' for part {}'.format(i),
+                    'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i))
+                })
+
+        with open(self.playbook_path, 'w') as f:
+            yaml.safe_dump(main_playbook, f)
+
+        logger.debug("Generate playbook done: " + self.playbook_path)
+
+    def get_runners(self):
+        runners = []
+        for playbook_path in self.playbooks:
+            runer = PlaybookRunner(
+                self.inventory_path,
+                playbook_path,
+                self.runtime_dir,
+                callback=PlaybookCallback(),
+            )
+            runners.append(runer)
+        return runners
+
+    def on_runner_done(self, runner, cb):
         raise NotImplementedError
 
-    def get_runner(self):
+    def on_runner_failed(self, runner, e):
-        raise NotImplementedError
+        print("Runner failed: {} {}".format(e, self))
 
     def run(self, **kwargs):
         self.generate()
-        runner = self.get_runner()
+        runners = self.get_runners()
-        return runner.run(**kwargs)
+        if len(runners) > 1:
+            print("### 分批次执行开始任务, 总共 {}\n".format(len(runners)))
+        else:
+            print(">>> 开始执行任务\n")
 
+        for i, runner in enumerate(runners, start=1):
+            if len(runners) > 1:
+                print(">>> 开始执行第 {} 批任务".format(i))
+            try:
+                cb = runner.run(**kwargs)
+                self.on_runner_done(runner, cb)
+            except Exception as e:
+                self.on_runner_failed(runner, e)
+            print('\n\n')

apps/assets/automations/change_password/database/postgresql/main.yml

@@ -1,24 +1,26 @@
-- hosts: mysql
+- hosts: postgre
   gather_facts: no
   vars:
     ansible_python_interpreter: /usr/local/bin/python
     jms_account:
       username: postgre
-      password: postgre
+      secret: postgre
     jms_asset:
       address: 127.0.0.1
       port: 5432
+      database: testdb
     account:
-      username: web1
+      username: test
       secret: jumpserver
 
   tasks:
     - name: Test PostgreSQL connection
-      community.postgresql.postgresql_info:
+      community.postgresql.postgresql_ping:
         login_user: "{{ jms_account.username }}"
         login_password: "{{ jms_account.secret }}"
         login_host: "{{ jms_asset.address }}"
         login_port: "{{ jms_asset.port }}"
+        login_db: "{{ jms_asset.database }}"
       register: db_info
 
     - name: Display PostgreSQL version
@@ -31,15 +33,15 @@
         login_password: "{{ jms_account.secret }}"
         login_host: "{{ jms_asset.address }}"
         login_port: "{{ jms_asset.port }}"
+        db: "{{ jms_asset.database }}"
         name: "{{ account.username }}"
         password: "{{ account.secret }}"
-        comment: Updated by jumpserver
-        state: present
       when: db_info is succeeded
 
     - name: Verify password
-      community.postgresql.postgresql_info:
+      community.postgresql.postgresql_ping:
         login_user: "{{ account.username }}"
         login_password: "{{ account.secret }}"
         login_host: "{{ jms_asset.address }}"
         login_port: "{{ jms_asset.port }}"
+        db: "{{ jms_asset.database }}"

apps/assets/automations/change_password/manager.py

@@ -1,44 +1,35 @@
-import os
-import shutil
 from copy import deepcopy
 from collections import defaultdict
 
-import yaml
-from django.utils.translation import gettext as _
-
-from ops.ansible import PlaybookRunner
 from ..base.manager import BasePlaybookManager
-from assets.automations.methods import platform_automation_methods
 
 
 class ChangePasswordManager(BasePlaybookManager):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.id_method_mapper = {
-            method['id']: method
-            for method in platform_automation_methods
-        }
         self.method_hosts_mapper = defaultdict(list)
         self.playbooks = []
 
-    def host_duplicator(self, host, asset=None, account=None, platform=None, **kwargs):
+    @classmethod
+    def method_type(cls):
+        return 'change_password'
+
+    def host_callback(self, host, asset=None, account=None, automation=None, **kwargs):
+        host = super().host_callback(host, asset=asset, account=account, automation=automation, **kwargs)
+        if host.get('exclude'):
+            return host
+
         accounts = asset.accounts.all()
         if account:
             accounts = accounts.exclude(id=account.id)
+
         if '*' not in self.automation.accounts:
             accounts = accounts.filter(username__in=self.automation.accounts)
 
-        automation = platform.automation
+        method_attr = getattr(automation, self.method_type() + '_method')
-        change_password_enabled = automation and \
+        method_hosts = self.method_hosts_mapper[method_attr]
-            automation.change_password_enabled and \
+        method_hosts = [h for h in method_hosts if h != host['name']]
-            automation.change_password_method and \
+        inventory_hosts = []
-            automation.change_password_method in self.id_method_mapper
-
-        if not change_password_enabled:
-            host['exclude'] = _('Change password disabled')
-            return [host]
-
-        hosts = []
         for account in accounts:
             h = deepcopy(host)
             h['name'] += '_' + account.username
@@ -48,59 +39,15 @@ class ChangePasswordManager(BasePlaybookManager):
                 'secret_type': account.secret_type,
                 'secret': account.secret,
             }
-            hosts.append(h)
+            inventory_hosts.append(h)
-            self.method_hosts_mapper[automation.change_password_method].append(h['name'])
+            method_hosts.append(h['name'])
-        return hosts
+        self.method_hosts_mapper[method_attr] = method_hosts
+        return inventory_hosts
 
-    def inventory_kwargs(self):
+    def on_runner_done(self, runner, cb):
-        return {
+        pass
-            'host_duplicator': self.host_duplicator
-        }
 
-    def generate_playbook(self):
+    def on_runner_failed(self, runner, e):
-        playbook = []
+        pass
-        for method_id, host_names in self.method_hosts_mapper.items():
-            method = self.id_method_mapper[method_id]
-            method_playbook_dir_path = method['dir']
-            method_playbook_dir_name = os.path.basename(method_playbook_dir_path)
-            sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name)
-            shutil.copytree(method_playbook_dir_path, sub_playbook_dir)
-            sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml')
-
-            with open(sub_playbook_path, 'r') as f:
-                host_playbook_play = yaml.safe_load(f)
-
-            if isinstance(host_playbook_play, list):
-                host_playbook_play = host_playbook_play[0]
-
-            step = 10
-            hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)]
-            for i, hosts in enumerate(hosts_grouped):
-                plays = []
-                play = deepcopy(host_playbook_play)
-                play['hosts'] = ':'.join(hosts)
-                plays.append(play)
-
-                playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i))
-                with open(playbook_path, 'w') as f:
-                    yaml.safe_dump(plays, f)
-                self.playbooks.append(playbook_path)
-
-                playbook.append({
-                    'name': method['name'] + ' for part {}'.format(i),
-                    'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i))
-                })
-
-        with open(self.playbook_path, 'w') as f:
-            yaml.safe_dump(playbook, f)
-
-        print("Generate playbook done: " + self.playbook_path)
-
-    def get_runner(self):
-        return PlaybookRunner(
-            self.inventory_path,
-            self.playbook_path,
-            self.playbook_dir_path
-        )
 
 

apps/assets/automations/gather_facts/database/mysql/main.yml

@@ -0,0 +1,28 @@
+- hosts: mysql
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+    jms_account:
+      username: root
+      secret: redhat
+    jms_asset:
+      address: 127.0.0.1
+      port: 3306
+
+  tasks:
+    - name: Gather facts info
+      community.mysql.mysql_info:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+      register: db_info
+
+    - name: Get info
+      set_fact:
+        info:
+          version: "{{ db_info.version.full }}"
+
+    - debug:
+        var: db_info
+

apps/assets/automations/gather_facts/database/mysql/manifest.yml

@@ -0,0 +1,6 @@
+id: gather_facts_mysql
+name: Gather facts from MySQL
+category: database
+type:
+  - mysql
+method: gather_facts

apps/assets/automations/gather_facts/database/postgresql/main.yml

@@ -0,0 +1,28 @@
+- hosts: postgre
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+    jms_account:
+      username: postgre
+      secret: postgre
+    jms_asset:
+      address: 127.0.0.1
+      port: 5432
+      database: testdb
+    account:
+      username: test
+      secret: jumpserver
+
+  tasks:
+    - name: Test PostgreSQL connection
+      community.postgresql.postgresql_info:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_db: "{{ jms_asset.database }}"
+      register: db_info
+
+    - name: Debug it
+      debug:
+        var: db_info

apps/assets/automations/gather_facts/database/postgresql/manifest.yml

@@ -0,0 +1,6 @@
+id: gather_facts_postgresql
+name: Gather facts for PostgreSQL
+category: database
+type:
+  - postgresql
+method: gather_facts

apps/assets/automations/gather_facts/database/sqlserver/main.yml

@@ -0,0 +1,10 @@
+{% for account in accounts %}
+- hosts: {{ account.asset.name }}
+  vars:
+    account:
+      username: {{ account.username }}
+      password: {{ account.password }}
+      public_key: {{ account.public_key }}
+  roles:
+    - change_password
+{% endfor %}

apps/assets/automations/gather_facts/database/sqlserver/manifest.yml

@@ -0,0 +1,8 @@
+id: gather_facts_sqlserver
+name: Change password for SQLServer
+version: 1
+category: database
+type:
+  - sqlserver
+method: gather_facts
+

apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: ping
+  ping:
+
+#- name: print variables
+#  debug:
+#    msg: "Username: {{ account.username }}, Password: {{ account.password }}"
+
+- name: Change password
+  user:
+    name: "{{ account.username }}"
+    password: "{{ account.password | password_hash('des') }}"
+    update_password: always
+  when: account.password
+
+- name: Change public key
+  authorized_key:
+    user: "{{ account.username }}"
+    key: "{{ account.public_key }}"
+    state: present
+  when: account.public_key
+
+- name: Verify password
+  ping:
+  vars:
+    ansible_user: "{{ account.username }}"
+    ansible_pass: "{{ account.password }}"
+    ansible_ssh_connection: paramiko

apps/assets/automations/gather_facts/demo_inventory.txt

@@ -0,0 +1,2 @@
+# all base inventory in base/base_inventory.txt
+asset_name(ip) ...base_inventory_vars

apps/assets/automations/gather_facts/host/posix/main.yml

@@ -0,0 +1,19 @@
+- hosts: website
+  gather_facts: yes
+  tasks:
+    - name: Get info
+      set_fact:
+        info:
+          arch: "{{ ansible_architecture }}"
+          distribution: "{{ ansible_distribution }}"
+          distribution_version: "{{ ansible_distribution_version }}"
+          kernel: "{{ ansible_kernel }}"
+          vendor: "{{ ansible_system_vendor }}"
+          model: "{{ ansible_product_name }}"
+          sn: "{{ ansible_product_serial }}"
+          cpu_vcpus: "{{ ansible_processor_vcpus }}"
+          memory: "{{ ansible_memtotal_mb }}"
+          disk_total: "{{ (ansible_mounts | map(attribute='size_total') | sum / 1024 / 1024 / 1024) | round(2) }}"
+
+    - debug:
+        var: info

apps/assets/automations/gather_facts/host/posix/manifest.yml

@@ -0,0 +1,8 @@
+id: gather_facts_posix
+name: Gather posix facts
+category: host
+type:
+  - linux
+  - windows
+  - unix
+method: gather_facts

apps/assets/automations/gather_facts/host/windows/main.yml

@@ -0,0 +1,24 @@
+- hosts: windows
+  gather_facts: yes
+  tasks:
+#    - name: Gather facts windows
+#      setup:
+#      register: facts
+#
+#    - debug:
+#        var: facts
+    - name: Get info
+      set_fact:
+        info:
+          arch: "{{ ansible_architecture2 }}"
+          distribution: "{{ ansible_distribution }}"
+          distribution_version: "{{ ansible_distribution_version }}"
+          kernel: "{{ ansible_kernel }}"
+          vendor: "{{ ansible_system_vendor }}"
+          model: "{{ ansible_product_name }}"
+          sn: "{{ ansible_product_serial }}"
+          cpu_vcpus: "{{ ansible_processor_vcpus }}"
+          memory: "{{ ansible_memtotal_mb }}"
+t
+    - debug:
+        var: info

apps/assets/automations/gather_facts/host/windows/manifest.yml

@@ -0,0 +1,7 @@
+id: gather_facts_windows
+name: Gather facts windows
+version: 1
+method: gather_facts
+category: host
+type:
+  - windows

apps/assets/automations/gather_facts/manager.py

@@ -0,0 +1,77 @@
+import os
+import shutil
+from copy import deepcopy
+from collections import defaultdict
+
+import yaml
+from django.utils.translation import gettext as _
+
+from ops.ansible import PlaybookRunner
+from ..base.manager import BasePlaybookManager
+from assets.automations.methods import platform_automation_methods
+
+
+class GatherFactsManager(BasePlaybookManager):
+    method_name = 'gather_facts'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.id_method_mapper = {
+            method['id']: method
+            for method in platform_automation_methods
+            if method['method'] == self.method_name
+        }
+        self.method_hosts_mapper = defaultdict(list)
+        self.playbooks = []
+
+    def inventory_kwargs(self):
+        return {
+        }
+
+    def generate_playbook(self):
+        playbook = []
+        for method_id, host_names in self.method_hosts_mapper.items():
+            method = self.id_method_mapper[method_id]
+            method_playbook_dir_path = method['dir']
+            method_playbook_dir_name = os.path.basename(method_playbook_dir_path)
+            sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name)
+            shutil.copytree(method_playbook_dir_path, sub_playbook_dir)
+            sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml')
+
+            with open(sub_playbook_path, 'r') as f:
+                host_playbook_play = yaml.safe_load(f)
+
+            if isinstance(host_playbook_play, list):
+                host_playbook_play = host_playbook_play[0]
+
+            step = 10
+            hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)]
+            for i, hosts in enumerate(hosts_grouped):
+                plays = []
+                play = deepcopy(host_playbook_play)
+                play['hosts'] = ':'.join(hosts)
+                plays.append(play)
+
+                playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i))
+                with open(playbook_path, 'w') as f:
+                    yaml.safe_dump(plays, f)
+                self.playbooks.append(playbook_path)
+
+                playbook.append({
+                    'name': method['name'] + ' for part {}'.format(i),
+                    'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i))
+                })
+
+        with open(self.playbook_path, 'w') as f:
+            yaml.safe_dump(playbook, f)
+
+        print("Generate playbook done: " + self.playbook_path)
+
+    def get_runner(self):
+        return PlaybookRunner(
+            self.inventory_path,
+            self.playbook_path,
+            self.runtime_dir
+        )
+
+

apps/assets/automations/generate_playbook/change_password.py

@@ -1,106 +0,0 @@
-import os
-import yaml
-import jinja2
-from typing import List
-
-from django.conf import settings
-from assets.models import Asset
-from .base import BaseGeneratePlaybook
-
-
-class GenerateChangePasswordPlaybook(BaseGeneratePlaybook):
-
-    def __init__(
-            self, assets: List[Asset], strategy, usernames, password='',
-            private_key='', public_key='', key_strategy=''
-    ):
-        super().__init__(assets, strategy)
-        self.password = password
-        self.public_key = public_key
-        self.private_key = private_key
-        self.key_strategy = key_strategy
-        self.relation_asset_map = self.get_username_relation_asset_map(usernames)
-
-    def get_username_relation_asset_map(self, usernames):
-        # TODO 没特权用户的资产 要考虑网关
-
-        complete_map = {
-            asset: list(asset.accounts.value_list('username', flat=True))
-            for asset in self.assets
-        }
-
-        if '*' in usernames:
-            return complete_map
-
-        relation_map = {}
-        for asset, usernames in complete_map.items():
-            usernames = list(set(usernames) & set(usernames))
-            if not usernames:
-                continue
-            relation_map[asset] = list(set(usernames) & set(usernames))
-        return relation_map
-
-    @property
-    def src_filepath(self):
-        return os.path.join(
-            settings.BASE_DIR, 'assets', 'playbooks', 'strategy',
-            'change_password', 'roles', self.strategy
-        )
-
-    def generate_hosts(self):
-        host_pathname = os.path.join(self.temp_folder, 'hosts')
-        with open(host_pathname, 'w', encoding='utf8') as f:
-            for asset in self.relation_asset_map.keys():
-                f.write(f'{asset.name}\n')
-
-    def generate_host_vars(self):
-        host_vars_pathname = os.path.join(self.temp_folder, 'hosts', 'host_vars')
-        os.makedirs(host_vars_pathname, exist_ok=True)
-        for asset, usernames in self.relation_asset_map.items():
-            host_vars = {
-                'ansible_host': asset.get_target_ip(),
-                'ansible_port': asset.get_target_ssh_port(),  # TODO 需要根绝协议取端口号
-                'ansible_user': asset.admin_user.username,
-                'ansible_pass': asset.admin_user.username,
-                'usernames': usernames,
-            }
-            pathname = os.path.join(host_vars_pathname, f'{asset.name}.yml')
-            with open(pathname, 'w', encoding='utf8') as f:
-                f.write(yaml.dump(host_vars, allow_unicode=True))
-
-    def generate_secret_key_files(self):
-        if not self.private_key and not self.public_key:
-            return
-
-        file_pathname = os.path.join(self.temp_folder, self.strategy, 'files')
-        public_pathname = os.path.join(file_pathname, 'id_rsa.pub')
-        private_pathname = os.path.join(file_pathname, 'id_rsa')
-
-        os.makedirs(file_pathname, exist_ok=True)
-        with open(public_pathname, 'w', encoding='utf8') as f:
-            f.write(self.public_key)
-        with open(private_pathname, 'w', encoding='utf8') as f:
-            f.write(self.private_key)
-
-    def generate_role_main(self):
-        task_main_pathname = os.path.join(self.temp_folder, 'main.yaml')
-        context = {
-            'password': self.password,
-            'key_strategy': self.key_strategy,
-            'private_key_file': 'id_rsa' if self.private_key else '',
-            'exclusive': 'no' if self.key_strategy == 'all' else 'yes',
-            'jms_key': self.public_key.split()[2].strip() if self.public_key else '',
-        }
-        with open(task_main_pathname, 'r+', encoding='utf8') as f:
-            string_var = f.read()
-            f.seek(0, 0)
-            response = jinja2.Template(string_var).render(context)
-            results = yaml.safe_load(response)
-            f.write(yaml.dump(results, allow_unicode=True))
-
-    def execute(self):
-        self.generate_temp_playbook()
-        self.generate_hosts()
-        self.generate_host_vars()
-        self.generate_secret_key_files()
-        self.generate_role_main()

apps/assets/automations/generate_playbook/verify.py

@@ -1,86 +0,0 @@
-import os
-import yaml
-from typing import List
-
-from django.conf import settings
-from assets.models import Asset
-from .base import BaseGeneratePlaybook
-
-
-class GenerateVerifyPlaybook(BaseGeneratePlaybook):
-
-    def __init__(
-            self, assets: List[Asset], strategy, usernames
-    ):
-        super().__init__(assets, strategy)
-        self.relation_asset_map = self.get_account_relation_asset_map(usernames)
-
-    def get_account_relation_asset_map(self, usernames):
-        # TODO 没特权用户的资产 要考虑网关
-        complete_map = {
-            asset: list(asset.accounts.all())
-            for asset in self.assets
-        }
-
-        if '*' in usernames:
-            return complete_map
-
-        relation_map = {}
-        for asset, accounts in complete_map.items():
-            account_map = {account.username: account for account in accounts}
-            accounts = [account_map[i] for i in (set(usernames) & set(account_map))]
-            if not accounts:
-                continue
-            relation_map[asset] = accounts
-        return relation_map
-
-    @property
-    def src_filepath(self):
-        return os.path.join(
-            settings.BASE_DIR, 'assets', 'playbooks', 'strategy',
-            'verify', 'roles', self.strategy
-        )
-
-    def generate_hosts(self):
-        host_pathname = os.path.join(self.temp_folder, 'hosts')
-        with open(host_pathname, 'w', encoding='utf8') as f:
-            for asset in self.relation_asset_map.keys():
-                f.write(f'{asset.name}\n')
-
-    def generate_host_vars(self):
-        host_vars_pathname = os.path.join(self.temp_folder, 'hosts', 'host_vars')
-        os.makedirs(host_vars_pathname, exist_ok=True)
-        for asset, accounts in self.relation_asset_map.items():
-            account_info = []
-            for account in accounts:
-                private_key_filename = f'{asset.name}_{account.username}' if account.private_key else ''
-                account_info.append({
-                    'username': account.username,
-                    'password': account.password,
-                    'private_key_filename': private_key_filename,
-                })
-            host_vars = {
-                'ansible_host': asset.get_target_ip(),
-                'ansible_port': asset.get_target_ssh_port(),  # TODO 需要根绝协议取端口号
-                'account_info': account_info,
-            }
-            pathname = os.path.join(host_vars_pathname, f'{asset.name}.yml')
-            with open(pathname, 'w', encoding='utf8') as f:
-                f.write(yaml.dump(host_vars, allow_unicode=True))
-
-    def generate_secret_key_files(self):
-        file_pathname = os.path.join(self.temp_folder, self.strategy, 'files')
-        os.makedirs(file_pathname, exist_ok=True)
-        for asset, accounts in self.relation_asset_map.items():
-            for account in accounts:
-                if account.private_key:
-                    path_name = os.path.join(file_pathname, f'{asset.name}_{account.username}')
-                    with open(path_name, 'w', encoding='utf8') as f:
-                        f.write(account.private_key)
-
-    def execute(self):
-        self.generate_temp_playbook()
-        self.generate_hosts()
-        self.generate_host_vars()
-        self.generate_secret_key_files()
-        # self.generate_role_main() # TODO Linux 暂时不需要

apps/assets/automations/ping/database/mysql/main.yml

@@ -0,0 +1,20 @@
+- hosts: mysql
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+    jms_account:
+      username: root
+      password: redhat
+    jms_asset:
+      address: 127.0.0.1
+      port: 3306
+
+  tasks:
+    - name: Test MySQL connection
+      community.mysql.mysql_info:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        filter: version
+      register: db_info

apps/assets/automations/ping/database/mysql/manifest.yml

@@ -0,0 +1,6 @@
+id: mysql_ping
+name: Ping MySQL
+category: database
+type:
+  - mysql
+method: ping

apps/assets/automations/ping/database/postgresql/main.yml

@@ -0,0 +1,23 @@
+- hosts: postgre
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+    jms_account:
+      username: postgre
+      secret: postgre
+    jms_asset:
+      address: 127.0.0.1
+      port: 5432
+      database: testdb
+    account:
+      username: test
+      secret: jumpserver
+
+  tasks:
+    - name: Test PostgreSQL connection
+      community.postgresql.postgresql_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_db: "{{ jms_asset.database }}"

apps/assets/automations/ping/database/postgresql/manifest.yml

@@ -0,0 +1,6 @@
+id: ping_postgresql
+name: Ping PostgreSQL
+category: database
+type:
+  - postgresql
+method: ping

apps/assets/automations/ping/demo_inventory.txt

@@ -0,0 +1,2 @@
+# all base inventory in base/base_inventory.txt
+asset_name(ip)_account_username account={"username": "", "password": "xxx"} ...base_inventory_vars

apps/assets/automations/ping/host/posix/main.yml

@@ -0,0 +1,5 @@
+- hosts: demo
+  gather_facts: no
+  tasks:
+    - name: Posix ping
+      ping:

apps/assets/automations/ping/host/posix/manifest.yml

@@ -0,0 +1,8 @@
+id: posix_ping
+name: Posix ping
+category: host
+type:
+  - linux
+  - windows
+  - unix
+method: ping

apps/assets/automations/ping/host/windows/main.yml

@@ -0,0 +1,5 @@
+- hosts: windows
+  gather_facts: no
+  tasks:
+    - name: Windows ping
+      win_ping:

apps/assets/automations/ping/host/windows/manifest.yml

@@ -0,0 +1,7 @@
+id: win_ping
+name: Windows ping
+version: 1
+method: change_password
+category: host
+type:
+  - windows

apps/assets/automations/ping/manager.py

@@ -0,0 +1,75 @@
+import os
+import shutil
+from copy import deepcopy
+from collections import defaultdict
+
+import yaml
+from django.utils.translation import gettext as _
+
+from ops.ansible import PlaybookRunner
+from ..base.manager import BasePlaybookManager
+from assets.automations.methods import platform_automation_methods
+
+
+class ChangePasswordManager(BasePlaybookManager):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.id_method_mapper = {
+            method['id']: method
+            for method in platform_automation_methods
+        }
+        self.method_hosts_mapper = defaultdict(list)
+        self.playbooks = []
+
+    def inventory_kwargs(self):
+        return {
+            'host_callback': self.host_duplicator
+        }
+
+    def generate_playbook(self):
+        playbook = []
+        for method_id, host_names in self.method_hosts_mapper.items():
+            method = self.id_method_mapper[method_id]
+            method_playbook_dir_path = method['dir']
+            method_playbook_dir_name = os.path.basename(method_playbook_dir_path)
+            sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name)
+            shutil.copytree(method_playbook_dir_path, sub_playbook_dir)
+            sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml')
+
+            with open(sub_playbook_path, 'r') as f:
+                host_playbook_play = yaml.safe_load(f)
+
+            if isinstance(host_playbook_play, list):
+                host_playbook_play = host_playbook_play[0]
+
+            step = 10
+            hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)]
+            for i, hosts in enumerate(hosts_grouped):
+                plays = []
+                play = deepcopy(host_playbook_play)
+                play['hosts'] = ':'.join(hosts)
+                plays.append(play)
+
+                playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i))
+                with open(playbook_path, 'w') as f:
+                    yaml.safe_dump(plays, f)
+                self.playbooks.append(playbook_path)
+
+                playbook.append({
+                    'name': method['name'] + ' for part {}'.format(i),
+                    'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i))
+                })
+
+        with open(self.playbook_path, 'w') as f:
+            yaml.safe_dump(playbook, f)
+
+        print("Generate playbook done: " + self.playbook_path)
+
+    def get_runner(self):
+        return PlaybookRunner(
+            self.inventory_path,
+            self.playbook_path,
+            self.runtime_dir
+        )
+
+

apps/assets/models/automations/change_secret.py

@@ -19,6 +19,10 @@ class ChangePasswordAutomation(BaseAutomation):
         verbose_name=_("Recipient")
     )
 
+    def save(self, *args, **kwargs):
+        self.type = 'change_password'
+        super().save(*args, **kwargs)
+
     class Meta:
         verbose_name = _("Change auth strategy")
 

apps/ops/ansible/inventory.py

@@ -10,18 +10,17 @@ __all__ = ['JMSInventory']
 
 
 class JMSInventory:
-    def __init__(self, assets, account='', account_policy='smart', host_var_callback=None, host_duplicator=None):
+    def __init__(self, assets, account='', account_policy='smart', host_callback=None):
         """
         :param assets:
         :param account: account username name if not set use account_policy
         :param account_policy:
-        :param host_var_callback:
+        :param host_callback: after generate host, call this callback to modify host
         """
         self.assets = self.clean_assets(assets)
         self.account_username = account
         self.account_policy = account_policy
-        self.host_var_callback = host_var_callback
+        self.host_callback = host_callback
-        self.host_duplicator = host_duplicator
 
     @staticmethod
     def clean_assets(assets):
@@ -100,15 +99,10 @@ class JMSInventory:
                 elif account.secret_type == 'private_key' and account.secret:
                     host['ssh_private_key'] = account.private_key_file
             else:
-                host['exclude'] = _("No account found")
+                host['error'] = _("No account found")
 
             if gateway:
                 host.update(self.make_proxy_command(gateway))
-
-        if self.host_var_callback:
-            callback_var = self.host_var_callback(asset)
-            if isinstance(callback_var, dict):
-                host.update(callback_var)
         return host
 
     def select_account(self, asset):
@@ -145,10 +139,18 @@ class JMSInventory:
             for asset in self.assets:
                 account = self.select_account(asset)
                 host = self.asset_to_host(asset, account, automation, protocols)
+
                 if not automation.ansible_enabled:
-                    host['exclude'] = _('Ansible disabled')
+                    host['error'] = _('Ansible disabled')
-                if self.host_duplicator:
+
-                    hosts.extend(self.host_duplicator(host, asset=asset, account=account, platform=platform))
+                if self.host_callback is not None:
+                    host = self.host_callback(
+                        host, asset=asset, account=account,
+                        platform=platform, automation=automation
+                    )
+
+                if isinstance(host, list):
+                    hosts.extend(host)
                 else:
                     hosts.append(host)
 
@@ -156,7 +158,7 @@ class JMSInventory:
         if exclude_hosts:
             print(_("Skip hosts below:"))
             for i, host in enumerate(exclude_hosts, start=1):
-                print("{}: [{}] \t{}".format(i, host['name'], host['exclude']))
+                print("{}: [{}] \t{}".format(i, host['name'], host['error']))
 
         hosts = list(filter(lambda x: not x.get('exclude'), hosts))
         data = {'all': {'hosts': {}}}

apps/ops/ansible/runner.py

@@ -52,12 +52,14 @@ class AdHocRunner:
 
 
 class PlaybookRunner:
-    def __init__(self, inventory, playbook, project_dir='/tmp/'):
+    def __init__(self, inventory, playbook, project_dir='/tmp/', callback=None):
         self.id = uuid.uuid4()
         self.inventory = inventory
         self.playbook = playbook
         self.project_dir = project_dir
-        self.cb = DefaultCallback()
+        if not callback:
+            callback = DefaultCallback()
+        self.cb = callback
 
     def run(self, verbosity=0, **kwargs):
         if verbosity is None and settings.DEBUG: