diff --git a/apps/assets/automations/__init__.py b/apps/assets/automations/__init__.py index 478e9740d..7c63c916a 100644 --- a/apps/assets/automations/__init__.py +++ b/apps/assets/automations/__init__.py @@ -1 +1 @@ -from .methods import platform_automation_methods +from .methods import platform_automation_methods, filter_platform_methods diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index e55a564bf..626ced70a 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -1,37 +1,65 @@ import os +import shutil +import yaml +from copy import deepcopy +from collections import defaultdict from django.conf import settings from django.utils import timezone +from django.utils.translation import gettext as _ -from ops.ansible import JMSInventory +from common.utils import get_logger +from assets.automations.methods import platform_automation_methods +from ops.ansible import JMSInventory, PlaybookRunner, DefaultCallback + +logger = get_logger(__name__) + + +class PlaybookCallback(DefaultCallback): + def playbook_on_stats(self, event_data, **kwargs): + print("\n*** 分任务结果") + super().playbook_on_stats(event_data, **kwargs) class BasePlaybookManager: + bulk_size = 100 ansible_account_policy = 'privileged_first' def __init__(self, execution): self.execution = execution self.automation = execution.automation + self.method_id_meta_mapper = { + method['id']: method + for method in platform_automation_methods + if method['method'] == self.__class__.method_type() + } + # 根据执行方式就行分组, 不同资产的改密、推送等操作可能会使用不同的执行方式 + # 然后根据执行方式分组, 再根据 bulk_size 分组, 生成不同的 playbook + # 避免一个 playbook 中包含太多的主机 + self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] - def get_grouped_assets(self): - return self.automation.all_assets_group_by_platform() + @classmethod + def method_type(cls): + raise NotImplementedError @property - def playbook_dir_path(self): + def runtime_dir(self): ansible_dir = settings.ANSIBLE_DIR path = os.path.join( - ansible_dir, self.automation.type, self.automation.name.replace(' ', '_'), + ansible_dir, self.automation.type, + self.automation.name.replace(' ', '_'), timezone.now().strftime('%Y%m%d_%H%M%S') ) return path @property def inventory_path(self): - return os.path.join(self.playbook_dir_path, 'inventory', 'hosts.json') + return os.path.join(self.runtime_dir, 'inventory', 'hosts.json') @property def playbook_path(self): - return os.path.join(self.playbook_dir_path, 'project', 'main.yml') + return os.path.join(self.runtime_dir, 'project', 'main.yml') def generate(self): self.prepare_playbook_dir() @@ -41,31 +69,108 @@ class BasePlaybookManager: def prepare_playbook_dir(self): inventory_dir = os.path.dirname(self.inventory_path) playbook_dir = os.path.dirname(self.playbook_path) - for d in [inventory_dir, playbook_dir, self.playbook_dir_path]: - print("Create dir: {}".format(d)) + for d in [inventory_dir, playbook_dir]: if not os.path.exists(d): os.makedirs(d, exist_ok=True, mode=0o755) - def inventory_kwargs(self): - raise NotImplementedError + def host_callback(self, host, automation=None, **kwargs): + enabled_attr = '{}_enabled'.format(self.__class__.method_type()) + method_attr = '{}_method'.format(self.__class__.method_type()) + + method_enabled = automation and \ + getattr(automation, enabled_attr) and \ + getattr(automation, method_attr) and \ + getattr(automation, method_attr) in self.method_id_meta_mapper + + if not method_enabled: + host['error'] = _('Change password disabled') + return host + + self.method_hosts_mapper[getattr(automation, method_attr)].append(host['name']) + return host def generate_inventory(self): inventory = JMSInventory( assets=self.automation.get_all_assets(), account_policy=self.ansible_account_policy, - **self.inventory_kwargs() + host_callback=self.host_callback ) inventory.write_to_file(self.inventory_path) - print("Generate inventory done: {}".format(self.inventory_path)) + logger.debug("Generate inventory done: {}".format(self.inventory_path)) def generate_playbook(self): + main_playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.method_id_meta_mapper.get(method_id) + if not method: + logger.error("Method not found: {}".format(method_id)) + continue + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + + hosts_bulked = [host_names[i:i+self.bulk_size] for i in range(0, len(host_names), self.bulk_size)] + for i, hosts in enumerate(hosts_bulked): + plays = [] + play = deepcopy(host_playbook_play) + play['hosts'] = ':'.join(hosts) + plays.append(play) + + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) + + main_playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(main_playbook, f) + + logger.debug("Generate playbook done: " + self.playbook_path) + + def get_runners(self): + runners = [] + for playbook_path in self.playbooks: + runer = PlaybookRunner( + self.inventory_path, + playbook_path, + self.runtime_dir, + callback=PlaybookCallback(), + ) + runners.append(runer) + return runners + + def on_runner_done(self, runner, cb): raise NotImplementedError - def get_runner(self): - raise NotImplementedError + def on_runner_failed(self, runner, e): + print("Runner failed: {} {}".format(e, self)) def run(self, **kwargs): self.generate() - runner = self.get_runner() - return runner.run(**kwargs) + runners = self.get_runners() + if len(runners) > 1: + print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) + else: + print(">>> 开始执行任务\n") + for i, runner in enumerate(runners, start=1): + if len(runners) > 1: + print(">>> 开始执行第 {} 批任务".format(i)) + try: + cb = runner.run(**kwargs) + self.on_runner_done(runner, cb) + except Exception as e: + self.on_runner_failed(runner, e) + print('\n\n') diff --git a/apps/assets/automations/change_password/database/change_password_mysql/main.yml b/apps/assets/automations/change_password/database/mysql/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_mysql/main.yml rename to apps/assets/automations/change_password/database/mysql/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_mysql/manifest.yml b/apps/assets/automations/change_password/database/mysql/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_mysql/manifest.yml rename to apps/assets/automations/change_password/database/mysql/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_oracle/main.yml b/apps/assets/automations/change_password/database/oracle/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_oracle/main.yml rename to apps/assets/automations/change_password/database/oracle/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_oracle/manifest.yml b/apps/assets/automations/change_password/database/oracle/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_oracle/manifest.yml rename to apps/assets/automations/change_password/database/oracle/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_postgresql/main.yml b/apps/assets/automations/change_password/database/postgresql/main.yml similarity index 79% rename from apps/assets/automations/change_password/database/change_password_postgresql/main.yml rename to apps/assets/automations/change_password/database/postgresql/main.yml index 0180c559c..ed4e60abf 100644 --- a/apps/assets/automations/change_password/database/change_password_postgresql/main.yml +++ b/apps/assets/automations/change_password/database/postgresql/main.yml @@ -1,24 +1,26 @@ -- hosts: mysql +- hosts: postgre gather_facts: no vars: ansible_python_interpreter: /usr/local/bin/python jms_account: username: postgre - password: postgre + secret: postgre jms_asset: address: 127.0.0.1 port: 5432 + database: testdb account: - username: web1 + username: test secret: jumpserver tasks: - name: Test PostgreSQL connection - community.postgresql.postgresql_info: + community.postgresql.postgresql_ping: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" + login_db: "{{ jms_asset.database }}" register: db_info - name: Display PostgreSQL version @@ -31,15 +33,15 @@ login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" + db: "{{ jms_asset.database }}" name: "{{ account.username }}" password: "{{ account.secret }}" - comment: Updated by jumpserver - state: present when: db_info is succeeded - name: Verify password - community.postgresql.postgresql_info: + community.postgresql.postgresql_ping: login_user: "{{ account.username }}" login_password: "{{ account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" + db: "{{ jms_asset.database }}" diff --git a/apps/assets/automations/change_password/database/change_password_postgresql/manifest.yml b/apps/assets/automations/change_password/database/postgresql/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_postgresql/manifest.yml rename to apps/assets/automations/change_password/database/postgresql/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_sqlserver/main.yml b/apps/assets/automations/change_password/database/sqlserver/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_sqlserver/main.yml rename to apps/assets/automations/change_password/database/sqlserver/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_sqlserver/manifest.yml b/apps/assets/automations/change_password/database/sqlserver/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_sqlserver/manifest.yml rename to apps/assets/automations/change_password/database/sqlserver/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml b/apps/assets/automations/change_password/database/sqlserver/roles/change_password/tasks/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml rename to apps/assets/automations/change_password/database/sqlserver/roles/change_password/tasks/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_aix/main.yml b/apps/assets/automations/change_password/host/aix/main.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_aix/main.yml rename to apps/assets/automations/change_password/host/aix/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_aix/manifest.yml b/apps/assets/automations/change_password/host/aix/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_aix/manifest.yml rename to apps/assets/automations/change_password/host/aix/manifest.yml diff --git a/apps/assets/automations/change_password/host/change_password_linux/main.yml b/apps/assets/automations/change_password/host/linux/main.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_linux/main.yml rename to apps/assets/automations/change_password/host/linux/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_linux/manifest.yml b/apps/assets/automations/change_password/host/linux/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_linux/manifest.yml rename to apps/assets/automations/change_password/host/linux/manifest.yml diff --git a/apps/assets/automations/change_password/host/change_password_local_windows/main.yml b/apps/assets/automations/change_password/host/windows/main.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_local_windows/main.yml rename to apps/assets/automations/change_password/host/windows/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_local_windows/manifest.yml b/apps/assets/automations/change_password/host/windows/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_local_windows/manifest.yml rename to apps/assets/automations/change_password/host/windows/manifest.yml diff --git a/apps/assets/automations/change_password/manager.py b/apps/assets/automations/change_password/manager.py index 6c315cb82..1991d6854 100644 --- a/apps/assets/automations/change_password/manager.py +++ b/apps/assets/automations/change_password/manager.py @@ -1,44 +1,35 @@ -import os -import shutil from copy import deepcopy from collections import defaultdict -import yaml -from django.utils.translation import gettext as _ - -from ops.ansible import PlaybookRunner from ..base.manager import BasePlaybookManager -from assets.automations.methods import platform_automation_methods class ChangePasswordManager(BasePlaybookManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.id_method_mapper = { - method['id']: method - for method in platform_automation_methods - } self.method_hosts_mapper = defaultdict(list) self.playbooks = [] - def host_duplicator(self, host, asset=None, account=None, platform=None, **kwargs): + @classmethod + def method_type(cls): + return 'change_password' + + def host_callback(self, host, asset=None, account=None, automation=None, **kwargs): + host = super().host_callback(host, asset=asset, account=account, automation=automation, **kwargs) + if host.get('exclude'): + return host + accounts = asset.accounts.all() if account: accounts = accounts.exclude(id=account.id) + if '*' not in self.automation.accounts: accounts = accounts.filter(username__in=self.automation.accounts) - automation = platform.automation - change_password_enabled = automation and \ - automation.change_password_enabled and \ - automation.change_password_method and \ - automation.change_password_method in self.id_method_mapper - - if not change_password_enabled: - host['exclude'] = _('Change password disabled') - return [host] - - hosts = [] + method_attr = getattr(automation, self.method_type() + '_method') + method_hosts = self.method_hosts_mapper[method_attr] + method_hosts = [h for h in method_hosts if h != host['name']] + inventory_hosts = [] for account in accounts: h = deepcopy(host) h['name'] += '_' + account.username @@ -48,59 +39,15 @@ class ChangePasswordManager(BasePlaybookManager): 'secret_type': account.secret_type, 'secret': account.secret, } - hosts.append(h) - self.method_hosts_mapper[automation.change_password_method].append(h['name']) - return hosts + inventory_hosts.append(h) + method_hosts.append(h['name']) + self.method_hosts_mapper[method_attr] = method_hosts + return inventory_hosts - def inventory_kwargs(self): - return { - 'host_duplicator': self.host_duplicator - } + def on_runner_done(self, runner, cb): + pass - def generate_playbook(self): - playbook = [] - for method_id, host_names in self.method_hosts_mapper.items(): - method = self.id_method_mapper[method_id] - method_playbook_dir_path = method['dir'] - method_playbook_dir_name = os.path.basename(method_playbook_dir_path) - sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) - shutil.copytree(method_playbook_dir_path, sub_playbook_dir) - sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') - - with open(sub_playbook_path, 'r') as f: - host_playbook_play = yaml.safe_load(f) - - if isinstance(host_playbook_play, list): - host_playbook_play = host_playbook_play[0] - - step = 10 - hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] - for i, hosts in enumerate(hosts_grouped): - plays = [] - play = deepcopy(host_playbook_play) - play['hosts'] = ':'.join(hosts) - plays.append(play) - - playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) - with open(playbook_path, 'w') as f: - yaml.safe_dump(plays, f) - self.playbooks.append(playbook_path) - - playbook.append({ - 'name': method['name'] + ' for part {}'.format(i), - 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) - }) - - with open(self.playbook_path, 'w') as f: - yaml.safe_dump(playbook, f) - - print("Generate playbook done: " + self.playbook_path) - - def get_runner(self): - return PlaybookRunner( - self.inventory_path, - self.playbook_path, - self.playbook_dir_path - ) + def on_runner_failed(self, runner, e): + pass diff --git a/apps/assets/automations/generate_playbook/__init__.py b/apps/assets/automations/gather_facts/__init__.py similarity index 100% rename from apps/assets/automations/generate_playbook/__init__.py rename to apps/assets/automations/gather_facts/__init__.py diff --git a/apps/assets/automations/gather_facts/database/mysql/main.yml b/apps/assets/automations/gather_facts/database/mysql/main.yml new file mode 100644 index 000000000..e7ba00880 --- /dev/null +++ b/apps/assets/automations/gather_facts/database/mysql/main.yml @@ -0,0 +1,28 @@ +- hosts: mysql + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: root + secret: redhat + jms_asset: + address: 127.0.0.1 + port: 3306 + + tasks: + - name: Gather facts info + community.mysql.mysql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + register: db_info + + - name: Get info + set_fact: + info: + version: "{{ db_info.version.full }}" + + - debug: + var: db_info + diff --git a/apps/assets/automations/gather_facts/database/mysql/manifest.yml b/apps/assets/automations/gather_facts/database/mysql/manifest.yml new file mode 100644 index 000000000..33109b29b --- /dev/null +++ b/apps/assets/automations/gather_facts/database/mysql/manifest.yml @@ -0,0 +1,6 @@ +id: gather_facts_mysql +name: Gather facts from MySQL +category: database +type: + - mysql +method: gather_facts diff --git a/apps/assets/automations/gather_facts/database/postgresql/main.yml b/apps/assets/automations/gather_facts/database/postgresql/main.yml new file mode 100644 index 000000000..6af98366b --- /dev/null +++ b/apps/assets/automations/gather_facts/database/postgresql/main.yml @@ -0,0 +1,28 @@ +- hosts: postgre + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: postgre + secret: postgre + jms_asset: + address: 127.0.0.1 + port: 5432 + database: testdb + account: + username: test + secret: jumpserver + + tasks: + - name: Test PostgreSQL connection + community.postgresql.postgresql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + login_db: "{{ jms_asset.database }}" + register: db_info + + - name: Debug it + debug: + var: db_info diff --git a/apps/assets/automations/gather_facts/database/postgresql/manifest.yml b/apps/assets/automations/gather_facts/database/postgresql/manifest.yml new file mode 100644 index 000000000..19bf255de --- /dev/null +++ b/apps/assets/automations/gather_facts/database/postgresql/manifest.yml @@ -0,0 +1,6 @@ +id: gather_facts_postgresql +name: Gather facts for PostgreSQL +category: database +type: + - postgresql +method: gather_facts diff --git a/apps/assets/automations/gather_facts/database/sqlserver/main.yml b/apps/assets/automations/gather_facts/database/sqlserver/main.yml new file mode 100644 index 000000000..402c7fa8d --- /dev/null +++ b/apps/assets/automations/gather_facts/database/sqlserver/main.yml @@ -0,0 +1,10 @@ +{% for account in accounts %} +- hosts: {{ account.asset.name }} + vars: + account: + username: {{ account.username }} + password: {{ account.password }} + public_key: {{ account.public_key }} + roles: + - change_password +{% endfor %} diff --git a/apps/assets/automations/gather_facts/database/sqlserver/manifest.yml b/apps/assets/automations/gather_facts/database/sqlserver/manifest.yml new file mode 100644 index 000000000..3c4c82de4 --- /dev/null +++ b/apps/assets/automations/gather_facts/database/sqlserver/manifest.yml @@ -0,0 +1,8 @@ +id: gather_facts_sqlserver +name: Change password for SQLServer +version: 1 +category: database +type: + - sqlserver +method: gather_facts + diff --git a/apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml b/apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml new file mode 100644 index 000000000..903cd9115 --- /dev/null +++ b/apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml @@ -0,0 +1,27 @@ +- name: ping + ping: + +#- name: print variables +# debug: +# msg: "Username: {{ account.username }}, Password: {{ account.password }}" + +- name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.password + +- name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + +- name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/automations/gather_facts/demo_inventory.txt b/apps/assets/automations/gather_facts/demo_inventory.txt new file mode 100644 index 000000000..ed011eae2 --- /dev/null +++ b/apps/assets/automations/gather_facts/demo_inventory.txt @@ -0,0 +1,2 @@ +# all base inventory in base/base_inventory.txt +asset_name(ip) ...base_inventory_vars diff --git a/apps/assets/automations/gather_facts/host/posix/main.yml b/apps/assets/automations/gather_facts/host/posix/main.yml new file mode 100644 index 000000000..6e900fccb --- /dev/null +++ b/apps/assets/automations/gather_facts/host/posix/main.yml @@ -0,0 +1,19 @@ +- hosts: website + gather_facts: yes + tasks: + - name: Get info + set_fact: + info: + arch: "{{ ansible_architecture }}" + distribution: "{{ ansible_distribution }}" + distribution_version: "{{ ansible_distribution_version }}" + kernel: "{{ ansible_kernel }}" + vendor: "{{ ansible_system_vendor }}" + model: "{{ ansible_product_name }}" + sn: "{{ ansible_product_serial }}" + cpu_vcpus: "{{ ansible_processor_vcpus }}" + memory: "{{ ansible_memtotal_mb }}" + disk_total: "{{ (ansible_mounts | map(attribute='size_total') | sum / 1024 / 1024 / 1024) | round(2) }}" + + - debug: + var: info diff --git a/apps/assets/automations/gather_facts/host/posix/manifest.yml b/apps/assets/automations/gather_facts/host/posix/manifest.yml new file mode 100644 index 000000000..e5622cf28 --- /dev/null +++ b/apps/assets/automations/gather_facts/host/posix/manifest.yml @@ -0,0 +1,8 @@ +id: gather_facts_posix +name: Gather posix facts +category: host +type: + - linux + - windows + - unix +method: gather_facts diff --git a/apps/assets/automations/gather_facts/host/windows/main.yml b/apps/assets/automations/gather_facts/host/windows/main.yml new file mode 100644 index 000000000..723aa7720 --- /dev/null +++ b/apps/assets/automations/gather_facts/host/windows/main.yml @@ -0,0 +1,24 @@ +- hosts: windows + gather_facts: yes + tasks: +# - name: Gather facts windows +# setup: +# register: facts +# +# - debug: +# var: facts + - name: Get info + set_fact: + info: + arch: "{{ ansible_architecture2 }}" + distribution: "{{ ansible_distribution }}" + distribution_version: "{{ ansible_distribution_version }}" + kernel: "{{ ansible_kernel }}" + vendor: "{{ ansible_system_vendor }}" + model: "{{ ansible_product_name }}" + sn: "{{ ansible_product_serial }}" + cpu_vcpus: "{{ ansible_processor_vcpus }}" + memory: "{{ ansible_memtotal_mb }}" +t + - debug: + var: info diff --git a/apps/assets/automations/gather_facts/host/windows/manifest.yml b/apps/assets/automations/gather_facts/host/windows/manifest.yml new file mode 100644 index 000000000..929a6626f --- /dev/null +++ b/apps/assets/automations/gather_facts/host/windows/manifest.yml @@ -0,0 +1,7 @@ +id: gather_facts_windows +name: Gather facts windows +version: 1 +method: gather_facts +category: host +type: + - windows diff --git a/apps/assets/automations/gather_facts/manager.py b/apps/assets/automations/gather_facts/manager.py new file mode 100644 index 000000000..7b56d728e --- /dev/null +++ b/apps/assets/automations/gather_facts/manager.py @@ -0,0 +1,77 @@ +import os +import shutil +from copy import deepcopy +from collections import defaultdict + +import yaml +from django.utils.translation import gettext as _ + +from ops.ansible import PlaybookRunner +from ..base.manager import BasePlaybookManager +from assets.automations.methods import platform_automation_methods + + +class GatherFactsManager(BasePlaybookManager): + method_name = 'gather_facts' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.id_method_mapper = { + method['id']: method + for method in platform_automation_methods + if method['method'] == self.method_name + } + self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] + + def inventory_kwargs(self): + return { + } + + def generate_playbook(self): + playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.id_method_mapper[method_id] + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + + step = 10 + hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] + for i, hosts in enumerate(hosts_grouped): + plays = [] + play = deepcopy(host_playbook_play) + play['hosts'] = ':'.join(hosts) + plays.append(play) + + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) + + playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(playbook, f) + + print("Generate playbook done: " + self.playbook_path) + + def get_runner(self): + return PlaybookRunner( + self.inventory_path, + self.playbook_path, + self.runtime_dir + ) + + diff --git a/apps/assets/automations/generate_playbook/change_password.py b/apps/assets/automations/generate_playbook/change_password.py deleted file mode 100644 index 20c3f0889..000000000 --- a/apps/assets/automations/generate_playbook/change_password.py +++ /dev/null @@ -1,106 +0,0 @@ -import os -import yaml -import jinja2 -from typing import List - -from django.conf import settings -from assets.models import Asset -from .base import BaseGeneratePlaybook - - -class GenerateChangePasswordPlaybook(BaseGeneratePlaybook): - - def __init__( - self, assets: List[Asset], strategy, usernames, password='', - private_key='', public_key='', key_strategy='' - ): - super().__init__(assets, strategy) - self.password = password - self.public_key = public_key - self.private_key = private_key - self.key_strategy = key_strategy - self.relation_asset_map = self.get_username_relation_asset_map(usernames) - - def get_username_relation_asset_map(self, usernames): - # TODO 没特权用户的资产 要考虑网关 - - complete_map = { - asset: list(asset.accounts.value_list('username', flat=True)) - for asset in self.assets - } - - if '*' in usernames: - return complete_map - - relation_map = {} - for asset, usernames in complete_map.items(): - usernames = list(set(usernames) & set(usernames)) - if not usernames: - continue - relation_map[asset] = list(set(usernames) & set(usernames)) - return relation_map - - @property - def src_filepath(self): - return os.path.join( - settings.BASE_DIR, 'assets', 'playbooks', 'strategy', - 'change_password', 'roles', self.strategy - ) - - def generate_hosts(self): - host_pathname = os.path.join(self.temp_folder, 'hosts') - with open(host_pathname, 'w', encoding='utf8') as f: - for asset in self.relation_asset_map.keys(): - f.write(f'{asset.name}\n') - - def generate_host_vars(self): - host_vars_pathname = os.path.join(self.temp_folder, 'hosts', 'host_vars') - os.makedirs(host_vars_pathname, exist_ok=True) - for asset, usernames in self.relation_asset_map.items(): - host_vars = { - 'ansible_host': asset.get_target_ip(), - 'ansible_port': asset.get_target_ssh_port(), # TODO 需要根绝协议取端口号 - 'ansible_user': asset.admin_user.username, - 'ansible_pass': asset.admin_user.username, - 'usernames': usernames, - } - pathname = os.path.join(host_vars_pathname, f'{asset.name}.yml') - with open(pathname, 'w', encoding='utf8') as f: - f.write(yaml.dump(host_vars, allow_unicode=True)) - - def generate_secret_key_files(self): - if not self.private_key and not self.public_key: - return - - file_pathname = os.path.join(self.temp_folder, self.strategy, 'files') - public_pathname = os.path.join(file_pathname, 'id_rsa.pub') - private_pathname = os.path.join(file_pathname, 'id_rsa') - - os.makedirs(file_pathname, exist_ok=True) - with open(public_pathname, 'w', encoding='utf8') as f: - f.write(self.public_key) - with open(private_pathname, 'w', encoding='utf8') as f: - f.write(self.private_key) - - def generate_role_main(self): - task_main_pathname = os.path.join(self.temp_folder, 'main.yaml') - context = { - 'password': self.password, - 'key_strategy': self.key_strategy, - 'private_key_file': 'id_rsa' if self.private_key else '', - 'exclusive': 'no' if self.key_strategy == 'all' else 'yes', - 'jms_key': self.public_key.split()[2].strip() if self.public_key else '', - } - with open(task_main_pathname, 'r+', encoding='utf8') as f: - string_var = f.read() - f.seek(0, 0) - response = jinja2.Template(string_var).render(context) - results = yaml.safe_load(response) - f.write(yaml.dump(results, allow_unicode=True)) - - def execute(self): - self.generate_temp_playbook() - self.generate_hosts() - self.generate_host_vars() - self.generate_secret_key_files() - self.generate_role_main() diff --git a/apps/assets/automations/generate_playbook/verify.py b/apps/assets/automations/generate_playbook/verify.py deleted file mode 100644 index 88695b814..000000000 --- a/apps/assets/automations/generate_playbook/verify.py +++ /dev/null @@ -1,86 +0,0 @@ -import os -import yaml -from typing import List - -from django.conf import settings -from assets.models import Asset -from .base import BaseGeneratePlaybook - - -class GenerateVerifyPlaybook(BaseGeneratePlaybook): - - def __init__( - self, assets: List[Asset], strategy, usernames - ): - super().__init__(assets, strategy) - self.relation_asset_map = self.get_account_relation_asset_map(usernames) - - def get_account_relation_asset_map(self, usernames): - # TODO 没特权用户的资产 要考虑网关 - complete_map = { - asset: list(asset.accounts.all()) - for asset in self.assets - } - - if '*' in usernames: - return complete_map - - relation_map = {} - for asset, accounts in complete_map.items(): - account_map = {account.username: account for account in accounts} - accounts = [account_map[i] for i in (set(usernames) & set(account_map))] - if not accounts: - continue - relation_map[asset] = accounts - return relation_map - - @property - def src_filepath(self): - return os.path.join( - settings.BASE_DIR, 'assets', 'playbooks', 'strategy', - 'verify', 'roles', self.strategy - ) - - def generate_hosts(self): - host_pathname = os.path.join(self.temp_folder, 'hosts') - with open(host_pathname, 'w', encoding='utf8') as f: - for asset in self.relation_asset_map.keys(): - f.write(f'{asset.name}\n') - - def generate_host_vars(self): - host_vars_pathname = os.path.join(self.temp_folder, 'hosts', 'host_vars') - os.makedirs(host_vars_pathname, exist_ok=True) - for asset, accounts in self.relation_asset_map.items(): - account_info = [] - for account in accounts: - private_key_filename = f'{asset.name}_{account.username}' if account.private_key else '' - account_info.append({ - 'username': account.username, - 'password': account.password, - 'private_key_filename': private_key_filename, - }) - host_vars = { - 'ansible_host': asset.get_target_ip(), - 'ansible_port': asset.get_target_ssh_port(), # TODO 需要根绝协议取端口号 - 'account_info': account_info, - } - pathname = os.path.join(host_vars_pathname, f'{asset.name}.yml') - with open(pathname, 'w', encoding='utf8') as f: - f.write(yaml.dump(host_vars, allow_unicode=True)) - - def generate_secret_key_files(self): - file_pathname = os.path.join(self.temp_folder, self.strategy, 'files') - os.makedirs(file_pathname, exist_ok=True) - for asset, accounts in self.relation_asset_map.items(): - for account in accounts: - if account.private_key: - path_name = os.path.join(file_pathname, f'{asset.name}_{account.username}') - with open(path_name, 'w', encoding='utf8') as f: - f.write(account.private_key) - - def execute(self): - self.generate_temp_playbook() - self.generate_hosts() - self.generate_host_vars() - self.generate_secret_key_files() - # self.generate_role_main() # TODO Linux 暂时不需要 diff --git a/apps/assets/automations/ping/__init__.py b/apps/assets/automations/ping/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/assets/automations/ping/database/mysql/main.yml b/apps/assets/automations/ping/database/mysql/main.yml new file mode 100644 index 000000000..fab498c76 --- /dev/null +++ b/apps/assets/automations/ping/database/mysql/main.yml @@ -0,0 +1,20 @@ +- hosts: mysql + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: root + password: redhat + jms_asset: + address: 127.0.0.1 + port: 3306 + + tasks: + - name: Test MySQL connection + community.mysql.mysql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + filter: version + register: db_info diff --git a/apps/assets/automations/ping/database/mysql/manifest.yml b/apps/assets/automations/ping/database/mysql/manifest.yml new file mode 100644 index 000000000..aded00b1f --- /dev/null +++ b/apps/assets/automations/ping/database/mysql/manifest.yml @@ -0,0 +1,6 @@ +id: mysql_ping +name: Ping MySQL +category: database +type: + - mysql +method: ping diff --git a/apps/assets/automations/ping/database/postgresql/main.yml b/apps/assets/automations/ping/database/postgresql/main.yml new file mode 100644 index 000000000..3bc2f7957 --- /dev/null +++ b/apps/assets/automations/ping/database/postgresql/main.yml @@ -0,0 +1,23 @@ +- hosts: postgre + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: postgre + secret: postgre + jms_asset: + address: 127.0.0.1 + port: 5432 + database: testdb + account: + username: test + secret: jumpserver + + tasks: + - name: Test PostgreSQL connection + community.postgresql.postgresql_ping: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + login_db: "{{ jms_asset.database }}" diff --git a/apps/assets/automations/ping/database/postgresql/manifest.yml b/apps/assets/automations/ping/database/postgresql/manifest.yml new file mode 100644 index 000000000..337b2b50d --- /dev/null +++ b/apps/assets/automations/ping/database/postgresql/manifest.yml @@ -0,0 +1,6 @@ +id: ping_postgresql +name: Ping PostgreSQL +category: database +type: + - postgresql +method: ping diff --git a/apps/assets/automations/ping/demo_inventory.txt b/apps/assets/automations/ping/demo_inventory.txt new file mode 100644 index 000000000..dcc7d1b6d --- /dev/null +++ b/apps/assets/automations/ping/demo_inventory.txt @@ -0,0 +1,2 @@ +# all base inventory in base/base_inventory.txt +asset_name(ip)_account_username account={"username": "", "password": "xxx"} ...base_inventory_vars diff --git a/apps/assets/automations/ping/host/posix/main.yml b/apps/assets/automations/ping/host/posix/main.yml new file mode 100644 index 000000000..c4c740367 --- /dev/null +++ b/apps/assets/automations/ping/host/posix/main.yml @@ -0,0 +1,5 @@ +- hosts: demo + gather_facts: no + tasks: + - name: Posix ping + ping: diff --git a/apps/assets/automations/ping/host/posix/manifest.yml b/apps/assets/automations/ping/host/posix/manifest.yml new file mode 100644 index 000000000..b07caa7f8 --- /dev/null +++ b/apps/assets/automations/ping/host/posix/manifest.yml @@ -0,0 +1,8 @@ +id: posix_ping +name: Posix ping +category: host +type: + - linux + - windows + - unix +method: ping diff --git a/apps/assets/automations/ping/host/windows/main.yml b/apps/assets/automations/ping/host/windows/main.yml new file mode 100644 index 000000000..495b82a3d --- /dev/null +++ b/apps/assets/automations/ping/host/windows/main.yml @@ -0,0 +1,5 @@ +- hosts: windows + gather_facts: no + tasks: + - name: Windows ping + win_ping: diff --git a/apps/assets/automations/ping/host/windows/manifest.yml b/apps/assets/automations/ping/host/windows/manifest.yml new file mode 100644 index 000000000..55e336f19 --- /dev/null +++ b/apps/assets/automations/ping/host/windows/manifest.yml @@ -0,0 +1,7 @@ +id: win_ping +name: Windows ping +version: 1 +method: change_password +category: host +type: + - windows diff --git a/apps/assets/automations/ping/manager.py b/apps/assets/automations/ping/manager.py new file mode 100644 index 000000000..36017438b --- /dev/null +++ b/apps/assets/automations/ping/manager.py @@ -0,0 +1,75 @@ +import os +import shutil +from copy import deepcopy +from collections import defaultdict + +import yaml +from django.utils.translation import gettext as _ + +from ops.ansible import PlaybookRunner +from ..base.manager import BasePlaybookManager +from assets.automations.methods import platform_automation_methods + + +class ChangePasswordManager(BasePlaybookManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.id_method_mapper = { + method['id']: method + for method in platform_automation_methods + } + self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] + + def inventory_kwargs(self): + return { + 'host_callback': self.host_duplicator + } + + def generate_playbook(self): + playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.id_method_mapper[method_id] + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + + step = 10 + hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] + for i, hosts in enumerate(hosts_grouped): + plays = [] + play = deepcopy(host_playbook_play) + play['hosts'] = ':'.join(hosts) + plays.append(play) + + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) + + playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(playbook, f) + + print("Generate playbook done: " + self.playbook_path) + + def get_runner(self): + return PlaybookRunner( + self.inventory_path, + self.playbook_path, + self.runtime_dir + ) + + diff --git a/apps/assets/models/automations/change_secret.py b/apps/assets/models/automations/change_secret.py index 4a2c304c2..dc0403a12 100644 --- a/apps/assets/models/automations/change_secret.py +++ b/apps/assets/models/automations/change_secret.py @@ -19,6 +19,10 @@ class ChangePasswordAutomation(BaseAutomation): verbose_name=_("Recipient") ) + def save(self, *args, **kwargs): + self.type = 'change_password' + super().save(*args, **kwargs) + class Meta: verbose_name = _("Change auth strategy") diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 9d4498515..a91126b5f 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -10,18 +10,17 @@ __all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, assets, account='', account_policy='smart', host_var_callback=None, host_duplicator=None): + def __init__(self, assets, account='', account_policy='smart', host_callback=None): """ :param assets: :param account: account username name if not set use account_policy :param account_policy: - :param host_var_callback: + :param host_callback: after generate host, call this callback to modify host """ self.assets = self.clean_assets(assets) self.account_username = account self.account_policy = account_policy - self.host_var_callback = host_var_callback - self.host_duplicator = host_duplicator + self.host_callback = host_callback @staticmethod def clean_assets(assets): @@ -100,15 +99,10 @@ class JMSInventory: elif account.secret_type == 'private_key' and account.secret: host['ssh_private_key'] = account.private_key_file else: - host['exclude'] = _("No account found") + host['error'] = _("No account found") if gateway: host.update(self.make_proxy_command(gateway)) - - if self.host_var_callback: - callback_var = self.host_var_callback(asset) - if isinstance(callback_var, dict): - host.update(callback_var) return host def select_account(self, asset): @@ -145,10 +139,18 @@ class JMSInventory: for asset in self.assets: account = self.select_account(asset) host = self.asset_to_host(asset, account, automation, protocols) + if not automation.ansible_enabled: - host['exclude'] = _('Ansible disabled') - if self.host_duplicator: - hosts.extend(self.host_duplicator(host, asset=asset, account=account, platform=platform)) + host['error'] = _('Ansible disabled') + + if self.host_callback is not None: + host = self.host_callback( + host, asset=asset, account=account, + platform=platform, automation=automation + ) + + if isinstance(host, list): + hosts.extend(host) else: hosts.append(host) @@ -156,7 +158,7 @@ class JMSInventory: if exclude_hosts: print(_("Skip hosts below:")) for i, host in enumerate(exclude_hosts, start=1): - print("{}: [{}] \t{}".format(i, host['name'], host['exclude'])) + print("{}: [{}] \t{}".format(i, host['name'], host['error'])) hosts = list(filter(lambda x: not x.get('exclude'), hosts)) data = {'all': {'hosts': {}}} diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index e8f232e74..fbf3245ae 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -52,12 +52,14 @@ class AdHocRunner: class PlaybookRunner: - def __init__(self, inventory, playbook, project_dir='/tmp/'): + def __init__(self, inventory, playbook, project_dir='/tmp/', callback=None): self.id = uuid.uuid4() self.inventory = inventory self.playbook = playbook self.project_dir = project_dir - self.cb = DefaultCallback() + if not callback: + callback = DefaultCallback() + self.cb = callback def run(self, verbosity=0, **kwargs): if verbosity is None and settings.DEBUG: