From 90fdaca9550ae0738764870ecfc9e8da42bd80f5 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Tue, 7 Feb 2023 08:52:48 +0800 Subject: [PATCH 01/44] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96Activity?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/api/account/account.py | 4 +- apps/accounts/api/automations/base.py | 3 +- apps/accounts/models/automations/base.py | 4 +- apps/accounts/tasks/automation.py | 4 +- apps/accounts/tasks/common.py | 4 +- apps/accounts/tasks/verify_account.py | 16 +++---- apps/assets/api/asset/asset.py | 12 +++++- apps/assets/automations/base/manager.py | 23 +++++++++- apps/assets/models/automations/base.py | 8 ++-- apps/assets/tasks/common.py | 4 +- apps/assets/tasks/gather_facts.py | 12 +++--- apps/assets/tasks/ping.py | 18 ++++---- apps/audits/api.py | 37 ++++++++++++++--- apps/audits/const.py | 7 ++++ apps/audits/handler.py | 53 ------------------------ apps/audits/models.py | 31 +++++++++++++- apps/audits/serializers.py | 25 ++++++++--- apps/audits/signal_handlers.py | 23 +++++++--- apps/audits/signals.py | 6 +++ apps/audits/tasks.py | 9 +++- apps/audits/utils.py | 17 +++++++- apps/jumpserver/conf.py | 1 + apps/jumpserver/settings/custom.py | 1 + apps/rbac/const.py | 1 + apps/settings/serializers/cleaning.py | 5 ++- apps/terminal/api/session/session.py | 24 ++++++++++- 26 files changed, 239 insertions(+), 113 deletions(-) create mode 100644 apps/audits/signals.py diff --git a/apps/accounts/api/account/account.py b/apps/accounts/api/account/account.py index 9aa31c72d..728e7f333 100644 --- a/apps/accounts/api/account/account.py +++ b/apps/accounts/api/account/account.py @@ -53,7 +53,9 @@ class AccountViewSet(OrgBulkModelViewSet): account = super().get_object() account_ids = [account.id] asset_ids = [account.asset_id] - task = verify_accounts_connectivity.delay(account_ids, asset_ids) + task = verify_accounts_connectivity.delay( + account_ids, asset_ids, user=request.user + ) return Response(data={'task': task.id}) diff --git a/apps/accounts/api/automations/base.py b/apps/accounts/api/automations/base.py index 12fcd7b17..e8f762c91 100644 --- a/apps/accounts/api/automations/base.py +++ b/apps/accounts/api/automations/base.py @@ -110,6 +110,7 @@ class AutomationExecutionViewSet( serializer.is_valid(raise_exception=True) automation = serializer.validated_data.get('automation') task = execute_automation.delay( - pid=automation.pk, trigger=Trigger.manual, tp=self.tp + pid=automation.pk, trigger=Trigger.manual, + tp=self.tp, user=request.user ) return Response({'task': task.id}, status=status.HTTP_201_CREATED) diff --git a/apps/accounts/models/automations/base.py b/apps/accounts/models/automations/base.py index ce45d6a81..4c5038481 100644 --- a/apps/accounts/models/automations/base.py +++ b/apps/accounts/models/automations/base.py @@ -35,7 +35,7 @@ class AutomationExecution(AssetAutomationExecution): ('add_pushaccountexecution', _('Can add push account execution')), ] - def start(self): + def start(self, **kwargs): from accounts.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) - return manager.run() + return manager.run(**kwargs) diff --git a/apps/accounts/tasks/automation.py b/apps/accounts/tasks/automation.py index 67d4c4a90..e25a1aaf8 100644 --- a/apps/accounts/tasks/automation.py +++ b/apps/accounts/tasks/automation.py @@ -9,7 +9,7 @@ logger = get_logger(__file__) @shared_task(queue='ansible', verbose_name=_('Account execute automation')) -def execute_automation(pid, trigger, tp): +def execute_automation(pid, trigger, tp, **kwargs): model = AutomationTypes.get_type_model(tp) with tmp_to_root_org(): instance = get_object_or_none(model, pk=pid) @@ -17,4 +17,4 @@ def execute_automation(pid, trigger, tp): logger.error("No automation task found: {}".format(pid)) return with tmp_to_org(instance.org): - instance.execute(trigger) + instance.execute(trigger, **kwargs) diff --git a/apps/accounts/tasks/common.py b/apps/accounts/tasks/common.py index 1f422ab5f..19c750c4d 100644 --- a/apps/accounts/tasks/common.py +++ b/apps/accounts/tasks/common.py @@ -5,7 +5,7 @@ from assets.tasks.common import generate_data from common.const.choices import Trigger -def automation_execute_start(task_name, tp, child_snapshot=None): +def automation_execute_start(task_name, tp, child_snapshot=None, **kwargs): from accounts.models import AutomationExecution data = generate_data(task_name, tp, child_snapshot) @@ -19,4 +19,4 @@ def automation_execute_start(task_name, tp, child_snapshot=None): execution = AutomationExecution.objects.create( trigger=Trigger.manual, **data ) - execution.start() + execution.start(**kwargs) diff --git a/apps/accounts/tasks/verify_account.py b/apps/accounts/tasks/verify_account.py index 4c478ce89..4ec741499 100644 --- a/apps/accounts/tasks/verify_account.py +++ b/apps/accounts/tasks/verify_account.py @@ -14,7 +14,7 @@ __all__ = [ ] -def verify_connectivity_util(assets, tp, accounts, task_name): +def verify_connectivity_util(assets, tp, accounts, task_name, **kwargs): if not assets or not accounts: return account_usernames = list(accounts.values_list('username', flat=True)) @@ -22,28 +22,30 @@ def verify_connectivity_util(assets, tp, accounts, task_name): 'accounts': account_usernames, 'assets': [str(asset.id) for asset in assets], } - automation_execute_start(task_name, tp, child_snapshot) + automation_execute_start(task_name, tp, child_snapshot, **kwargs) @org_aware_func("assets") -def verify_accounts_connectivity_util(accounts, assets, task_name): +def verify_accounts_connectivity_util(accounts, assets, task_name, **kwargs): gateway_assets = assets.filter(platform__name=GATEWAY_NAME) verify_connectivity_util( - gateway_assets, AutomationTypes.verify_gateway_account, accounts, task_name + gateway_assets, AutomationTypes.verify_gateway_account, + accounts, task_name, **kwargs ) non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) verify_connectivity_util( - non_gateway_assets, AutomationTypes.verify_account, accounts, task_name + non_gateway_assets, AutomationTypes.verify_account, + accounts, task_name, **kwargs ) @shared_task(queue="ansible", verbose_name=_('Verify asset account availability')) -def verify_accounts_connectivity(account_ids, asset_ids): +def verify_accounts_connectivity(account_ids, asset_ids, **kwargs): from assets.models import Asset from accounts.models import Account, VerifyAccountAutomation assets = Asset.objects.filter(id__in=asset_ids) accounts = Account.objects.filter(id__in=account_ids) task_name = gettext_noop("Verify accounts connectivity") task_name = VerifyAccountAutomation.generate_unique_name(task_name) - return verify_accounts_connectivity_util(accounts, assets, task_name) + return verify_accounts_connectivity_util(accounts, assets, task_name, **kwargs) diff --git a/apps/assets/api/asset/asset.py b/apps/assets/api/asset/asset.py index c0dda34c7..c9544cc3b 100644 --- a/apps/assets/api/asset/asset.py +++ b/apps/assets/api/asset/asset.py @@ -3,6 +3,7 @@ import django_filters from django.db.models import Q from django.utils.translation import ugettext_lazy as _ +from rest_framework.request import Request from rest_framework.decorators import action from rest_framework.response import Response @@ -105,14 +106,21 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet): class AssetsTaskMixin: + request: Request + def perform_assets_task(self, serializer): data = serializer.validated_data assets = data.get("assets", []) asset_ids = [asset.id for asset in assets] + user = self.request.user if data["action"] == "refresh": - task = update_assets_hardware_info_manual.delay(asset_ids) + task = update_assets_hardware_info_manual.delay( + asset_ids, user=user + ) else: - task = test_assets_connectivity_manual.delay(asset_ids) + task = test_assets_connectivity_manual.delay( + asset_ids, user=user + ) return task def perform_create(self, serializer): diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 50a5160e2..37998b3b6 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -10,6 +10,9 @@ from django.utils import timezone from django.utils.translation import gettext as _ from assets.automations.methods import platform_automation_methods +from audits.signals import post_activity_log +from audits.const import ActivityChoices +from common.utils import reverse from common.utils import get_logger, lazyproperty from common.utils import ssh_pubkey_gen, ssh_key_string_to_obj from ops.ansible import JMSInventory, PlaybookRunner, DefaultCallback @@ -118,7 +121,22 @@ class BasePlaybookManager: yaml.safe_dump(plays, f) return sub_playbook_path - def get_runners(self): + def send_activity(self, assets, **kwargs): + user = kwargs.pop('user', _('Unknown')) + task_type = self.method_type().label + detail = 'User %s performs a task(%s) for this resource.' % ( + user, task_type + ) + detail_url = reverse( + 'ops:celery-task-log', kwargs={'pk': self.execution.id} + ) + for a in assets: + post_activity_log.send( + sender=self, resource_id=a.id, detail=detail, + detail_url=detail_url, type=ActivityChoices.task + ) + + def get_runners(self, **kwargs): runners = [] for platform, assets in self.get_assets_group_by_platform().items(): assets_bulked = [assets[i:i + self.bulk_size] for i in range(0, len(assets), self.bulk_size)] @@ -137,6 +155,7 @@ class BasePlaybookManager: callback=PlaybookCallback(), ) runners.append(runer) + self.send_activity(assets, **kwargs) return runners def on_host_success(self, host, result): @@ -166,7 +185,7 @@ class BasePlaybookManager: pass def run(self, *args, **kwargs): - runners = self.get_runners() + runners = self.get_runners(user=kwargs.pop('user')) if len(runners) > 1: print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) else: diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index e888fdf26..ac05d1080 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -76,7 +76,7 @@ class BaseAutomation(PeriodTaskModelMixin, JMSOrgBaseModel): def executed_amount(self): return self.executions.count() - def execute(self, trigger=Trigger.manual): + def execute(self, trigger=Trigger.manual, **kwargs): try: eid = current_task.request.id except AttributeError: @@ -86,7 +86,7 @@ class BaseAutomation(PeriodTaskModelMixin, JMSOrgBaseModel): id=eid, trigger=trigger, automation=self, snapshot=self.to_attr_json(), ) - return execution.start() + return execution.start(**kwargs) class AssetBaseAutomation(BaseAutomation): @@ -140,7 +140,7 @@ class AutomationExecution(OrgModelMixin): return {} return recipients - def start(self): + def start(self, **kwargs): from assets.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) - return manager.run() + return manager.run(**kwargs) diff --git a/apps/assets/tasks/common.py b/apps/assets/tasks/common.py index 9bafbcde8..2d90dcf92 100644 --- a/apps/assets/tasks/common.py +++ b/apps/assets/tasks/common.py @@ -29,7 +29,7 @@ def generate_data(task_name, tp, child_snapshot=None): return {'id': eid, 'snapshot': snapshot} -def automation_execute_start(task_name, tp, child_snapshot=None): +def automation_execute_start(task_name, tp, child_snapshot=None, **kwargs): from assets.models import AutomationExecution data = generate_data(task_name, tp, child_snapshot) @@ -43,4 +43,4 @@ def automation_execute_start(task_name, tp, child_snapshot=None): execution = AutomationExecution.objects.create( trigger=Trigger.manual, **data ) - execution.start() + execution.start(**kwargs) diff --git a/apps/assets/tasks/gather_facts.py b/apps/assets/tasks/gather_facts.py index ad678b92a..075394d3f 100644 --- a/apps/assets/tasks/gather_facts.py +++ b/apps/assets/tasks/gather_facts.py @@ -17,7 +17,7 @@ __all__ = [ ] -def update_fact_util(assets=None, nodes=None, task_name=None): +def update_fact_util(assets=None, nodes=None, task_name=None, **kwargs): from assets.models import GatherFactsAutomation if task_name is None: task_name = gettext_noop("Update some assets hardware info. ") @@ -30,16 +30,16 @@ def update_fact_util(assets=None, nodes=None, task_name=None): 'nodes': [str(node.id) for node in nodes], } tp = AutomationTypes.gather_facts - automation_execute_start(task_name, tp, child_snapshot) + automation_execute_start(task_name, tp, child_snapshot, **kwargs) @org_aware_func('assets') -def update_assets_fact_util(assets=None, task_name=None): +def update_assets_fact_util(assets=None, task_name=None, **kwargs): if assets is None: logger.info("No assets to update hardware info") return - update_fact_util(assets=assets, task_name=task_name) + update_fact_util(assets=assets, task_name=task_name, **kwargs) @org_aware_func('nodes') @@ -51,11 +51,11 @@ def update_nodes_fact_util(nodes=None, task_name=None): @shared_task(queue="ansible", verbose_name=_('Manually update the hardware information of assets')) -def update_assets_hardware_info_manual(asset_ids): +def update_assets_hardware_info_manual(asset_ids, **kwargs): from assets.models import Asset assets = Asset.objects.filter(id__in=asset_ids) task_name = gettext_noop("Update assets hardware info: ") - update_assets_fact_util(assets=assets, task_name=task_name) + update_assets_fact_util(assets=assets, task_name=task_name, **kwargs) @shared_task(queue="ansible", verbose_name=_('Manually update the hardware information of assets under a node')) diff --git a/apps/assets/tasks/ping.py b/apps/assets/tasks/ping.py index 7117c8bc7..c4e4bcf04 100644 --- a/apps/assets/tasks/ping.py +++ b/apps/assets/tasks/ping.py @@ -17,7 +17,7 @@ __all__ = [ ] -def test_connectivity_util(assets, tp, task_name, local_port=None): +def test_connectivity_util(assets, tp, task_name, local_port=None, **kwargs): if not assets: return @@ -27,11 +27,11 @@ def test_connectivity_util(assets, tp, task_name, local_port=None): child_snapshot = {'local_port': local_port} child_snapshot['assets'] = [str(asset.id) for asset in assets] - automation_execute_start(task_name, tp, child_snapshot) + automation_execute_start(task_name, tp, child_snapshot, **kwargs) @org_aware_func('assets') -def test_asset_connectivity_util(assets, task_name=None, local_port=None): +def test_asset_connectivity_util(assets, task_name=None, local_port=None, **kwargs): from assets.models import PingAutomation if task_name is None: task_name = gettext_noop("Test assets connectivity ") @@ -40,19 +40,23 @@ def test_asset_connectivity_util(assets, task_name=None, local_port=None): gateway_assets = assets.filter(platform__name=GATEWAY_NAME) test_connectivity_util( - gateway_assets, AutomationTypes.ping_gateway, task_name, local_port + gateway_assets, AutomationTypes.ping_gateway, + task_name, local_port, **kwargs ) non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) - test_connectivity_util(non_gateway_assets, AutomationTypes.ping, task_name) + test_connectivity_util( + non_gateway_assets, AutomationTypes.ping, + task_name, **kwargs + ) @shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of a asset')) -def test_assets_connectivity_manual(asset_ids, local_port=None): +def test_assets_connectivity_manual(asset_ids, local_port=None, **kwargs): from assets.models import Asset assets = Asset.objects.filter(id__in=asset_ids) task_name = gettext_noop("Test assets connectivity ") - test_asset_connectivity_util(assets, task_name, local_port) + test_asset_connectivity_util(assets, task_name, local_port, **kwargs) @shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of assets under a node')) diff --git a/apps/audits/api.py b/apps/audits/api.py index a8c178ac2..0fcde1396 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -3,6 +3,7 @@ from importlib import import_module from django.conf import settings +from django.db.models import F, Value, CharField from rest_framework import generics from rest_framework.permissions import IsAuthenticated from rest_framework.mixins import ListModelMixin, CreateModelMixin, RetrieveModelMixin @@ -14,7 +15,8 @@ from common.plugins.es import QuerySet as ESQuerySet from orgs.utils import current_org, tmp_to_root_org from orgs.mixins.api import OrgGenericViewSet, OrgBulkModelViewSet from .backends import TYPE_ENGINE_MAPPING -from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog +from .const import ActivityChoices +from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog, ActivityLog from .serializers import FTPLogSerializer, UserLoginLogSerializer, JobAuditLogSerializer from .serializers import ( OperateLogSerializer, OperateLogActionDetailSerializer, @@ -79,15 +81,38 @@ class MyLoginLogAPIView(UserLoginCommonMixin, generics.ListAPIView): class ResourceActivityAPIView(generics.ListAPIView): serializer_class = ActivitiesOperatorLogSerializer rbac_perms = { - 'GET': 'audits.view_operatelog', + 'GET': 'audits.view_activitylog', } - def get_queryset(self): - resource_id = self.request.query_params.get('resource_id') - with tmp_to_root_org(): - queryset = OperateLog.objects.filter(resource_id=resource_id)[:30] + @staticmethod + def get_operate_log_qs(fields, limit=30, **filters): + queryset = OperateLog.objects.filter(**filters).annotate( + r_type=Value(ActivityChoices.operate_log, CharField()), + r_detail_url=Value(None, CharField()), + r_detail=Value(None, CharField()), + r_user=F('user'), r_action=F('action'), + ).values(*fields)[:limit] return queryset + @staticmethod + def get_activity_log_qs(fields, limit=30, **filters): + queryset = ActivityLog.objects.filter(**filters).annotate( + r_type=F('type'), r_detail_url=F('detail_url'), r_detail=F('detail'), + r_user=Value(None, CharField()), + r_action=Value(None, CharField()), + ).values(*fields)[:limit] + return queryset + + def get_queryset(self): + limit = 30 + resource_id = self.request.query_params.get('resource_id') + fields = ('id', 'datetime', 'r_detail', 'r_detail_url', 'r_user', 'r_action', 'r_type') + with tmp_to_root_org(): + qs1 = self.get_operate_log_qs(fields, resource_id=resource_id) + qs2 = self.get_activity_log_qs(fields, resource_id=resource_id) + queryset = qs2.union(qs1) + return queryset[:limit] + class OperateLogViewSet(RetrieveModelMixin, ListModelMixin, OrgGenericViewSet): model = OperateLog diff --git a/apps/audits/const.py b/apps/audits/const.py index 1dc37de47..90df97a6c 100644 --- a/apps/audits/const.py +++ b/apps/audits/const.py @@ -35,6 +35,13 @@ class LoginTypeChoices(TextChoices): unknown = "U", _("Unknown") +class ActivityChoices(TextChoices): + operate_log = 'O', _('Operate log') + session_log = 'S', _('Session log') + login_log = 'L', _('Login log') + task = 'T', _('Task') + + class MFAChoices(IntegerChoices): disabled = 0, _("Disabled") enabled = 1, _("Enabled") diff --git a/apps/audits/handler.py b/apps/audits/handler.py index 7aff4c427..03fa4ef2a 100644 --- a/apps/audits/handler.py +++ b/apps/audits/handler.py @@ -130,58 +130,6 @@ class OperatorLogHandler(metaclass=Singleton): after = self.__data_processing(after) return before, after - @staticmethod - def _get_Session_params(resource, **kwargs): - # 更新会话的日志不在Activity中体现， - # 否则会话结束，录像文件结束操作的会话记录都会体现出来 - params = {} - action = kwargs.get('data', {}).get('action', 'create') - detail = _( - '{} used account[{}], login method[{}] login the asset.' - ).format( - resource.user, resource.account, resource.login_from_display - ) - if action == ActionChoices.create: - params = { - 'action': ActionChoices.connect, - 'resource_id': str(resource.asset_id), - 'user': resource.user, 'detail': detail - } - return params - - @staticmethod - def _get_ChangeSecretRecord_params(resource, **kwargs): - detail = _( - 'User {} has executed change auth plan for this account.({})' - ).format( - resource.created_by, _(resource.status.title()) - ) - return { - 'action': ActionChoices.change_auth, 'detail': detail, - 'resource_id': str(resource.account_id), - } - - @staticmethod - def _get_UserLoginLog_params(resource, **kwargs): - username = resource.username - login_status = _('Success') if resource.status else _('Failed') - detail = _('User {} login into this service.[{}]').format( - resource.username, login_status - ) - user_id = User.objects.filter(username=username).\ - values_list('id', flat=True)[0] - return { - 'action': ActionChoices.login, 'detail': detail, - 'resource_id': str(user_id), - } - - def _activity_handle(self, data, object_name, resource): - param_func = getattr(self, '_get_%s_params' % object_name, None) - if param_func is not None: - params = param_func(resource, data=data) - data.update(params) - return data - def create_or_update_operate_log( self, action, resource_type, resource=None, force=False, log_id=None, before=None, after=None, @@ -204,7 +152,6 @@ class OperatorLogHandler(metaclass=Singleton): 'remote_addr': remote_addr, 'before': before, 'after': after, 'org_id': get_current_org_id(), 'resource_id': str(resource.id) } - data = self._activity_handle(data, object_name, resource=resource) with transaction.atomic(): if self.log_client.ping(timeout=1): client = self.log_client diff --git a/apps/audits/models.py b/apps/audits/models.py index 86c7597a7..8d8369c46 100644 --- a/apps/audits/models.py +++ b/apps/audits/models.py @@ -12,6 +12,7 @@ from orgs.utils import current_org from .const import ( OperateChoices, ActionChoices, + ActivityChoices, LoginTypeChoices, MFAChoices, LoginStatusChoices, @@ -20,6 +21,7 @@ from .const import ( __all__ = [ "FTPLog", "OperateLog", + "ActivityLog", "PasswordChangeLog", "UserLoginLog", ] @@ -59,7 +61,6 @@ class OperateLog(OrgModelMixin): remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True) datetime = models.DateTimeField(auto_now=True, verbose_name=_('Datetime'), db_index=True) diff = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True) - detail = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Detail')) def __str__(self): return "<{}> {} <{}>".format(self.user, self.action, self.resource) @@ -93,6 +94,34 @@ class OperateLog(OrgModelMixin): ordering = ('-datetime',) +class ActivityLog(OrgModelMixin): + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + type = models.CharField( + choices=ActivityChoices.choices, max_length=2, + null=True, default=None, verbose_name=_("Activity type"), + ) + resource_id = models.CharField( + max_length=36, blank=True, default='', + db_index=True, verbose_name=_("Resource") + ) + datetime = models.DateTimeField( + auto_now=True, verbose_name=_('Datetime'), db_index=True + ) + detail = models.TextField(default='', blank=True, verbose_name=_('Detail')) + detail_url = models.CharField( + max_length=256, default=None, null=True, verbose_name=_('Detail url') + ) + + class Meta: + verbose_name = _("Activity log") + ordering = ('-datetime',) + + def save(self, *args, **kwargs): + if current_org.is_root() and not self.org_id: + self.org_id = Organization.ROOT_ID + return super(ActivityLog, self).save(*args, **kwargs) + + class PasswordChangeLog(models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True) user = models.CharField(max_length=128, verbose_name=_("User")) diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index 6700c3b68..0eebce6f7 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -5,6 +5,7 @@ from rest_framework import serializers from audits.backends.db import OperateLogStore from common.serializers.fields import LabeledChoiceField +from common.utils import reverse from common.utils.timezone import as_current_tz from ops.models.job import JobAuditLog from ops.serializers.job import JobExecutionSerializer @@ -13,7 +14,7 @@ from . import models from .const import ( ActionChoices, OperateChoices, MFAChoices, LoginStatusChoices, - LoginTypeChoices, + LoginTypeChoices, ActivityChoices, ) @@ -107,17 +108,29 @@ class SessionAuditSerializer(serializers.ModelSerializer): class ActivitiesOperatorLogSerializer(serializers.Serializer): timestamp = serializers.SerializerMethodField() + detail_url = serializers.SerializerMethodField() content = serializers.SerializerMethodField() @staticmethod def get_timestamp(obj): - return as_current_tz(obj.datetime).strftime('%Y-%m-%d %H:%M:%S') + return as_current_tz(obj['datetime']).strftime('%Y-%m-%d %H:%M:%S') @staticmethod def get_content(obj): - action = obj.action.replace('_', ' ').capitalize() - if not obj.detail: - ctn = _('User {} {} this resource.').format(obj.user, _(action)) + if not obj['r_detail']: + action = obj['r_action'].replace('_', ' ').capitalize() + ctn = _('User {} {} this resource.').format(obj['r_user'], _(action)) else: - ctn = obj.detail + ctn = obj['r_detail'] return ctn + + @staticmethod + def get_detail_url(obj): + detail_url = obj['r_detail_url'] + if obj['r_type'] == ActivityChoices.operate_log: + detail_url = reverse( + view_name='audits:operate-log-detail', + kwargs={'pk': obj['id']}, + api_to_ui=True, is_audit=True + ) + return detail_url diff --git a/apps/audits/signal_handlers.py b/apps/audits/signal_handlers.py index 8cc396496..f6f75b430 100644 --- a/apps/audits/signal_handlers.py +++ b/apps/audits/signal_handlers.py @@ -18,6 +18,7 @@ from audits.handler import ( get_instance_current_with_cache_diff, cache_instance_before_data, create_or_update_operate_log, get_instance_dict_from_cache ) +from audits.models import ActivityLog from audits.utils import model_to_dict_for_operate_log as model_to_dict from authentication.signals import post_auth_failed, post_auth_success from authentication.utils import check_different_city_login_if_need @@ -34,6 +35,8 @@ from users.signals import post_user_change_password from . import models, serializers from .const import MODELS_NEED_RECORD, ActionChoices from .utils import write_login_log +from .signals import post_activity_log + logger = get_logger(__name__) sys_logger = get_syslogger(__name__) @@ -242,7 +245,7 @@ def get_login_backend(request): return backend_label -def generate_data(username, request, login_type=None): +def generate_data(username, request, login_type=None, user_id=None): user_agent = request.META.get('HTTP_USER_AGENT', '') login_ip = get_request_ip(request) or '0.0.0.0' @@ -255,6 +258,7 @@ def generate_data(username, request, login_type=None): backend = str(get_login_backend(request)) data = { + 'user_id': user_id, 'username': username, 'ip': login_ip, 'type': login_type, @@ -269,7 +273,9 @@ def generate_data(username, request, login_type=None): def on_user_auth_success(sender, user, request, login_type=None, **kwargs): logger.debug('User login success: {}'.format(user.username)) check_different_city_login_if_need(user, request) - data = generate_data(user.username, request, login_type=login_type) + data = generate_data( + user.username, request, login_type=login_type, user_id=user.id + ) request.session['login_time'] = data['datetime'].strftime("%Y-%m-%d %H:%M:%S") data.update({'mfa': int(user.mfa_enabled), 'status': True}) write_login_log(**data) @@ -286,8 +292,8 @@ def on_user_auth_failed(sender, username, request, reason='', **kwargs): @receiver(django_ready) def on_django_start_set_operate_log_monitor_models(sender, **kwargs): exclude_apps = { - 'django_cas_ng', 'captcha', 'admin', 'jms_oidc_rp', - 'django_celery_beat', 'contenttypes', 'sessions', 'auth' + 'django_cas_ng', 'captcha', 'admin', 'jms_oidc_rp', 'audits', + 'django_celery_beat', 'contenttypes', 'sessions', 'auth', } exclude_models = { 'UserPasswordHistory', 'ContentType', @@ -302,7 +308,6 @@ def on_django_start_set_operate_log_monitor_models(sender, **kwargs): 'PermedAsset', 'PermedAccount', 'MenuPermission', 'Permission', 'TicketSession', 'ApplyLoginTicket', 'ApplyCommandTicket', 'ApplyLoginAssetTicket', - 'FTPLog', 'OperateLog', 'PasswordChangeLog' } for i, app in enumerate(apps.get_models(), 1): app_name = app._meta.app_label @@ -312,3 +317,11 @@ def on_django_start_set_operate_log_monitor_models(sender, **kwargs): model_name.endswith('Execution'): continue MODELS_NEED_RECORD.add(model_name) + + +@receiver(post_activity_log) +def on_activity_log_trigger(sender, **kwargs): + ActivityLog.objects.create( + resource_id=kwargs['resource_id'], + detail=kwargs.get('detail'), detail_url=kwargs.get('detail_url') + ) diff --git a/apps/audits/signals.py b/apps/audits/signals.py new file mode 100644 index 000000000..229c63933 --- /dev/null +++ b/apps/audits/signals.py @@ -0,0 +1,6 @@ +from django.dispatch import Signal + + +post_activity_log = Signal( + providing_args=('resource_id', 'detail', 'detail_url') +) diff --git a/apps/audits/tasks.py b/apps/audits/tasks.py index 5b58c5fd2..33204760f 100644 --- a/apps/audits/tasks.py +++ b/apps/audits/tasks.py @@ -7,7 +7,7 @@ from celery import shared_task from ops.celery.decorator import ( register_as_period_task ) -from .models import UserLoginLog, OperateLog, FTPLog +from .models import UserLoginLog, OperateLog, FTPLog, ActivityLog from common.utils import get_log_keep_day @@ -25,6 +25,13 @@ def clean_operation_log_period(): OperateLog.objects.filter(datetime__lt=expired_day).delete() +def clean_activity_log_period(): + now = timezone.now() + days = get_log_keep_day('ACTIVITY_LOG_KEEP_DAYS') + expired_day = now - datetime.timedelta(days=days) + ActivityLog.objects.filter(datetime__lt=expired_day).delete() + + def clean_ftp_log_period(): now = timezone.now() days = get_log_keep_day('FTP_LOG_KEEP_DAYS') diff --git a/apps/audits/utils.py b/apps/audits/utils.py index 6f8f9e730..5d1fb5722 100644 --- a/apps/audits/utils.py +++ b/apps/audits/utils.py @@ -5,11 +5,14 @@ from itertools import chain from django.http import HttpResponse from django.db import models +from django.utils.translation import gettext_lazy as _ +from audits.const import ActivityChoices from settings.serializers import SettingsSerializer from common.utils import validate_ip, get_ip_city, get_logger from common.db import fields from .const import DEFAULT_CITY +from .signals import post_activity_log logger = get_logger(__name__) @@ -44,7 +47,19 @@ def write_login_log(*args, **kwargs): else: city = get_ip_city(ip) or DEFAULT_CITY kwargs.update({'ip': ip, 'city': city}) - UserLoginLog.objects.create(**kwargs) + user_id = kwargs.pop('user_id', None) + audit_log = UserLoginLog.objects.create(**kwargs) + + # 发送Activity信号 + if user_id is not None: + login_status = _('Success') if audit_log.status else _('Failed') + detail = _('User {} login into this service.[{}]').format( + audit_log.username, login_status + ) + post_activity_log.send( + sender=UserLoginLog, resource_id=user_id, detail=detail, + type=ActivityChoices.login_log + ) def get_resource_display(resource): diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 55b94c87f..a31a85ec0 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -512,6 +512,7 @@ class Config(dict): 'LOGIN_LOG_KEEP_DAYS': 200, 'TASK_LOG_KEEP_DAYS': 90, 'OPERATE_LOG_KEEP_DAYS': 200, + 'ACTIVITY_LOG_KEEP_DAYS': 200, 'FTP_LOG_KEEP_DAYS': 200, 'CLOUD_SYNC_TASK_EXECUTION_KEEP_DAYS': 30, diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index 7cb92915d..fa6bc398f 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -117,6 +117,7 @@ WS_LISTEN_PORT = CONFIG.WS_LISTEN_PORT LOGIN_LOG_KEEP_DAYS = CONFIG.LOGIN_LOG_KEEP_DAYS TASK_LOG_KEEP_DAYS = CONFIG.TASK_LOG_KEEP_DAYS OPERATE_LOG_KEEP_DAYS = CONFIG.OPERATE_LOG_KEEP_DAYS +ACTIVITY_LOG_KEEP_DAYS = CONFIG.ACTIVITY_LOG_KEEP_DAYS FTP_LOG_KEEP_DAYS = CONFIG.FTP_LOG_KEEP_DAYS ORG_CHANGE_TO_URL = CONFIG.ORG_CHANGE_TO_URL WINDOWS_SKIP_ALL_MANUAL_PASSWORD = CONFIG.WINDOWS_SKIP_ALL_MANUAL_PASSWORD diff --git a/apps/rbac/const.py b/apps/rbac/const.py index 192a0fe18..e345a52e4 100644 --- a/apps/rbac/const.py +++ b/apps/rbac/const.py @@ -79,6 +79,7 @@ exclude_permissions = ( ('orgs', 'organizationmember', '*', '*'), ('settings', 'setting', 'add,change,delete', 'setting'), ('audits', 'operatelog', 'add,delete,change', 'operatelog'), + ('audits', 'activitylog', 'add,delete,change', 'activitylog'), ('audits', 'passwordchangelog', 'add,change,delete', 'passwordchangelog'), ('audits', 'userloginlog', 'add,change,delete,change', 'userloginlog'), ('audits', 'ftplog', 'change,delete', 'ftplog'), diff --git a/apps/settings/serializers/cleaning.py b/apps/settings/serializers/cleaning.py index a180a9ac5..cc6fb00bf 100644 --- a/apps/settings/serializers/cleaning.py +++ b/apps/settings/serializers/cleaning.py @@ -31,4 +31,7 @@ class CleaningSerializer(serializers.Serializer): min_value=1, max_value=99999, required=True, label=_('Session keep duration'), help_text=_('Unit: days, Session, record, command will be delete if more than duration, only in database') ) - + ACTIVITY_LOG_KEEP_DAYS = serializers.IntegerField( + min_value=1, max_value=9999, + label=_("Activity log keep days"), help_text=_("Unit: day") + ) diff --git a/apps/terminal/api/session/session.py b/apps/terminal/api/session/session.py index 6f4f2863f..1284ae8ed 100644 --- a/apps/terminal/api/session/session.py +++ b/apps/terminal/api/session/session.py @@ -15,6 +15,8 @@ from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from audits.signals import post_activity_log +from audits.const import ActivityChoices from common.const.http import GET from common.drf.filters import DatetimeRangeFilter from common.drf.renders import PassthroughRenderer @@ -120,10 +122,30 @@ class SessionViewSet(OrgBulkModelViewSet): queryset = queryset.select_for_update() return queryset + @staticmethod + def send_activity(serializer): + # 发送Activity信号 + data = serializer.validated_data + user, asset_id = data['user'], data["asset_id"] + account, login_from = data['account'], data["login_from"] + login_from = Session(login_from=login_from).get_login_from_display() + detail = _( + '{} used account[{}], login method[{}] login the asset.' + ).format( + user, account, login_from + ) + post_activity_log.send( + sender=Session, resource_id=asset_id, detail=detail, + type=ActivityChoices.session_log + ) + def perform_create(self, serializer): if hasattr(self.request.user, 'terminal'): serializer.validated_data["terminal"] = self.request.user.terminal - return super().perform_create(serializer) + + resp = super().perform_create(serializer) + self.send_activity(serializer) + return resp class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet): From cb362b2fe4daa8faaae34cebc8c9c09afaf60009 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Tue, 7 Feb 2023 08:58:14 +0800 Subject: [PATCH 02/44] =?UTF-8?q?perf:=20=E7=94=9F=E6=88=90=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0021_auto_20230207_0857.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 apps/audits/migrations/0021_auto_20230207_0857.py diff --git a/apps/audits/migrations/0021_auto_20230207_0857.py b/apps/audits/migrations/0021_auto_20230207_0857.py new file mode 100644 index 000000000..9df98ace5 --- /dev/null +++ b/apps/audits/migrations/0021_auto_20230207_0857.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.16 on 2023-02-07 00:57 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('audits', '0020_auto_20230117_1004'), + ] + + operations = [ + migrations.CreateModel( + name='ActivityLog', + fields=[ + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('type', models.CharField(choices=[('O', 'Operate log'), ('S', 'Session log'), ('L', 'Login log'), ('T', 'Task')], default=None, max_length=2, null=True, verbose_name='Activity type')), + ('resource_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Resource')), + ('datetime', models.DateTimeField(auto_now=True, db_index=True, verbose_name='Datetime')), + ('detail', models.TextField(blank=True, default='', verbose_name='Detail')), + ('detail_url', models.CharField(default=None, max_length=256, null=True, verbose_name='Detail url')), + ], + options={ + 'verbose_name': 'Activity log', + 'ordering': ('-datetime',), + }, + ), + ] From a1300e288634ed73075d57dc0accb7f758881cc9 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Tue, 7 Feb 2023 09:49:17 +0800 Subject: [PATCH 03/44] =?UTF-8?q?perf:=20=E5=A2=9E=E5=8A=A0=E7=99=BB?= =?UTF-8?q?=E5=BD=95=E6=97=A5=E5=BF=97=E8=AF=A6=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/audits/api.py | 4 ++-- apps/audits/signal_handlers.py | 4 ++++ apps/audits/utils.py | 9 +++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/audits/api.py b/apps/audits/api.py index 0fcde1396..06eae0fa1 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -49,8 +49,8 @@ class UserLoginCommonMixin: date_range_filter_fields = [ ('datetime', ('date_from', 'date_to')) ] - filterset_fields = ['username', 'ip', 'city', 'type', 'status', 'mfa'] - search_fields = ['username', 'ip', 'city'] + filterset_fields = ['id', 'username', 'ip', 'city', 'type', 'status', 'mfa'] + search_fields = ['id', 'username', 'ip', 'city'] class UserLoginLogViewSet(UserLoginCommonMixin, ListModelMixin, JMSGenericViewSet): diff --git a/apps/audits/signal_handlers.py b/apps/audits/signal_handlers.py index 7710273ce..2b76fac01 100644 --- a/apps/audits/signal_handlers.py +++ b/apps/audits/signal_handlers.py @@ -259,6 +259,10 @@ def generate_data(username, request, login_type=None, user_id=None): with translation.override('en'): backend = str(get_login_backend(request)) + if user_id is None: + user = User.objects.filter(username=username).first() + user_id = getattr(user, 'id', None) + data = { 'user_id': user_id, 'username': username, diff --git a/apps/audits/utils.py b/apps/audits/utils.py index d5cd4ff60..851ddae3b 100644 --- a/apps/audits/utils.py +++ b/apps/audits/utils.py @@ -6,7 +6,7 @@ from django.db import models from django.http import HttpResponse from django.utils.translation import gettext_lazy as _ -from common.utils import validate_ip, get_ip_city, get_logger +from common.utils import validate_ip, get_ip_city, get_logger, reverse from audits.const import ActivityChoices from settings.serializers import SettingsSerializer from .const import DEFAULT_CITY @@ -53,8 +53,13 @@ def write_login_log(*args, **kwargs): detail = _('User {} login into this service.[{}]').format( audit_log.username, login_status ) + detail_url = '%s?id=%s' % ( + reverse('api-audits:login-log-list', api_to_ui=True, is_audit=True,), + audit_log.id + ) post_activity_log.send( - sender=UserLoginLog, resource_id=user_id, detail=detail, + sender=UserLoginLog, resource_id=user_id, + detail=detail, detail_url=detail_url, type=ActivityChoices.login_log ) From fb285adcce4c0e96f3320ca947f66db144d8b374 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Wed, 8 Feb 2023 11:32:05 +0800 Subject: [PATCH 04/44] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/automations/base/manager.py | 21 +- apps/audits/api.py | 16 +- .../migrations/0021_auto_20230207_0857.py | 2 +- apps/audits/models.py | 4 +- apps/audits/serializers.py | 27 +- apps/audits/signal_handlers.py | 333 ------------------ apps/audits/signal_handlers/__init__.py | 4 + apps/audits/signal_handlers/activity_log.py | 190 ++++++++++ apps/audits/signal_handlers/login_log.py | 96 +++++ apps/audits/signal_handlers/operate_log.py | 180 ++++++++++ apps/audits/signal_handlers/other.py | 68 ++++ apps/audits/signals.py | 6 - apps/audits/utils.py | 24 +- apps/terminal/api/session/session.py | 24 +- 14 files changed, 573 insertions(+), 422 deletions(-) delete mode 100644 apps/audits/signal_handlers.py create mode 100644 apps/audits/signal_handlers/__init__.py create mode 100644 apps/audits/signal_handlers/activity_log.py create mode 100644 apps/audits/signal_handlers/login_log.py create mode 100644 apps/audits/signal_handlers/operate_log.py create mode 100644 apps/audits/signal_handlers/other.py delete mode 100644 apps/audits/signals.py diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 3e4cc95d9..2254b1683 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -10,9 +10,6 @@ from django.utils import timezone from django.utils.translation import gettext as _ from assets.automations.methods import platform_automation_methods -from audits.signals import post_activity_log -from audits.const import ActivityChoices -from common.utils import reverse from common.utils import get_logger, lazyproperty from common.utils import ssh_pubkey_gen, ssh_key_string_to_obj from ops.ansible import JMSInventory, PlaybookRunner, DefaultCallback @@ -150,21 +147,6 @@ class BasePlaybookManager: yaml.safe_dump(plays, f) return sub_playbook_path - def send_activity(self, assets, **kwargs): - user = kwargs.pop('user', _('Unknown')) - task_type = self.method_type().label - detail = 'User %s performs a task(%s) for this resource.' % ( - user, task_type - ) - detail_url = reverse( - 'ops:celery-task-log', kwargs={'pk': self.execution.id} - ) - for a in assets: - post_activity_log.send( - sender=self, resource_id=a.id, detail=detail, - detail_url=detail_url, type=ActivityChoices.task - ) - def get_runners(self, **kwargs): runners = [] for platform, assets in self.get_assets_group_by_platform().items(): @@ -184,7 +166,6 @@ class BasePlaybookManager: callback=PlaybookCallback(), ) runners.append(runer) - self.send_activity(assets, **kwargs) return runners def on_host_success(self, host, result): @@ -214,7 +195,7 @@ class BasePlaybookManager: pass def run(self, *args, **kwargs): - runners = self.get_runners(user=kwargs.pop('user')) + runners = self.get_runners(user=kwargs.pop('user', None)) if len(runners) > 1: print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) else: diff --git a/apps/audits/api.py b/apps/audits/api.py index 06eae0fa1..663527e1a 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -20,7 +20,7 @@ from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog, Activit from .serializers import FTPLogSerializer, UserLoginLogSerializer, JobAuditLogSerializer from .serializers import ( OperateLogSerializer, OperateLogActionDetailSerializer, - PasswordChangeLogSerializer, ActivitiesOperatorLogSerializer, + PasswordChangeLogSerializer, ActivityOperatorLogSerializer, ) @@ -79,7 +79,7 @@ class MyLoginLogAPIView(UserLoginCommonMixin, generics.ListAPIView): class ResourceActivityAPIView(generics.ListAPIView): - serializer_class = ActivitiesOperatorLogSerializer + serializer_class = ActivityOperatorLogSerializer rbac_perms = { 'GET': 'audits.view_activitylog', } @@ -88,8 +88,7 @@ class ResourceActivityAPIView(generics.ListAPIView): def get_operate_log_qs(fields, limit=30, **filters): queryset = OperateLog.objects.filter(**filters).annotate( r_type=Value(ActivityChoices.operate_log, CharField()), - r_detail_url=Value(None, CharField()), - r_detail=Value(None, CharField()), + r_detail_id=F('id'), r_detail=Value(None, CharField()), r_user=F('user'), r_action=F('action'), ).values(*fields)[:limit] return queryset @@ -97,8 +96,8 @@ class ResourceActivityAPIView(generics.ListAPIView): @staticmethod def get_activity_log_qs(fields, limit=30, **filters): queryset = ActivityLog.objects.filter(**filters).annotate( - r_type=F('type'), r_detail_url=F('detail_url'), r_detail=F('detail'), - r_user=Value(None, CharField()), + r_type=F('type'), r_detail_id=F('detail_id'), + r_detail=F('detail'), r_user=Value(None, CharField()), r_action=Value(None, CharField()), ).values(*fields)[:limit] return queryset @@ -106,7 +105,10 @@ class ResourceActivityAPIView(generics.ListAPIView): def get_queryset(self): limit = 30 resource_id = self.request.query_params.get('resource_id') - fields = ('id', 'datetime', 'r_detail', 'r_detail_url', 'r_user', 'r_action', 'r_type') + fields = ( + 'id', 'datetime', 'r_detail', 'r_detail_id', + 'r_user', 'r_action', 'r_type' + ) with tmp_to_root_org(): qs1 = self.get_operate_log_qs(fields, resource_id=resource_id) qs2 = self.get_activity_log_qs(fields, resource_id=resource_id) diff --git a/apps/audits/migrations/0021_auto_20230207_0857.py b/apps/audits/migrations/0021_auto_20230207_0857.py index 9df98ace5..e9e8b939e 100644 --- a/apps/audits/migrations/0021_auto_20230207_0857.py +++ b/apps/audits/migrations/0021_auto_20230207_0857.py @@ -20,7 +20,7 @@ class Migration(migrations.Migration): ('resource_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Resource')), ('datetime', models.DateTimeField(auto_now=True, db_index=True, verbose_name='Datetime')), ('detail', models.TextField(blank=True, default='', verbose_name='Detail')), - ('detail_url', models.CharField(default=None, max_length=256, null=True, verbose_name='Detail url')), + ('detail_id', models.CharField(default=None, max_length=36, null=True, verbose_name='Detail ID')), ], options={ 'verbose_name': 'Activity log', diff --git a/apps/audits/models.py b/apps/audits/models.py index 8ae162c79..29a67911f 100644 --- a/apps/audits/models.py +++ b/apps/audits/models.py @@ -108,8 +108,8 @@ class ActivityLog(OrgModelMixin): auto_now=True, verbose_name=_('Datetime'), db_index=True ) detail = models.TextField(default='', blank=True, verbose_name=_('Detail')) - detail_url = models.CharField( - max_length=256, default=None, null=True, verbose_name=_('Detail url') + detail_id = models.CharField( + max_length=36, default=None, null=True, verbose_name=_('Detail ID') ) class Meta: diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index 0eebce6f7..ebe7394d9 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -106,7 +106,7 @@ class SessionAuditSerializer(serializers.ModelSerializer): fields = "__all__" -class ActivitiesOperatorLogSerializer(serializers.Serializer): +class ActivityOperatorLogSerializer(serializers.Serializer): timestamp = serializers.SerializerMethodField() detail_url = serializers.SerializerMethodField() content = serializers.SerializerMethodField() @@ -126,11 +126,22 @@ class ActivitiesOperatorLogSerializer(serializers.Serializer): @staticmethod def get_detail_url(obj): - detail_url = obj['r_detail_url'] - if obj['r_type'] == ActivityChoices.operate_log: - detail_url = reverse( - view_name='audits:operate-log-detail', - kwargs={'pk': obj['id']}, - api_to_ui=True, is_audit=True - ) + detail_url = '' + detail_id, obj_type = obj['r_detail_id'], obj['r_type'] + if detail_id: + if obj_type == ActivityChoices.operate_log: + detail_url = reverse( + view_name='audits:operate-log-detail', + kwargs={'pk': obj['id']}, + api_to_ui=True, is_audit=True + ) + elif obj_type == ActivityChoices.task: + detail_url = reverse( + 'ops:celery-task-log', kwargs={'pk': detail_id} + ) + elif obj_type == ActivityChoices.login_log: + detail_url = '%s?id=%s' % ( + reverse('api-audits:login-log-list', api_to_ui=True, is_audit=True), + detail_id + ) return detail_url diff --git a/apps/audits/signal_handlers.py b/apps/audits/signal_handlers.py deleted file mode 100644 index 2b76fac01..000000000 --- a/apps/audits/signal_handlers.py +++ /dev/null @@ -1,333 +0,0 @@ -# -*- coding: utf-8 -*- -# -import uuid - -from django.apps import apps -from django.conf import settings -from django.contrib.auth import BACKEND_SESSION_KEY -from django.db import transaction -from django.db.models.signals import post_save, pre_save, m2m_changed, pre_delete -from django.dispatch import receiver -from django.utils import timezone, translation -from django.utils.functional import LazyObject -from django.utils.translation import ugettext_lazy as _ -from rest_framework.renderers import JSONRenderer -from rest_framework.request import Request - -from audits.handler import ( - get_instance_current_with_cache_diff, cache_instance_before_data, - create_or_update_operate_log, get_instance_dict_from_cache -) -from audits.models import ActivityLog -from audits.utils import model_to_dict_for_operate_log as model_to_dict -from authentication.signals import post_auth_failed, post_auth_success -from authentication.utils import check_different_city_login_if_need -from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR, SKIP_SIGNAL -from common.signals import django_ready -from common.utils import get_request_ip, get_logger, get_syslogger -from common.utils.encode import data_to_json -from jumpserver.utils import current_request -from terminal.models import Session, Command -from terminal.serializers import SessionSerializer, SessionCommandSerializer -from users.models import User -from users.signals import post_user_change_password -from . import models, serializers -from .const import MODELS_NEED_RECORD, ActionChoices -from .utils import write_login_log -from .signals import post_activity_log - - -logger = get_logger(__name__) -sys_logger = get_syslogger(__name__) -json_render = JSONRenderer() - - -class AuthBackendLabelMapping(LazyObject): - @staticmethod - def get_login_backends(): - backend_label_mapping = {} - for source, backends in User.SOURCE_BACKEND_MAPPING.items(): - for backend in backends: - backend_label_mapping[backend] = source.label - backend_label_mapping[settings.AUTH_BACKEND_PUBKEY] = _("SSH Key") - backend_label_mapping[settings.AUTH_BACKEND_MODEL] = _("Password") - backend_label_mapping[settings.AUTH_BACKEND_SSO] = _("SSO") - backend_label_mapping[settings.AUTH_BACKEND_AUTH_TOKEN] = _("Auth Token") - backend_label_mapping[settings.AUTH_BACKEND_WECOM] = _("WeCom") - backend_label_mapping[settings.AUTH_BACKEND_FEISHU] = _("FeiShu") - backend_label_mapping[settings.AUTH_BACKEND_DINGTALK] = _("DingTalk") - backend_label_mapping[settings.AUTH_BACKEND_TEMP_TOKEN] = _("Temporary token") - return backend_label_mapping - - def _setup(self): - self._wrapped = self.get_login_backends() - - -AUTH_BACKEND_LABEL_MAPPING = AuthBackendLabelMapping() - -M2M_ACTION = { - POST_ADD: ActionChoices.create, - POST_REMOVE: ActionChoices.delete, - POST_CLEAR: ActionChoices.delete, -} - - -@receiver(m2m_changed) -def on_m2m_changed(sender, action, instance, reverse, model, pk_set, **kwargs): - if action not in M2M_ACTION: - return - if not instance: - return - - resource_type = instance._meta.verbose_name - current_instance = model_to_dict(instance, include_model_fields=False) - - instance_id = current_instance.get('id') - log_id, before_instance = get_instance_dict_from_cache(instance_id) - - field_name = str(model._meta.verbose_name) - objs = model.objects.filter(pk__in=pk_set) - objs_display = [str(o) for o in objs] - action = M2M_ACTION[action] - changed_field = current_instance.get(field_name, []) - - after, before, before_value = None, None, None - if action == ActionChoices.create: - before_value = list(set(changed_field) - set(objs_display)) - elif action == ActionChoices.delete: - before_value = list( - set(changed_field).symmetric_difference(set(objs_display)) - ) - - if changed_field: - after = {field_name: changed_field} - if before_value: - before = {field_name: before_value} - - if sorted(str(before)) == sorted(str(after)): - return - - create_or_update_operate_log( - ActionChoices.update, resource_type, - resource=instance, log_id=log_id, before=before, after=after - ) - - -def signal_of_operate_log_whether_continue(sender, instance, created, update_fields=None): - condition = True - if not instance: - condition = False - if instance and getattr(instance, SKIP_SIGNAL, False): - condition = False - # 终端模型的 create 事件由系统产生，不记录 - if instance._meta.object_name == 'Terminal' and created: - condition = False - # last_login 改变是最后登录日期, 每次登录都会改变 - if instance._meta.object_name == 'User' and update_fields and 'last_login' in update_fields: - condition = False - # 不在记录白名单中，跳过 - if sender._meta.object_name not in MODELS_NEED_RECORD: - condition = False - return condition - - -@receiver(pre_save) -def on_object_pre_create_or_update(sender, instance=None, raw=False, using=None, update_fields=None, **kwargs): - ok = signal_of_operate_log_whether_continue( - sender, instance, False, update_fields - ) - if not ok: - return - - # users.PrivateToken Model 没有 id 有 pk字段 - instance_id = getattr(instance, 'id', getattr(instance, 'pk', None)) - instance_before_data = {'id': instance_id} - raw_instance = type(instance).objects.filter(pk=instance_id).first() - - if raw_instance: - instance_before_data = model_to_dict(raw_instance) - operate_log_id = str(uuid.uuid4()) - instance_before_data['operate_log_id'] = operate_log_id - setattr(instance, 'operate_log_id', operate_log_id) - cache_instance_before_data(instance_before_data) - - -@receiver(post_save) -def on_object_created_or_update(sender, instance=None, created=False, update_fields=None, **kwargs): - ok = signal_of_operate_log_whether_continue( - sender, instance, created, update_fields - ) - if not ok: - return - - log_id, before, after = None, None, None - if created: - action = models.ActionChoices.create - after = model_to_dict(instance) - log_id = getattr(instance, 'operate_log_id', None) - else: - action = ActionChoices.update - current_instance = model_to_dict(instance) - log_id, before, after = get_instance_current_with_cache_diff(current_instance) - - resource_type = sender._meta.verbose_name - object_name = sender._meta.object_name - create_or_update_operate_log( - action, resource_type, resource=instance, log_id=log_id, - before=before, after=after, object_name=object_name - ) - - -@receiver(pre_delete) -def on_object_delete(sender, instance=None, **kwargs): - ok = signal_of_operate_log_whether_continue(sender, instance, False) - if not ok: - return - - resource_type = sender._meta.verbose_name - create_or_update_operate_log( - ActionChoices.delete, resource_type, - resource=instance, before=model_to_dict(instance) - ) - - -@receiver(post_user_change_password, sender=User) -def on_user_change_password(sender, user=None, **kwargs): - if not current_request: - remote_addr = '127.0.0.1' - change_by = 'System' - else: - remote_addr = get_request_ip(current_request) - if not current_request.user.is_authenticated: - change_by = str(user) - else: - change_by = str(current_request.user) - with transaction.atomic(): - models.PasswordChangeLog.objects.create( - user=str(user), change_by=change_by, - remote_addr=remote_addr, - ) - - -def on_audits_log_create(sender, instance=None, **kwargs): - if sender == models.UserLoginLog: - category = "login_log" - serializer_cls = serializers.UserLoginLogSerializer - elif sender == models.FTPLog: - category = "ftp_log" - serializer_cls = serializers.FTPLogSerializer - elif sender == models.OperateLog: - category = "operation_log" - serializer_cls = serializers.OperateLogSerializer - elif sender == models.PasswordChangeLog: - category = "password_change_log" - serializer_cls = serializers.PasswordChangeLogSerializer - elif sender == Session: - category = "host_session_log" - serializer_cls = SessionSerializer - elif sender == Command: - category = "session_command_log" - serializer_cls = SessionCommandSerializer - else: - return - - serializer = serializer_cls(instance) - data = data_to_json(serializer.data, indent=None) - msg = "{} - {}".format(category, data) - sys_logger.info(msg) - - -def get_login_backend(request): - backend = request.session.get('auth_backend', '') or \ - request.session.get(BACKEND_SESSION_KEY, '') - - backend_label = AUTH_BACKEND_LABEL_MAPPING.get(backend, None) - if backend_label is None: - backend_label = '' - return backend_label - - -def generate_data(username, request, login_type=None, user_id=None): - user_agent = request.META.get('HTTP_USER_AGENT', '') - login_ip = get_request_ip(request) or '0.0.0.0' - - if login_type is None and isinstance(request, Request): - login_type = request.META.get('HTTP_X_JMS_LOGIN_TYPE', 'U') - if login_type is None: - login_type = 'W' - - with translation.override('en'): - backend = str(get_login_backend(request)) - - if user_id is None: - user = User.objects.filter(username=username).first() - user_id = getattr(user, 'id', None) - - data = { - 'user_id': user_id, - 'username': username, - 'ip': login_ip, - 'type': login_type, - 'user_agent': user_agent[0:254], - 'datetime': timezone.now(), - 'backend': backend, - } - return data - - -@receiver(post_auth_success) -def on_user_auth_success(sender, user, request, login_type=None, **kwargs): - logger.debug('User login success: {}'.format(user.username)) - check_different_city_login_if_need(user, request) - data = generate_data( - user.username, request, login_type=login_type, user_id=user.id - ) - request.session['login_time'] = data['datetime'].strftime("%Y-%m-%d %H:%M:%S") - data.update({'mfa': int(user.mfa_enabled), 'status': True}) - write_login_log(**data) - - -@receiver(post_auth_failed) -def on_user_auth_failed(sender, username, request, reason='', **kwargs): - logger.debug('User login failed: {}'.format(username)) - data = generate_data(username, request) - data.update({'reason': reason[:128], 'status': False}) - write_login_log(**data) - - -@receiver(django_ready) -def on_django_start_set_operate_log_monitor_models(sender, **kwargs): - exclude_apps = { - 'django_cas_ng', 'captcha', 'admin', 'jms_oidc_rp', 'audits', - 'django_celery_beat', 'contenttypes', 'sessions', 'auth', - } - exclude_models = { - 'UserPasswordHistory', 'ContentType', - 'MessageContent', 'SiteMessage', - 'PlatformAutomation', 'PlatformProtocol', 'Protocol', - 'HistoricalAccount', 'GatheredUser', 'ApprovalRule', - 'BaseAutomation', 'CeleryTask', 'Command', 'JobAuditLog', - 'ConnectionToken', 'SessionJoinRecord', - 'HistoricalJob', 'Status', 'TicketStep', 'Ticket', - 'UserAssetGrantedTreeNodeRelation', 'TicketAssignee', - 'SuperTicket', 'SuperConnectionToken', 'PermNode', - 'PermedAsset', 'PermedAccount', 'MenuPermission', - 'Permission', 'TicketSession', 'ApplyLoginTicket', - 'ApplyCommandTicket', 'ApplyLoginAssetTicket', - } - for i, app in enumerate(apps.get_models(), 1): - app_name = app._meta.app_label - model_name = app._meta.object_name - if app_name in exclude_apps or \ - model_name in exclude_models or \ - model_name.endswith('Execution'): - continue - MODELS_NEED_RECORD.add(model_name) - - -@receiver(post_activity_log) -def on_activity_log_trigger(sender, **kwargs): - ActivityLog.objects.create( - resource_id=kwargs['resource_id'], - detail=kwargs.get('detail'), detail_url=kwargs.get('detail_url') - ) diff --git a/apps/audits/signal_handlers/__init__.py b/apps/audits/signal_handlers/__init__.py new file mode 100644 index 000000000..82e0f4917 --- /dev/null +++ b/apps/audits/signal_handlers/__init__.py @@ -0,0 +1,4 @@ +from .activity_log import * +from .login_log import * +from .operate_log import * +from .other import * diff --git a/apps/audits/signal_handlers/activity_log.py b/apps/audits/signal_handlers/activity_log.py new file mode 100644 index 000000000..8a09935ef --- /dev/null +++ b/apps/audits/signal_handlers/activity_log.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +from celery import signals +from django.db.models.signals import post_save +from django.utils.translation import ugettext_lazy as _ + +from audits.models import ActivityLog +from assets.models import Asset, Node +from accounts.const import AutomationTypes +from accounts.models import AccountBackupAutomation +from common.utils import get_object_or_none +from ops.celery import app +from orgs.utils import tmp_to_root_org +from terminal.models import Session +from users.models import User +from jumpserver.utils import current_request + +from ..const import ActivityChoices + + +class ActivityLogHandler(object): + + @staticmethod + def _func_accounts_execute_automation(*args, **kwargs): + asset_ids = [] + pid, tp = kwargs.get('pid'), kwargs.get('tp') + model = AutomationTypes.get_type_model(tp) + task_type_label = tp.label + with tmp_to_root_org(): + instance = get_object_or_none(model, pk=pid) + if instance is not None: + asset_ids = instance.get_all_assets().values_list('id', flat=True) + return task_type_label, asset_ids + + @staticmethod + def _func_accounts_push_accounts_to_assets(*args, **kwargs): + return '', args[0][1] + + @staticmethod + def _func_accounts_execute_account_backup_plan(*args, **kwargs): + asset_ids, pid = [], kwargs.get('pid') + with tmp_to_root_org(): + instance = get_object_or_none(AccountBackupAutomation, pk=pid) + if instance is not None: + asset_ids = Asset.objects.filter( + platform__type__in=instance.types + ).values_list('id', flat=True) + return '', asset_ids + + @staticmethod + def _func_assets_verify_accounts_connectivity(*args, **kwargs): + return '', args[0][1] + + @staticmethod + def _func_accounts_verify_accounts_connectivity(*args, **kwargs): + return '', args[0][1] + + @staticmethod + def _func_assets_test_assets_connectivity_manual(*args, **kwargs): + return '', args[0][0] + + @staticmethod + def _func_assets_test_node_assets_connectivity_manual(*args, **kwargs): + asset_ids = [] + node = get_object_or_none(Node, pk=args[0][0]) + if node is not None: + asset_ids = node.get_all_assets().values_list('id', flat=True) + return '', asset_ids + + @staticmethod + def _func_assets_update_assets_hardware_info_manual(*args, **kwargs): + return '', args[0][0] + + @staticmethod + def _func_assets_update_node_assets_hardware_info_manual(*args, **kwargs): + asset_ids = [] + node = get_object_or_none(Node, pk=args[0][0]) + if node is not None: + asset_ids = node.get_all_assets().values_list('id', flat=True) + return '', asset_ids + + def get_celery_task_info(self, task_name, *args, **kwargs): + task_display, resource_ids = self.get_info_by_task_name( + task_name, *args, **kwargs + ) + return task_display, resource_ids + + @staticmethod + def get_task_display(task_name, **kwargs): + task = app.tasks.get(task_name) + return getattr(task, 'verbose_name', _('Unknown')) + + def get_info_by_task_name(self, task_name, *args, **kwargs): + resource_ids = [] + task_name_list = str(task_name).split('.') + if len(task_name_list) < 2: + return '', resource_ids + + task_display = self.get_task_display(task_name) + model, name = task_name_list[0], task_name_list[-1] + func_name = '_func_%s_%s' % (model, name) + handle_func = getattr(self, func_name, None) + if handle_func is not None: + task_type, resource_ids = handle_func(*args, **kwargs) + if task_type: + task_display = '%s-%s' % (task_display, task_type) + return task_display, resource_ids + + @staticmethod + def session_for_activity(obj): + detail = _( + '{} used account[{}], login method[{}] login the asset.' + ).format( + obj.user, obj.account, obj.login_from_display + ) + return obj.asset_id, detail, ActivityChoices.session_log + + @staticmethod + def login_log_for_activity(obj): + login_status = _('Success') if obj.status else _('Failed') + detail = _('User {} login into this service.[{}]').format( + obj.username, login_status + ) + user_id = User.objects.filter(username=obj.username).values('id').first() + return user_id['id'], detail, ActivityChoices.login_log + + +activity_handler = ActivityLogHandler() + + +@signals.before_task_publish.connect +def before_task_publish_for_activity_log(headers=None, **kwargs): + task_id, task_name = headers.get('id'), headers.get('task') + args, kwargs = kwargs['body'][:2] + task_display, resource_ids = activity_handler.get_celery_task_info( + task_name, args, **kwargs + ) + activities = [] + detail = _('User %s performs a task(%s) for this resource.') % ( + getattr(current_request, 'user', None), task_display + ) + for resource_id in resource_ids: + activities.append( + ActivityLog( + resource_id=resource_id, type=ActivityChoices.task, detail=detail + ) + ) + ActivityLog.objects.bulk_create(activities) + + activity_info = { + 'activity_ids': [a.id for a in activities] + } + kwargs['activity_info'] = activity_info + + +@signals.task_prerun.connect +def on_celery_task_pre_run_for_activity_log(task_id='', **kwargs): + activity_info = kwargs['kwargs'].pop('activity_info', None) + if activity_info is None: + return + + activities = [] + for activity_id in activity_info['activity_ids']: + activities.append( + ActivityLog(id=activity_id, detail_id=task_id) + ) + ActivityLog.objects.bulk_update(activities, ('detail_id', )) + + +@post_save.connect +def on_object_created( + sender, instance=None, created=False, update_fields=None, **kwargs +): + handler_mapping = { + 'Session': activity_handler.session_for_activity, + 'UserLoginLog': activity_handler.login_log_for_activity + } + model_name = sender._meta.object_name + if not created or model_name not in handler_mapping.keys(): + return + + resource_id, detail, a_type = handler_mapping[model_name](instance) + + ActivityLog.objects.create( + resource_id=resource_id, type=a_type, + detail=detail, detail_id=instance.id + ) + + + diff --git a/apps/audits/signal_handlers/login_log.py b/apps/audits/signal_handlers/login_log.py new file mode 100644 index 000000000..34e8665ac --- /dev/null +++ b/apps/audits/signal_handlers/login_log.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# +from django.utils.functional import LazyObject +from django.utils.translation import ugettext_lazy as _ +from django.conf import settings +from django.contrib.auth import BACKEND_SESSION_KEY +from django.dispatch import receiver +from django.utils import timezone, translation +from rest_framework.request import Request + +from authentication.signals import post_auth_failed, post_auth_success +from authentication.utils import check_different_city_login_if_need +from common.utils import get_request_ip, get_logger +from users.models import User + +from ..utils import write_login_log + + +logger = get_logger(__name__) + + +class AuthBackendLabelMapping(LazyObject): + @staticmethod + def get_login_backends(): + backend_label_mapping = {} + for source, backends in User.SOURCE_BACKEND_MAPPING.items(): + for backend in backends: + backend_label_mapping[backend] = source.label + backend_label_mapping[settings.AUTH_BACKEND_PUBKEY] = _("SSH Key") + backend_label_mapping[settings.AUTH_BACKEND_MODEL] = _("Password") + backend_label_mapping[settings.AUTH_BACKEND_SSO] = _("SSO") + backend_label_mapping[settings.AUTH_BACKEND_AUTH_TOKEN] = _("Auth Token") + backend_label_mapping[settings.AUTH_BACKEND_WECOM] = _("WeCom") + backend_label_mapping[settings.AUTH_BACKEND_FEISHU] = _("FeiShu") + backend_label_mapping[settings.AUTH_BACKEND_DINGTALK] = _("DingTalk") + backend_label_mapping[settings.AUTH_BACKEND_TEMP_TOKEN] = _("Temporary token") + return backend_label_mapping + + def _setup(self): + self._wrapped = self.get_login_backends() + + +AUTH_BACKEND_LABEL_MAPPING = AuthBackendLabelMapping() + + +def get_login_backend(request): + backend = request.session.get('auth_backend', '') or \ + request.session.get(BACKEND_SESSION_KEY, '') + + backend_label = AUTH_BACKEND_LABEL_MAPPING.get(backend, None) + if backend_label is None: + backend_label = '' + return backend_label + + +def generate_data(username, request, login_type=None): + user_agent = request.META.get('HTTP_USER_AGENT', '') + login_ip = get_request_ip(request) or '0.0.0.0' + + if login_type is None and isinstance(request, Request): + login_type = request.META.get('HTTP_X_JMS_LOGIN_TYPE', 'U') + if login_type is None: + login_type = 'W' + + with translation.override('en'): + backend = str(get_login_backend(request)) + + data = { + 'username': username, + 'ip': login_ip, + 'type': login_type, + 'user_agent': user_agent[0:254], + 'datetime': timezone.now(), + 'backend': backend, + } + return data + + +@receiver(post_auth_success) +def on_user_auth_success(sender, user, request, login_type=None, **kwargs): + logger.debug('User login success: {}'.format(user.username)) + check_different_city_login_if_need(user, request) + data = generate_data( + user.username, request, login_type=login_type + ) + request.session['login_time'] = data['datetime'].strftime("%Y-%m-%d %H:%M:%S") + data.update({'mfa': int(user.mfa_enabled), 'status': True}) + write_login_log(**data) + + +@receiver(post_auth_failed) +def on_user_auth_failed(sender, username, request, reason='', **kwargs): + logger.debug('User login failed: {}'.format(username)) + data = generate_data(username, request) + data.update({'reason': reason[:128], 'status': False}) + write_login_log(**data) diff --git a/apps/audits/signal_handlers/operate_log.py b/apps/audits/signal_handlers/operate_log.py new file mode 100644 index 000000000..0c0fb9f48 --- /dev/null +++ b/apps/audits/signal_handlers/operate_log.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +import uuid + +from django.apps import apps +from django.dispatch import receiver +from django.db.models.signals import post_save, pre_save, m2m_changed, pre_delete + +from audits.handler import ( + get_instance_current_with_cache_diff, cache_instance_before_data, + create_or_update_operate_log, get_instance_dict_from_cache +) +from audits.utils import model_to_dict_for_operate_log as model_to_dict +from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR, SKIP_SIGNAL +from common.signals import django_ready + +from ..const import MODELS_NEED_RECORD, ActionChoices + + +M2M_ACTION = { + POST_ADD: ActionChoices.create, + POST_REMOVE: ActionChoices.delete, + POST_CLEAR: ActionChoices.delete, +} + + +@receiver(m2m_changed) +def on_m2m_changed(sender, action, instance, reverse, model, pk_set, **kwargs): + if action not in M2M_ACTION: + return + if not instance: + return + + resource_type = instance._meta.verbose_name + current_instance = model_to_dict(instance, include_model_fields=False) + + instance_id = current_instance.get('id') + log_id, before_instance = get_instance_dict_from_cache(instance_id) + + field_name = str(model._meta.verbose_name) + objs = model.objects.filter(pk__in=pk_set) + objs_display = [str(o) for o in objs] + action = M2M_ACTION[action] + changed_field = current_instance.get(field_name, []) + + after, before, before_value = None, None, None + if action == ActionChoices.create: + before_value = list(set(changed_field) - set(objs_display)) + elif action == ActionChoices.delete: + before_value = list( + set(changed_field).symmetric_difference(set(objs_display)) + ) + + if changed_field: + after = {field_name: changed_field} + if before_value: + before = {field_name: before_value} + + if sorted(str(before)) == sorted(str(after)): + return + + create_or_update_operate_log( + ActionChoices.update, resource_type, + resource=instance, log_id=log_id, before=before, after=after + ) + + +def signal_of_operate_log_whether_continue( + sender, instance, created, update_fields=None +): + condition = True + if not instance: + condition = False + if instance and getattr(instance, SKIP_SIGNAL, False): + condition = False + # 终端模型的 create 事件由系统产生，不记录 + if instance._meta.object_name == 'Terminal' and created: + condition = False + # last_login 改变是最后登录日期, 每次登录都会改变 + if instance._meta.object_name == 'User' and \ + update_fields and 'last_login' in update_fields: + condition = False + # 不在记录白名单中，跳过 + if sender._meta.object_name not in MODELS_NEED_RECORD: + condition = False + return condition + + +@receiver(pre_save) +def on_object_pre_create_or_update( + sender, instance=None, raw=False, using=None, update_fields=None, **kwargs +): + ok = signal_of_operate_log_whether_continue( + sender, instance, False, update_fields + ) + if not ok: + return + + # users.PrivateToken Model 没有 id 有 pk字段 + instance_id = getattr(instance, 'id', getattr(instance, 'pk', None)) + instance_before_data = {'id': instance_id} + raw_instance = type(instance).objects.filter(pk=instance_id).first() + + if raw_instance: + instance_before_data = model_to_dict(raw_instance) + operate_log_id = str(uuid.uuid4()) + instance_before_data['operate_log_id'] = operate_log_id + setattr(instance, 'operate_log_id', operate_log_id) + cache_instance_before_data(instance_before_data) + + +@receiver(post_save) +def on_object_created_or_update( + sender, instance=None, created=False, update_fields=None, **kwargs +): + ok = signal_of_operate_log_whether_continue( + sender, instance, created, update_fields + ) + if not ok: + return + + log_id, before, after = None, None, None + if created: + action = ActionChoices.create + after = model_to_dict(instance) + log_id = getattr(instance, 'operate_log_id', None) + else: + action = ActionChoices.update + current_instance = model_to_dict(instance) + log_id, before, after = get_instance_current_with_cache_diff(current_instance) + + resource_type = sender._meta.verbose_name + object_name = sender._meta.object_name + create_or_update_operate_log( + action, resource_type, resource=instance, log_id=log_id, + before=before, after=after, object_name=object_name + ) + + +@receiver(pre_delete) +def on_object_delete(sender, instance=None, **kwargs): + ok = signal_of_operate_log_whether_continue(sender, instance, False) + if not ok: + return + + resource_type = sender._meta.verbose_name + create_or_update_operate_log( + ActionChoices.delete, resource_type, + resource=instance, before=model_to_dict(instance) + ) + + +@receiver(django_ready) +def on_django_start_set_operate_log_monitor_models(sender, **kwargs): + exclude_apps = { + 'django_cas_ng', 'captcha', 'admin', 'jms_oidc_rp', 'audits', + 'django_celery_beat', 'contenttypes', 'sessions', 'auth', + } + exclude_models = { + 'UserPasswordHistory', 'ContentType', + 'MessageContent', 'SiteMessage', + 'PlatformAutomation', 'PlatformProtocol', 'Protocol', + 'HistoricalAccount', 'GatheredUser', 'ApprovalRule', + 'BaseAutomation', 'CeleryTask', 'Command', 'JobAuditLog', + 'ConnectionToken', 'SessionJoinRecord', + 'HistoricalJob', 'Status', 'TicketStep', 'Ticket', + 'UserAssetGrantedTreeNodeRelation', 'TicketAssignee', + 'SuperTicket', 'SuperConnectionToken', 'PermNode', + 'PermedAsset', 'PermedAccount', 'MenuPermission', + 'Permission', 'TicketSession', 'ApplyLoginTicket', + 'ApplyCommandTicket', 'ApplyLoginAssetTicket', + } + for i, app in enumerate(apps.get_models(), 1): + app_name = app._meta.app_label + model_name = app._meta.object_name + if app_name in exclude_apps or \ + model_name in exclude_models or \ + model_name.endswith('Execution'): + continue + MODELS_NEED_RECORD.add(model_name) diff --git a/apps/audits/signal_handlers/other.py b/apps/audits/signal_handlers/other.py new file mode 100644 index 000000000..07d3694fe --- /dev/null +++ b/apps/audits/signal_handlers/other.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +from django.dispatch import receiver +from django.db import transaction + +from audits.models import ( + PasswordChangeLog, UserLoginLog, FTPLog, OperateLog +) +from audits.serializers import ( + UserLoginLogSerializer, FTPLogSerializer, OperateLogSerializer, + PasswordChangeLogSerializer +) +from common.utils import get_request_ip, get_syslogger +from common.utils.encode import data_to_json +from jumpserver.utils import current_request +from users.models import User +from users.signals import post_user_change_password +from terminal.models import Session, Command +from terminal.serializers import SessionSerializer, SessionCommandSerializer + + +sys_logger = get_syslogger(__name__) + + +@receiver(post_user_change_password, sender=User) +def on_user_change_password(sender, user=None, **kwargs): + if not current_request: + remote_addr = '127.0.0.1' + change_by = 'System' + else: + remote_addr = get_request_ip(current_request) + if not current_request.user.is_authenticated: + change_by = str(user) + else: + change_by = str(current_request.user) + with transaction.atomic(): + PasswordChangeLog.objects.create( + user=str(user), change_by=change_by, + remote_addr=remote_addr, + ) + + +def on_audits_log_create(sender, instance=None, **kwargs): + if sender == UserLoginLog: + category = "login_log" + serializer_cls = UserLoginLogSerializer + elif sender == FTPLog: + category = "ftp_log" + serializer_cls = FTPLogSerializer + elif sender == OperateLog: + category = "operation_log" + serializer_cls = OperateLogSerializer + elif sender == PasswordChangeLog: + category = "password_change_log" + serializer_cls = PasswordChangeLogSerializer + elif sender == Session: + category = "host_session_log" + serializer_cls = SessionSerializer + elif sender == Command: + category = "session_command_log" + serializer_cls = SessionCommandSerializer + else: + return + + serializer = serializer_cls(instance) + data = data_to_json(serializer.data, indent=None) + msg = "{} - {}".format(category, data) + sys_logger.info(msg) diff --git a/apps/audits/signals.py b/apps/audits/signals.py deleted file mode 100644 index 229c63933..000000000 --- a/apps/audits/signals.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.dispatch import Signal - - -post_activity_log = Signal( - providing_args=('resource_id', 'detail', 'detail_url') -) diff --git a/apps/audits/utils.py b/apps/audits/utils.py index 851ddae3b..3ff42177c 100644 --- a/apps/audits/utils.py +++ b/apps/audits/utils.py @@ -4,13 +4,10 @@ from itertools import chain from django.db import models from django.http import HttpResponse -from django.utils.translation import gettext_lazy as _ -from common.utils import validate_ip, get_ip_city, get_logger, reverse -from audits.const import ActivityChoices +from common.utils import validate_ip, get_ip_city, get_logger from settings.serializers import SettingsSerializer from .const import DEFAULT_CITY -from .signals import post_activity_log logger = get_logger(__name__) @@ -44,24 +41,7 @@ def write_login_log(*args, **kwargs): else: city = get_ip_city(ip) or DEFAULT_CITY kwargs.update({'ip': ip, 'city': city}) - user_id = kwargs.pop('user_id', None) - audit_log = UserLoginLog.objects.create(**kwargs) - - # 发送Activity信号 - if user_id is not None: - login_status = _('Success') if audit_log.status else _('Failed') - detail = _('User {} login into this service.[{}]').format( - audit_log.username, login_status - ) - detail_url = '%s?id=%s' % ( - reverse('api-audits:login-log-list', api_to_ui=True, is_audit=True,), - audit_log.id - ) - post_activity_log.send( - sender=UserLoginLog, resource_id=user_id, - detail=detail, detail_url=detail_url, - type=ActivityChoices.login_log - ) + UserLoginLog.objects.create(**kwargs) def get_resource_display(resource): diff --git a/apps/terminal/api/session/session.py b/apps/terminal/api/session/session.py index 1284ae8ed..6f4f2863f 100644 --- a/apps/terminal/api/session/session.py +++ b/apps/terminal/api/session/session.py @@ -15,8 +15,6 @@ from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response -from audits.signals import post_activity_log -from audits.const import ActivityChoices from common.const.http import GET from common.drf.filters import DatetimeRangeFilter from common.drf.renders import PassthroughRenderer @@ -122,30 +120,10 @@ class SessionViewSet(OrgBulkModelViewSet): queryset = queryset.select_for_update() return queryset - @staticmethod - def send_activity(serializer): - # 发送Activity信号 - data = serializer.validated_data - user, asset_id = data['user'], data["asset_id"] - account, login_from = data['account'], data["login_from"] - login_from = Session(login_from=login_from).get_login_from_display() - detail = _( - '{} used account[{}], login method[{}] login the asset.' - ).format( - user, account, login_from - ) - post_activity_log.send( - sender=Session, resource_id=asset_id, detail=detail, - type=ActivityChoices.session_log - ) - def perform_create(self, serializer): if hasattr(self.request.user, 'terminal'): serializer.validated_data["terminal"] = self.request.user.terminal - - resp = super().perform_create(serializer) - self.send_activity(serializer) - return resp + return super().perform_create(serializer) class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet): From fa54df6d99fdd6bcc4bf4636b50bbfeb68208ac6 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Wed, 8 Feb 2023 11:43:35 +0800 Subject: [PATCH 05/44] =?UTF-8?q?perf:=20=E8=BF=98=E5=8E=9F=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/api/account/account.py | 4 +--- apps/accounts/api/automations/base.py | 3 +-- apps/accounts/models/automations/base.py | 4 ++-- apps/accounts/tasks/automation.py | 4 ++-- apps/accounts/tasks/common.py | 4 ++-- apps/accounts/tasks/verify_account.py | 16 +++++++--------- apps/assets/api/asset/asset.py | 14 +++----------- apps/assets/automations/base/manager.py | 4 ++-- apps/assets/models/automations/base.py | 8 ++++---- apps/assets/tasks/common.py | 4 ++-- apps/assets/tasks/gather_facts.py | 12 ++++++------ apps/assets/tasks/ping.py | 16 +++++++--------- 12 files changed, 39 insertions(+), 54 deletions(-) diff --git a/apps/accounts/api/account/account.py b/apps/accounts/api/account/account.py index 728e7f333..9aa31c72d 100644 --- a/apps/accounts/api/account/account.py +++ b/apps/accounts/api/account/account.py @@ -53,9 +53,7 @@ class AccountViewSet(OrgBulkModelViewSet): account = super().get_object() account_ids = [account.id] asset_ids = [account.asset_id] - task = verify_accounts_connectivity.delay( - account_ids, asset_ids, user=request.user - ) + task = verify_accounts_connectivity.delay(account_ids, asset_ids) return Response(data={'task': task.id}) diff --git a/apps/accounts/api/automations/base.py b/apps/accounts/api/automations/base.py index e8f762c91..12fcd7b17 100644 --- a/apps/accounts/api/automations/base.py +++ b/apps/accounts/api/automations/base.py @@ -110,7 +110,6 @@ class AutomationExecutionViewSet( serializer.is_valid(raise_exception=True) automation = serializer.validated_data.get('automation') task = execute_automation.delay( - pid=automation.pk, trigger=Trigger.manual, - tp=self.tp, user=request.user + pid=automation.pk, trigger=Trigger.manual, tp=self.tp ) return Response({'task': task.id}, status=status.HTTP_201_CREATED) diff --git a/apps/accounts/models/automations/base.py b/apps/accounts/models/automations/base.py index 4c5038481..ce45d6a81 100644 --- a/apps/accounts/models/automations/base.py +++ b/apps/accounts/models/automations/base.py @@ -35,7 +35,7 @@ class AutomationExecution(AssetAutomationExecution): ('add_pushaccountexecution', _('Can add push account execution')), ] - def start(self, **kwargs): + def start(self): from accounts.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) - return manager.run(**kwargs) + return manager.run() diff --git a/apps/accounts/tasks/automation.py b/apps/accounts/tasks/automation.py index e25a1aaf8..67d4c4a90 100644 --- a/apps/accounts/tasks/automation.py +++ b/apps/accounts/tasks/automation.py @@ -9,7 +9,7 @@ logger = get_logger(__file__) @shared_task(queue='ansible', verbose_name=_('Account execute automation')) -def execute_automation(pid, trigger, tp, **kwargs): +def execute_automation(pid, trigger, tp): model = AutomationTypes.get_type_model(tp) with tmp_to_root_org(): instance = get_object_or_none(model, pk=pid) @@ -17,4 +17,4 @@ def execute_automation(pid, trigger, tp, **kwargs): logger.error("No automation task found: {}".format(pid)) return with tmp_to_org(instance.org): - instance.execute(trigger, **kwargs) + instance.execute(trigger) diff --git a/apps/accounts/tasks/common.py b/apps/accounts/tasks/common.py index 19c750c4d..1f422ab5f 100644 --- a/apps/accounts/tasks/common.py +++ b/apps/accounts/tasks/common.py @@ -5,7 +5,7 @@ from assets.tasks.common import generate_data from common.const.choices import Trigger -def automation_execute_start(task_name, tp, child_snapshot=None, **kwargs): +def automation_execute_start(task_name, tp, child_snapshot=None): from accounts.models import AutomationExecution data = generate_data(task_name, tp, child_snapshot) @@ -19,4 +19,4 @@ def automation_execute_start(task_name, tp, child_snapshot=None, **kwargs): execution = AutomationExecution.objects.create( trigger=Trigger.manual, **data ) - execution.start(**kwargs) + execution.start() diff --git a/apps/accounts/tasks/verify_account.py b/apps/accounts/tasks/verify_account.py index 4ec741499..4c478ce89 100644 --- a/apps/accounts/tasks/verify_account.py +++ b/apps/accounts/tasks/verify_account.py @@ -14,7 +14,7 @@ __all__ = [ ] -def verify_connectivity_util(assets, tp, accounts, task_name, **kwargs): +def verify_connectivity_util(assets, tp, accounts, task_name): if not assets or not accounts: return account_usernames = list(accounts.values_list('username', flat=True)) @@ -22,30 +22,28 @@ def verify_connectivity_util(assets, tp, accounts, task_name, **kwargs): 'accounts': account_usernames, 'assets': [str(asset.id) for asset in assets], } - automation_execute_start(task_name, tp, child_snapshot, **kwargs) + automation_execute_start(task_name, tp, child_snapshot) @org_aware_func("assets") -def verify_accounts_connectivity_util(accounts, assets, task_name, **kwargs): +def verify_accounts_connectivity_util(accounts, assets, task_name): gateway_assets = assets.filter(platform__name=GATEWAY_NAME) verify_connectivity_util( - gateway_assets, AutomationTypes.verify_gateway_account, - accounts, task_name, **kwargs + gateway_assets, AutomationTypes.verify_gateway_account, accounts, task_name ) non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) verify_connectivity_util( - non_gateway_assets, AutomationTypes.verify_account, - accounts, task_name, **kwargs + non_gateway_assets, AutomationTypes.verify_account, accounts, task_name ) @shared_task(queue="ansible", verbose_name=_('Verify asset account availability')) -def verify_accounts_connectivity(account_ids, asset_ids, **kwargs): +def verify_accounts_connectivity(account_ids, asset_ids): from assets.models import Asset from accounts.models import Account, VerifyAccountAutomation assets = Asset.objects.filter(id__in=asset_ids) accounts = Account.objects.filter(id__in=account_ids) task_name = gettext_noop("Verify accounts connectivity") task_name = VerifyAccountAutomation.generate_unique_name(task_name) - return verify_accounts_connectivity_util(accounts, assets, task_name, **kwargs) + return verify_accounts_connectivity_util(accounts, assets, task_name) diff --git a/apps/assets/api/asset/asset.py b/apps/assets/api/asset/asset.py index ea2f1dfbf..13f4b86ac 100644 --- a/apps/assets/api/asset/asset.py +++ b/apps/assets/api/asset/asset.py @@ -2,8 +2,7 @@ # import django_filters from django.db.models import Q -from django.utils.translation import ugettext_lazy as _ -from rest_framework.request import Request +from django.utils.translation import gettext as _ from rest_framework.decorators import action from rest_framework.response import Response @@ -139,21 +138,14 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet): class AssetsTaskMixin: - request: Request - def perform_assets_task(self, serializer): data = serializer.validated_data assets = data.get("assets", []) asset_ids = [asset.id for asset in assets] - user = self.request.user if data["action"] == "refresh": - task = update_assets_hardware_info_manual.delay( - asset_ids, user=user - ) + task = update_assets_hardware_info_manual.delay(asset_ids) else: - task = test_assets_connectivity_manual.delay( - asset_ids, user=user - ) + task = test_assets_connectivity_manual.delay(asset_ids) return task def perform_create(self, serializer): diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 2254b1683..6247c85d7 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -147,7 +147,7 @@ class BasePlaybookManager: yaml.safe_dump(plays, f) return sub_playbook_path - def get_runners(self, **kwargs): + def get_runners(self): runners = [] for platform, assets in self.get_assets_group_by_platform().items(): assets_bulked = [assets[i:i + self.bulk_size] for i in range(0, len(assets), self.bulk_size)] @@ -195,7 +195,7 @@ class BasePlaybookManager: pass def run(self, *args, **kwargs): - runners = self.get_runners(user=kwargs.pop('user', None)) + runners = self.get_runners() if len(runners) > 1: print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) else: diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index ac05d1080..e888fdf26 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -76,7 +76,7 @@ class BaseAutomation(PeriodTaskModelMixin, JMSOrgBaseModel): def executed_amount(self): return self.executions.count() - def execute(self, trigger=Trigger.manual, **kwargs): + def execute(self, trigger=Trigger.manual): try: eid = current_task.request.id except AttributeError: @@ -86,7 +86,7 @@ class BaseAutomation(PeriodTaskModelMixin, JMSOrgBaseModel): id=eid, trigger=trigger, automation=self, snapshot=self.to_attr_json(), ) - return execution.start(**kwargs) + return execution.start() class AssetBaseAutomation(BaseAutomation): @@ -140,7 +140,7 @@ class AutomationExecution(OrgModelMixin): return {} return recipients - def start(self, **kwargs): + def start(self): from assets.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) - return manager.run(**kwargs) + return manager.run() diff --git a/apps/assets/tasks/common.py b/apps/assets/tasks/common.py index 2d90dcf92..9bafbcde8 100644 --- a/apps/assets/tasks/common.py +++ b/apps/assets/tasks/common.py @@ -29,7 +29,7 @@ def generate_data(task_name, tp, child_snapshot=None): return {'id': eid, 'snapshot': snapshot} -def automation_execute_start(task_name, tp, child_snapshot=None, **kwargs): +def automation_execute_start(task_name, tp, child_snapshot=None): from assets.models import AutomationExecution data = generate_data(task_name, tp, child_snapshot) @@ -43,4 +43,4 @@ def automation_execute_start(task_name, tp, child_snapshot=None, **kwargs): execution = AutomationExecution.objects.create( trigger=Trigger.manual, **data ) - execution.start(**kwargs) + execution.start() diff --git a/apps/assets/tasks/gather_facts.py b/apps/assets/tasks/gather_facts.py index 075394d3f..ad678b92a 100644 --- a/apps/assets/tasks/gather_facts.py +++ b/apps/assets/tasks/gather_facts.py @@ -17,7 +17,7 @@ __all__ = [ ] -def update_fact_util(assets=None, nodes=None, task_name=None, **kwargs): +def update_fact_util(assets=None, nodes=None, task_name=None): from assets.models import GatherFactsAutomation if task_name is None: task_name = gettext_noop("Update some assets hardware info. ") @@ -30,16 +30,16 @@ def update_fact_util(assets=None, nodes=None, task_name=None, **kwargs): 'nodes': [str(node.id) for node in nodes], } tp = AutomationTypes.gather_facts - automation_execute_start(task_name, tp, child_snapshot, **kwargs) + automation_execute_start(task_name, tp, child_snapshot) @org_aware_func('assets') -def update_assets_fact_util(assets=None, task_name=None, **kwargs): +def update_assets_fact_util(assets=None, task_name=None): if assets is None: logger.info("No assets to update hardware info") return - update_fact_util(assets=assets, task_name=task_name, **kwargs) + update_fact_util(assets=assets, task_name=task_name) @org_aware_func('nodes') @@ -51,11 +51,11 @@ def update_nodes_fact_util(nodes=None, task_name=None): @shared_task(queue="ansible", verbose_name=_('Manually update the hardware information of assets')) -def update_assets_hardware_info_manual(asset_ids, **kwargs): +def update_assets_hardware_info_manual(asset_ids): from assets.models import Asset assets = Asset.objects.filter(id__in=asset_ids) task_name = gettext_noop("Update assets hardware info: ") - update_assets_fact_util(assets=assets, task_name=task_name, **kwargs) + update_assets_fact_util(assets=assets, task_name=task_name) @shared_task(queue="ansible", verbose_name=_('Manually update the hardware information of assets under a node')) diff --git a/apps/assets/tasks/ping.py b/apps/assets/tasks/ping.py index c4e4bcf04..ef65e2c22 100644 --- a/apps/assets/tasks/ping.py +++ b/apps/assets/tasks/ping.py @@ -17,7 +17,7 @@ __all__ = [ ] -def test_connectivity_util(assets, tp, task_name, local_port=None, **kwargs): +def test_connectivity_util(assets, tp, task_name, local_port=None): if not assets: return @@ -27,11 +27,11 @@ def test_connectivity_util(assets, tp, task_name, local_port=None, **kwargs): child_snapshot = {'local_port': local_port} child_snapshot['assets'] = [str(asset.id) for asset in assets] - automation_execute_start(task_name, tp, child_snapshot, **kwargs) + automation_execute_start(task_name, tp, child_snapshot) @org_aware_func('assets') -def test_asset_connectivity_util(assets, task_name=None, local_port=None, **kwargs): +def test_asset_connectivity_util(assets, task_name=None, local_port=None): from assets.models import PingAutomation if task_name is None: task_name = gettext_noop("Test assets connectivity ") @@ -40,23 +40,21 @@ def test_asset_connectivity_util(assets, task_name=None, local_port=None, **kwar gateway_assets = assets.filter(platform__name=GATEWAY_NAME) test_connectivity_util( - gateway_assets, AutomationTypes.ping_gateway, - task_name, local_port, **kwargs + gateway_assets, AutomationTypes.ping_gateway, task_name, local_port ) non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) test_connectivity_util( - non_gateway_assets, AutomationTypes.ping, - task_name, **kwargs + non_gateway_assets, AutomationTypes.ping, task_name ) @shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of a asset')) -def test_assets_connectivity_manual(asset_ids, local_port=None, **kwargs): +def test_assets_connectivity_manual(asset_ids, local_port=None): from assets.models import Asset assets = Asset.objects.filter(id__in=asset_ids) task_name = gettext_noop("Test assets connectivity ") - test_asset_connectivity_util(assets, task_name, local_port, **kwargs) + test_asset_connectivity_util(assets, task_name, local_port) @shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of assets under a node')) From e3c0425a7ddedb6fb6c3e118832eb25271c593a7 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Wed, 8 Feb 2023 11:46:10 +0800 Subject: [PATCH 06/44] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/tasks/ping.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/assets/tasks/ping.py b/apps/assets/tasks/ping.py index ef65e2c22..7117c8bc7 100644 --- a/apps/assets/tasks/ping.py +++ b/apps/assets/tasks/ping.py @@ -44,9 +44,7 @@ def test_asset_connectivity_util(assets, task_name=None, local_port=None): ) non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) - test_connectivity_util( - non_gateway_assets, AutomationTypes.ping, task_name - ) + test_connectivity_util(non_gateway_assets, AutomationTypes.ping, task_name) @shared_task(queue="ansible", verbose_name=_('Manually test the connectivity of a asset')) From 953adf6cfbe95aba0651b51a4b168da6c0fa0d21 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 8 Feb 2023 14:12:52 +0800 Subject: [PATCH 07/44] =?UTF-8?q?fix:=20(smart=20endpoint=20=E8=8E=B7?= =?UTF-8?q?=E5=8F=96500=E7=9A=84=E9=97=AE=E9=A2=98)=20(=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=9B=91=E6=8E=A7=E4=BC=9A=E8=AF=9D=E6=89=93=E5=BC=80=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/models/component/endpoint.py | 2 +- apps/terminal/models/session/session.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/terminal/models/component/endpoint.py b/apps/terminal/models/component/endpoint.py index 47d7a7894..8699eb898 100644 --- a/apps/terminal/models/component/endpoint.py +++ b/apps/terminal/models/component/endpoint.py @@ -74,7 +74,7 @@ class Endpoint(JMSBaseModel): from assets.models import Asset from terminal.models import Session if isinstance(instance, Session): - instance = instance.get_asset_or_application() + instance = instance.get_asset() if not isinstance(instance, Asset): return None values = instance.labels.filter(name='endpoint').values_list('value', flat=True) diff --git a/apps/terminal/models/session/session.py b/apps/terminal/models/session/session.py index 0241f10e5..70513753d 100644 --- a/apps/terminal/models/session/session.py +++ b/apps/terminal/models/session/session.py @@ -178,14 +178,11 @@ class Session(OrgModelMixin): def login_from_display(self): return self.get_login_from_display() - def get_asset_or_application(self): - instance = get_object_or_none(Asset, pk=self.asset_id) - if not instance: - instance = get_object_or_none(Application, pk=self.asset_id) - return instance + def get_asset(self): + return get_object_or_none(Asset, pk=self.asset_id) def get_target_ip(self): - instance = self.get_asset_or_application() + instance = self.get_asset() target_ip = instance.get_target_ip() if instance else '' return target_ip From 2a964bfa6c0138c72447ac0452d0167bb5e5b563 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 8 Feb 2023 14:27:37 +0800 Subject: [PATCH 08/44] =?UTF-8?q?fix:=20(smart=20endpoint=20=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E5=88=B0=E5=85=A8=E5=B1=80=E7=BB=84=E7=BB=87=E4=B8=8B?= =?UTF-8?q?=E5=8C=B9=E9=85=8D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/api/component/endpoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/terminal/api/component/endpoint.py b/apps/terminal/api/component/endpoint.py index 3b34f4a83..c7518932f 100644 --- a/apps/terminal/api/component/endpoint.py +++ b/apps/terminal/api/component/endpoint.py @@ -24,6 +24,7 @@ class SmartEndpointViewMixin: target_protocol: None @action(methods=['get'], detail=False, permission_classes=[IsValidUserOrConnectionToken]) + @tmp_to_root_org() def smart(self, request, *args, **kwargs): self.target_instance = self.get_target_instance() self.target_protocol = self.get_target_protocol() From 2c952c2877f6879390527549a240d2b7bcb827fa Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 14:30:05 +0800 Subject: [PATCH 09/44] fix: account create bug (#9465) Co-authored-by: feng <1304903146@qq.com> --- apps/accounts/serializers/account/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/accounts/serializers/account/account.py b/apps/accounts/serializers/account/account.py index 97033eda9..3017b950c 100644 --- a/apps/accounts/serializers/account/account.py +++ b/apps/accounts/serializers/account/account.py @@ -47,7 +47,7 @@ class AccountSerializerCreateValidateMixin: def create(self, validated_data): push_now = validated_data.pop('push_now', None) - instance = super().create(validated_data, push_now) + instance = super().create(validated_data) self.push_account(instance, push_now) return instance From 37f119260e9ff178a869c9cc3654038664a1b903 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 8 Feb 2023 15:16:00 +0800 Subject: [PATCH 10/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20connection=20?= =?UTF-8?q?token=20=E7=9B=B8=E5=85=B3=E7=9A=84=E8=BF=94=E5=9B=9E=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/serializers/connection_token.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/authentication/serializers/connection_token.py b/apps/authentication/serializers/connection_token.py index c2805c78d..7d68f4516 100644 --- a/apps/authentication/serializers/connection_token.py +++ b/apps/authentication/serializers/connection_token.py @@ -1,8 +1,9 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from common.serializers.fields import EncryptedField +from perms.serializers.permission import ActionChoicesField from orgs.mixins.serializers import OrgResourceModelSerializerMixin +from common.serializers.fields import EncryptedField from ..models import ConnectionToken __all__ = [ @@ -16,6 +17,7 @@ class ConnectionTokenSerializer(OrgResourceModelSerializerMixin): label=_("Input secret"), max_length=40960, required=False, allow_blank=True ) from_ticket_info = serializers.SerializerMethodField(label=_("Ticket info")) + actions = ActionChoicesField() class Meta: model = ConnectionToken @@ -29,7 +31,7 @@ class ConnectionTokenSerializer(OrgResourceModelSerializerMixin): ] read_only_fields = [ # 普通 Token 不支持指定 user - 'user', 'expire_time', + 'user', 'expire_time', 'is_expired', 'user_display', 'asset_display', ] fields = fields_small + read_only_fields From 5c1acae4c5da0cb68a318e9b0a24f55da63e3568 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:36:45 +0800 Subject: [PATCH 11/44] perf: push account ssh (#9467) Co-authored-by: feng <1304903146@qq.com> --- apps/accounts/automations/base/manager.py | 4 +- .../automations/change_secret/manager.py | 14 ++-- .../automations/push_account/manager.py | 79 ++++++++++++++++--- .../automations/verify_account/manager.py | 4 +- 4 files changed, 79 insertions(+), 22 deletions(-) diff --git a/apps/accounts/automations/base/manager.py b/apps/accounts/automations/base/manager.py index d2e96c7a6..2dd91d794 100644 --- a/apps/accounts/automations/base/manager.py +++ b/apps/accounts/automations/base/manager.py @@ -1,14 +1,14 @@ from copy import deepcopy from common.utils import get_logger -from accounts.const import AutomationTypes, SecretType +from accounts.const import SecretType from assets.automations.base.manager import BasePlaybookManager from accounts.automations.methods import platform_automation_methods logger = get_logger(__name__) -class PushOrVerifyHostCallbackMixin: +class VerifyHostCallbackMixin: execution: callable get_accounts: callable host_account_mapper: dict diff --git a/apps/accounts/automations/change_secret/manager.py b/apps/accounts/automations/change_secret/manager.py index eb2d9d185..411506fe7 100644 --- a/apps/accounts/automations/change_secret/manager.py +++ b/apps/accounts/automations/change_secret/manager.py @@ -33,18 +33,12 @@ class ChangeSecretManager(AccountBasePlaybookManager): 'ssh_key_change_strategy', SSHKeyStrategy.add ) self.snapshot_account_usernames = self.execution.snapshot['accounts'] - self._password_generated = None - self._ssh_key_generated = None self.name_recorder_mapper = {} # 做个映射，方便后面处理 @classmethod def method_type(cls): return AutomationTypes.change_secret - @lazyproperty - def related_accounts(self): - pass - def get_kwargs(self, account, secret): kwargs = {} if self.secret_type != SecretType.SSH_KEY: @@ -152,12 +146,16 @@ class ChangeSecretManager(AccountBasePlaybookManager): def on_runner_failed(self, runner, e): logger.error("Change secret error: ", e) - def run(self, *args, **kwargs): + def check_secret(self): if self.secret_strategy == SecretStrategy.custom \ and not self.execution.snapshot['secret']: print('Custom secret is empty') - return + return False + return True + def run(self, *args, **kwargs): + if not self.check_secret(): + return super().run(*args, **kwargs) recorders = self.name_recorder_mapper.values() recorders = list(recorders) diff --git a/apps/accounts/automations/push_account/manager.py b/apps/accounts/automations/push_account/manager.py index 293a42967..4eb047ed0 100644 --- a/apps/accounts/automations/push_account/manager.py +++ b/apps/accounts/automations/push_account/manager.py @@ -1,26 +1,23 @@ +from copy import deepcopy + from django.db.models import QuerySet from common.utils import get_logger -from accounts.const import AutomationTypes from accounts.models import Account -from ..base.manager import PushOrVerifyHostCallbackMixin, AccountBasePlaybookManager +from accounts.const import AutomationTypes, SecretType +from ..base.manager import AccountBasePlaybookManager +from ..change_secret.manager import ChangeSecretManager logger = get_logger(__name__) -class PushAccountManager(PushOrVerifyHostCallbackMixin, AccountBasePlaybookManager): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.secret_type = self.execution.snapshot['secret_type'] - self.host_account_mapper = {} +class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager): @classmethod def method_type(cls): return AutomationTypes.push_account def create_nonlocal_accounts(self, accounts, snapshot_account_usernames, asset): - secret = self.execution.snapshot['secret'] secret_type = self.secret_type usernames = accounts.filter(secret_type=secret_type).values_list( 'username', flat=True @@ -29,7 +26,7 @@ class PushAccountManager(PushOrVerifyHostCallbackMixin, AccountBasePlaybookManag create_account_objs = [ Account( name=f'{username}-{secret_type}', username=username, - secret=secret, secret_type=secret_type, asset=asset, + secret_type=secret_type, asset=asset, ) for username in create_usernames ] @@ -50,6 +47,68 @@ class PushAccountManager(PushOrVerifyHostCallbackMixin, AccountBasePlaybookManag ) return accounts + def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs): + host = super(ChangeSecretManager, self).host_callback( + host, asset=asset, account=account, automation=automation, + path_dir=path_dir, **kwargs + ) + if host.get('error'): + return host + + accounts = asset.accounts.all() + accounts = self.get_accounts(account, accounts) + + inventory_hosts = [] + host['secret_type'] = self.secret_type + for account in accounts: + h = deepcopy(host) + h['name'] += '_' + account.username + new_secret = self.get_secret() + + private_key_path = None + if self.secret_type == SecretType.SSH_KEY: + private_key_path = self.generate_private_key_path(new_secret, path_dir) + new_secret = self.generate_public_key(new_secret) + + self.name_recorder_mapper[h['name']] = { + 'account': account, 'new_secret': new_secret, + } + + h['kwargs'] = self.get_kwargs(account, new_secret) + h['account'] = { + 'name': account.name, + 'username': account.username, + 'secret_type': account.secret_type, + 'secret': new_secret, + 'private_key_path': private_key_path + } + if asset.platform.type == 'oracle': + h['account']['mode'] = 'sysdba' if account.privileged else None + inventory_hosts.append(h) + return inventory_hosts + + def on_host_success(self, host, result): + account_info = self.name_recorder_mapper.get(host) + if not account_info: + return + account = account_info['account'] + new_secret = account_info['new_secret'] + if not account: + return + account.secret = new_secret + account.save(update_fields=['secret']) + + def on_host_error(self, host, error, result): + pass + + def on_runner_failed(self, runner, e): + logger.error("Pust account error: ", e) + + def run(self, *args, **kwargs): + if not self.check_secret(): + return + super().run(*args, **kwargs) + # @classmethod # def trigger_by_asset_create(cls, asset): # automations = PushAccountAutomation.objects.filter( diff --git a/apps/accounts/automations/verify_account/manager.py b/apps/accounts/automations/verify_account/manager.py index fc25794bf..2b6c831dc 100644 --- a/apps/accounts/automations/verify_account/manager.py +++ b/apps/accounts/automations/verify_account/manager.py @@ -2,12 +2,12 @@ from django.db.models import QuerySet from accounts.const import AutomationTypes, Connectivity from common.utils import get_logger -from ..base.manager import PushOrVerifyHostCallbackMixin, AccountBasePlaybookManager +from ..base.manager import VerifyHostCallbackMixin, AccountBasePlaybookManager logger = get_logger(__name__) -class VerifyAccountManager(PushOrVerifyHostCallbackMixin, AccountBasePlaybookManager): +class VerifyAccountManager(VerifyHostCallbackMixin, AccountBasePlaybookManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From 03ad4124eb21283e68b312b3a5c85f2a6dab24ec Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 8 Feb 2023 16:44:11 +0800 Subject: [PATCH 12/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=89=8B?= =?UTF-8?q?=E5=8A=A8=E7=99=BB=E5=BD=95=E8=B5=84=E4=BA=A7=E6=97=B6=EF=BC=8C?= =?UTF-8?q?LoginACL=20QuerySet=20=E4=BC=9A=E8=8E=B7=E5=8F=96=E6=89=80?= =?UTF-8?q?=E6=9C=89=E7=BB=84=E7=BB=87=E4=B8=8B=E8=A7=84=E5=88=99=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98(@INPUT=E8=B4=A6=E5=8F=B7=E6=B2=A1=E6=9C=89or?= =?UTF-8?q?g=5Fid)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/models/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/acls/models/base.py b/apps/acls/models/base.py index 061fcfd01..4649cd626 100644 --- a/apps/acls/models/base.py +++ b/apps/acls/models/base.py @@ -113,14 +113,14 @@ class UserAssetAccountBaseACL(BaseACL, OrgModelMixin): org_id = None if user: queryset = queryset.filter_user(user.username) - if asset: - org_id = asset.org_id - queryset = queryset.filter_asset(asset.name, asset.address) if account: org_id = account.org_id queryset = queryset.filter_account(account.username) if account_username: queryset = queryset.filter_account(username=account_username) + if asset: + org_id = asset.org_id + queryset = queryset.filter_asset(asset.name, asset.address) if org_id: kwargs['org_id'] = org_id if kwargs: From ddff968be124cf4a25caa069099fd7745f37def6 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:13:05 +0800 Subject: [PATCH 13/44] perf: postgresql ansible (#9469) Co-authored-by: feng <1304903146@qq.com> --- .../change_secret/database/postgresql/main.yml | 13 ++++++++----- .../automations/change_secret/host/windows/main.yml | 1 + .../verify_account/database/postgresql/main.yml | 3 +++ .../automations/ping/database/postgresql/main.yml | 2 ++ apps/assets/const/database.py | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/apps/accounts/automations/change_secret/database/postgresql/main.yml b/apps/accounts/automations/change_secret/database/postgresql/main.yml index 6a903171f..96ee89dce 100644 --- a/apps/accounts/automations/change_secret/database/postgresql/main.yml +++ b/apps/accounts/automations/change_secret/database/postgresql/main.yml @@ -11,12 +11,13 @@ login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" login_db: "{{ jms_asset.spec_info.db_name }}" - register: db_info + register: result + failed_when: not result.is_available - name: Display PostgreSQL version debug: - var: db_info.server_version.full - when: db_info is succeeded + var: result.server_version.full + when: result is succeeded - name: Change PostgreSQL password community.postgresql.postgresql_user: @@ -27,7 +28,7 @@ db: "{{ jms_asset.spec_info.db_name }}" name: "{{ account.username }}" password: "{{ account.secret }}" - when: db_info is succeeded + when: result is succeeded register: change_info - name: Verify password @@ -38,5 +39,7 @@ login_port: "{{ jms_asset.port }}" db: "{{ jms_asset.spec_info.db_name }}" when: - - db_info is succeeded + - result is succeeded - change_info is succeeded + register: result + failed_when: not result.is_available diff --git a/apps/accounts/automations/change_secret/host/windows/main.yml b/apps/accounts/automations/change_secret/host/windows/main.yml index 0c27301dc..9ba28c075 100644 --- a/apps/accounts/automations/change_secret/host/windows/main.yml +++ b/apps/accounts/automations/change_secret/host/windows/main.yml @@ -12,6 +12,7 @@ ansible.windows.win_user: name: "{{ account.username }}" password: "{{ account.secret }}" + groups: "Administrators" update_password: always when: account.secret_type == "password" diff --git a/apps/accounts/automations/verify_account/database/postgresql/main.yml b/apps/accounts/automations/verify_account/database/postgresql/main.yml index cb6344b39..077a9ce76 100644 --- a/apps/accounts/automations/verify_account/database/postgresql/main.yml +++ b/apps/accounts/automations/verify_account/database/postgresql/main.yml @@ -3,6 +3,7 @@ vars: ansible_python_interpreter: /usr/local/bin/python + tasks: - name: Verify account community.postgresql.postgresql_ping: @@ -11,3 +12,5 @@ login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" db: "{{ jms_asset.spec_info.db_name }}" + register: result + failed_when: not result.is_available diff --git a/apps/assets/automations/ping/database/postgresql/main.yml b/apps/assets/automations/ping/database/postgresql/main.yml index f48c7d4f7..b7ecb55ee 100644 --- a/apps/assets/automations/ping/database/postgresql/main.yml +++ b/apps/assets/automations/ping/database/postgresql/main.yml @@ -11,3 +11,5 @@ login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" login_db: "{{ jms_asset.spec_info.db_name }}" + register: result + failed_when: not result.is_available diff --git a/apps/assets/const/database.py b/apps/assets/const/database.py index 13979890f..623b5b291 100644 --- a/apps/assets/const/database.py +++ b/apps/assets/const/database.py @@ -30,7 +30,7 @@ class DatabaseTypes(BaseType): 'ansible_connection': 'local', }, 'ping_enabled': True, - 'gather_facts_enabled': True, + 'gather_facts_enabled': False, 'gather_accounts_enabled': True, 'verify_account_enabled': True, 'change_secret_enabled': True, From f3405b09376643d1e0e05e39b0b30a4c25ec5708 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:27:49 +0800 Subject: [PATCH 14/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E7=94=A8=E6=88=B7=E6=8E=A7=E5=88=B6=E5=8F=B0=E6=9C=AC?= =?UTF-8?q?=E5=91=A8=E6=96=B0=E5=A2=9E=E6=95=B0=E9=87=8F=E4=B8=8D=E5=8F=98?= =?UTF-8?q?=E5=8C=96=E7=9A=84=E9=97=AE=E9=A2=98=20(#9470)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aaron3S --- apps/orgs/signal_handlers/cache.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/orgs/signal_handlers/cache.py b/apps/orgs/signal_handlers/cache.py index 0acebf6b9..c702dc6d2 100644 --- a/apps/orgs/signal_handlers/cache.py +++ b/apps/orgs/signal_handlers/cache.py @@ -30,32 +30,36 @@ def refresh_cache(name, org): logger.warning('refresh cache fail: {}'.format(name)) -def refresh_user_amount_cache(user): +def refresh_all_orgs_user_amount_cache(user): orgs = user.orgs.distinct() for org in orgs: refresh_cache('users_amount', org) + refresh_cache('new_users_amount_this_week', org) @receiver(post_save, sender=OrgRoleBinding) def on_user_create_or_invite_refresh_cache(sender, instance, created, **kwargs): if created: refresh_cache('users_amount', instance.org) + refresh_cache('new_users_amount_this_week', instance.org) @receiver(post_save, sender=SystemRoleBinding) def on_user_global_create_refresh_cache(sender, instance, created, **kwargs): if created and current_org.is_root(): refresh_cache('users_amount', current_org) + refresh_cache('new_users_amount_this_week', current_org) @receiver(pre_user_leave_org) def on_user_remove_refresh_cache(sender, org=None, **kwargs): refresh_cache('users_amount', org) + refresh_cache('new_users_amount_this_week', org) @receiver(pre_delete, sender=User) def on_user_delete_refresh_cache(sender, instance, **kwargs): - refresh_user_amount_cache(instance) + refresh_all_orgs_user_amount_cache(instance) # @receiver(m2m_changed, sender=OrganizationMember) From ab76d8cce078ed209257ac2f0df9e727752a4996 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 19:32:09 +0800 Subject: [PATCH 15/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E8=B5=84=E4=BA=A7=E8=B5=84=E6=BA=90=20dashboard=20?= =?UTF-8?q?=E4=B8=8D=E5=8F=98=E5=8C=96=E7=9A=84=E9=97=AE=E9=A2=98=20(#9471?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aaron3S --- apps/assets/signal_handlers/asset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/assets/signal_handlers/asset.py b/apps/assets/signal_handlers/asset.py index 0026d200e..55851f9c6 100644 --- a/apps/assets/signal_handlers/asset.py +++ b/apps/assets/signal_handlers/asset.py @@ -103,8 +103,9 @@ def on_asset_post_delete(instance: Asset, using, **kwargs): ) -def resend_to_asset_signals(sender, signal, **kwargs): - signal.send(sender=Asset, **kwargs) +@on_transaction_commit +def resend_to_asset_signals(sender, signal, instance, **kwargs): + signal.send(sender=Asset, instance=instance.asset_ptr, **kwargs) for model in (Host, Database, Device, Web, Cloud): From e3d1474b9b7fb38a69e69377f401a53b1bd9179a Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 20:14:33 +0800 Subject: [PATCH 16/44] perf: k8s tree error msg (#9472) Co-authored-by: feng <1304903146@qq.com> --- apps/assets/utils/k8s.py | 20 ++++++++++++------- .../serializers/connection_token.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/apps/assets/utils/k8s.py b/apps/assets/utils/k8s.py index 5ffd1612d..68b0b0b2f 100644 --- a/apps/assets/utils/k8s.py +++ b/apps/assets/utils/k8s.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from urllib3.exceptions import MaxRetryError from urllib.parse import urlencode +from urllib3.exceptions import MaxRetryError, LocationParseError from kubernetes import client from kubernetes.client import api_client @@ -8,7 +8,7 @@ from kubernetes.client.api import core_v1_api from kubernetes.client.exceptions import ApiException from common.utils import get_logger - +from common.exceptions import JMSException from ..const import CloudTypes, Category logger = get_logger(__file__) @@ -58,15 +58,21 @@ class KubernetesClient: api = self.get_api() try: ret = api.list_pod_for_all_namespaces(watch=False, _request_timeout=(3, 3)) + except LocationParseError as e: + logger.warning("Kubernetes API request url error: {}".format(e)) + raise JMSException(code='k8s_tree_error', detail=e) except MaxRetryError: - logger.warning('Kubernetes connection timed out') - return + msg = "Kubernetes API request timeout" + logger.warning(msg) + raise JMSException(code='k8s_tree_error', detail=msg) except ApiException as e: if e.status == 401: - logger.warning('Kubernetes User not authenticated') + msg = "Kubernetes API request unauthorized" + logger.warning(msg) else: - logger.warning(e) - return + msg = e + logger.warning(msg) + raise JMSException(code='k8s_tree_error', detail=msg) data = {} for i in ret.items: namespace = i.metadata.namespace diff --git a/apps/authentication/serializers/connection_token.py b/apps/authentication/serializers/connection_token.py index 7d68f4516..0c17974d5 100644 --- a/apps/authentication/serializers/connection_token.py +++ b/apps/authentication/serializers/connection_token.py @@ -17,7 +17,7 @@ class ConnectionTokenSerializer(OrgResourceModelSerializerMixin): label=_("Input secret"), max_length=40960, required=False, allow_blank=True ) from_ticket_info = serializers.SerializerMethodField(label=_("Ticket info")) - actions = ActionChoicesField() + actions = ActionChoicesField(read_only=True, label=_("Actions")) class Meta: model = ConnectionToken From 7b9523d6be72253f1954e3bdbe0ace943887a152 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 20:39:24 +0800 Subject: [PATCH 17/44] perf: ansible accounts is_active (#9473) Co-authored-by: feng <1304903146@qq.com> --- apps/ops/ansible/inventory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index a340b510d..f1ea649e6 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -139,7 +139,9 @@ class JMSInventory: return host def select_account(self, asset): - accounts = list(asset.accounts.all()) + accounts = list(asset.accounts.all(is_active=True)) + if not accounts: + return None account_selected = None account_usernames = self.account_prefer From d6d45f64fa4258dcf90ac06ab31e0d1d794462d8 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 8 Feb 2023 20:42:06 +0800 Subject: [PATCH 18/44] perf: inventiry account (#9474) Co-authored-by: feng <1304903146@qq.com> --- apps/ops/ansible/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index f1ea649e6..e83f50c41 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -139,7 +139,7 @@ class JMSInventory: return host def select_account(self, asset): - accounts = list(asset.accounts.all(is_active=True)) + accounts = list(asset.accounts.filter(is_active=True)) if not accounts: return None account_selected = None From 33d5cdedeadbf5cc37176dcc4bda3d6b02db739e Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Thu, 9 Feb 2023 11:02:09 +0800 Subject: [PATCH 19/44] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/audits/serializers.py | 34 +++++++++++---------- apps/audits/signal_handlers/activity_log.py | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index ebe7394d9..bc260dbb7 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -128,20 +128,22 @@ class ActivityOperatorLogSerializer(serializers.Serializer): def get_detail_url(obj): detail_url = '' detail_id, obj_type = obj['r_detail_id'], obj['r_type'] - if detail_id: - if obj_type == ActivityChoices.operate_log: - detail_url = reverse( - view_name='audits:operate-log-detail', - kwargs={'pk': obj['id']}, - api_to_ui=True, is_audit=True - ) - elif obj_type == ActivityChoices.task: - detail_url = reverse( - 'ops:celery-task-log', kwargs={'pk': detail_id} - ) - elif obj_type == ActivityChoices.login_log: - detail_url = '%s?id=%s' % ( - reverse('api-audits:login-log-list', api_to_ui=True, is_audit=True), - detail_id - ) + if not detail_id: + return detail_url + + if obj_type == ActivityChoices.operate_log: + detail_url = reverse( + view_name='audits:operate-log-detail', + kwargs={'pk': obj['id']}, + api_to_ui=True, is_audit=True + ) + elif obj_type == ActivityChoices.task: + detail_url = reverse( + 'ops:celery-task-log', kwargs={'pk': detail_id} + ) + elif obj_type == ActivityChoices.login_log: + detail_url = '%s?id=%s' % ( + reverse('api-audits:login-log-list', api_to_ui=True, is_audit=True), + detail_id + ) return detail_url diff --git a/apps/audits/signal_handlers/activity_log.py b/apps/audits/signal_handlers/activity_log.py index 8a09935ef..ce1b99fe2 100644 --- a/apps/audits/signal_handlers/activity_log.py +++ b/apps/audits/signal_handlers/activity_log.py @@ -176,7 +176,7 @@ def on_object_created( 'UserLoginLog': activity_handler.login_log_for_activity } model_name = sender._meta.object_name - if not created or model_name not in handler_mapping.keys(): + if not created or model_name not in handler_mapping: return resource_id, detail, a_type = handler_mapping[model_name](instance) From df428feaeb2a586c0fb64b04c578c873e58e9ffa Mon Sep 17 00:00:00 2001 From: Bai Date: Thu, 9 Feb 2023 11:06:48 +0800 Subject: [PATCH 20/44] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9=20command=20rev?= =?UTF-8?q?iew=20=E8=BF=94=E5=9B=9E=E4=B8=BA=20response=20=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/api/command_acl.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/acls/api/command_acl.py b/apps/acls/api/command_acl.py index 44a342743..e78b18572 100644 --- a/apps/acls/api/command_acl.py +++ b/apps/acls/api/command_acl.py @@ -1,7 +1,5 @@ from rest_framework.decorators import action from rest_framework.response import Response - -from common.utils import reverse from orgs.mixins.api import OrgBulkModelViewSet from .. import models, serializers @@ -36,4 +34,4 @@ class CommandFilterACLViewSet(OrgBulkModelViewSet): } ticket = serializer.cmd_filter_acl.create_command_review_ticket(**data) info = ticket.get_extra_info_of_review(user=request.user) - return info + return Response(data=info) From c088739a5dc6604562962df07227c8a405db5b04 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:49:16 +0800 Subject: [PATCH 21/44] perf: celery task search (#9477) Co-authored-by: feng <1304903146@qq.com> --- apps/ops/api/celery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/ops/api/celery.py b/apps/ops/api/celery.py index 34cb7140f..0fcc725ad 100644 --- a/apps/ops/api/celery.py +++ b/apps/ops/api/celery.py @@ -106,6 +106,8 @@ class CeleryTaskViewSet( mixins.ListModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet ): + filterset_fields = ('id', 'name') + search_fields = filterset_fields serializer_class = CeleryTaskSerializer def get_queryset(self): From 4f6a17290a5b73c33b88ec298953111d347f8f9c Mon Sep 17 00:00:00 2001 From: Bai Date: Thu, 9 Feb 2023 12:28:44 +0800 Subject: [PATCH 22/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E8=B5=84=E4=BA=A7=E6=97=B6=E8=B4=A6=E5=8F=B7=E7=A6=81?= =?UTF-8?q?=E7=94=A8=E4=B8=8D=E7=94=9F=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/serializers/asset/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index 8057731bc..974762364 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -65,7 +65,7 @@ class AssetAccountSerializer( class Meta: model = Account fields_mini = [ - 'id', 'name', 'username', 'privileged', + 'id', 'name', 'username', 'privileged', 'is_active', 'version', 'secret_type', ] fields_write_only = [ From c7c5805b18bb07a2e3050148cdd988d65be16925 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 14:11:54 +0800 Subject: [PATCH 23/44] fix: update platform script (#9479) * fix: update platform script * perf: check protocol setting --------- Co-authored-by: Eric --- apps/terminal/applets/chrome/app.py | 20 ++++++++++------ apps/terminal/applets/chrome/common.py | 32 ++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/apps/terminal/applets/chrome/app.py b/apps/terminal/applets/chrome/app.py index 426320f4f..8fbe0241f 100644 --- a/apps/terminal/applets/chrome/app.py +++ b/apps/terminal/applets/chrome/app.py @@ -93,14 +93,21 @@ class WebAPP(object): self.asset = asset self.account = account self.platform = platform - - self.extra_data = self.asset.spec_info self._steps = list() - autofill_type = self.asset.spec_info.autofill + + extra_data = self.asset.spec_info + autofill_type = extra_data.autofill + if not autofill_type: + protocol_setting = self.platform.get_protocol_setting("http") + if not protocol_setting: + print("No protocol setting found") + return + extra_data = protocol_setting + autofill_type = extra_data.autofill if autofill_type == "basic": - self._steps = self._default_custom_steps() + self._steps = self._default_custom_steps(extra_data) elif autofill_type == "script": - script_list = self.asset.spec_info.script + script_list = extra_data.script steps = sorted(script_list, key=lambda step_item: step_item.step) for item in steps: val = item.value @@ -110,9 +117,8 @@ class WebAPP(object): item.value = val self._steps.append(item) - def _default_custom_steps(self) -> list: + def _default_custom_steps(self, spec_info) -> list: account = self.account - spec_info = self.asset.spec_info default_steps = [ Step({ "step": 1, diff --git a/apps/terminal/applets/chrome/common.py b/apps/terminal/applets/chrome/common.py index dbac3a22b..3c527f615 100644 --- a/apps/terminal/applets/chrome/common.py +++ b/apps/terminal/applets/chrome/common.py @@ -77,15 +77,18 @@ def wait_pid(pid): break -class DictObj: - def __init__(self, in_dict: dict): - assert isinstance(in_dict, dict) - for key, val in in_dict.items(): +class DictObj(dict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for key, val in self.items(): if isinstance(val, (list, tuple)): setattr(self, key, [DictObj(x) if isinstance(x, dict) else x for x in val]) else: setattr(self, key, DictObj(val) if isinstance(val, dict) else val) + def __getattr__(self, item): + return self.get(item, None) + class User(DictObj): id: str @@ -151,11 +154,32 @@ class Account(DictObj): secret_type: LabelValue +class ProtocolSetting(DictObj): + autofill: str + username_selector: str + password_selector: str + submit_selector: str + script: list[Step] + + +class PlatformProtocolSetting(DictObj): + name: str + port: int + setting: ProtocolSetting + + class Platform(DictObj): id: str name: str charset: LabelValue type: LabelValue + protocols: list[PlatformProtocolSetting] + + def get_protocol_setting(self, protocol): + for item in self.protocols: + if item.name == protocol: + return item.setting + return None class Manifest(DictObj): From 238dc2e5599bf73eb2d3ea200ef35177a093c7ec Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Thu, 9 Feb 2023 14:35:57 +0800 Subject: [PATCH 24/44] =?UTF-8?q?fix:=20=E7=A6=81=E7=94=A8omnidb=20?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=20=20sqlserver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/connect_methods.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/terminal/connect_methods.py b/apps/terminal/connect_methods.py index 3b915619b..d69e24dc8 100644 --- a/apps/terminal/connect_methods.py +++ b/apps/terminal/connect_methods.py @@ -170,8 +170,8 @@ class ConnectMethodUtil: 'web_methods': [WebMethod.web_gui], 'listen': [Protocol.http], 'support': [ - Protocol.mysql, Protocol.postgresql, Protocol.oracle, - Protocol.sqlserver, Protocol.mariadb + Protocol.mysql, Protocol.postgresql, + Protocol.oracle, Protocol.mariadb ], 'match': 'm2m' }, From 921d8f6a2813ecf00287304d595c72353227a527 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Thu, 9 Feb 2023 15:48:42 +0800 Subject: [PATCH 25/44] =?UTF-8?q?fix:=20=E4=BF=9D=E6=8A=A4=E5=BF=85?= =?UTF-8?q?=E8=A6=81=E6=96=87=E4=BB=B6=E4=B8=8D=E8=A2=AB=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/api/playbook.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index 322129b8b..43e609345 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -59,6 +59,8 @@ class PlaybookFileBrowserAPIView(APIView): rbac_perms = () permission_classes = () + protected_files = ['root', 'main.yml'] + def get(self, request, **kwargs): playbook_id = kwargs.get('pk') playbook = get_object_or_404(Playbook, id=playbook_id) @@ -132,6 +134,10 @@ class PlaybookFileBrowserAPIView(APIView): work_path = playbook.work_dir file_key = request.data.get('key', '') + + if file_key in self.protected_files: + return Response({'msg': '{} can not be modified'.format(file_key)}, status=400) + if os.path.dirname(file_key) == 'root': file_key = os.path.basename(file_key) @@ -154,15 +160,14 @@ class PlaybookFileBrowserAPIView(APIView): return Response({'msg': 'ok'}) def delete(self, request, **kwargs): - not_delete_allowed = ['root', 'main.yml'] playbook_id = kwargs.get('pk') playbook = get_object_or_404(Playbook, id=playbook_id) work_path = playbook.work_dir file_key = request.query_params.get('key', '') if not file_key: - return Response(status=400) - if file_key in not_delete_allowed: - return Response(status=400) + return Response({'msg': 'key is required'}, status=400) + if file_key in self.protected_files: + return Response({'msg': ' {} can not be delete'.format(file_key)}, status=400) file_path = os.path.join(work_path, file_key) if os.path.isdir(file_path): shutil.rmtree(file_path) From 403faf9663afff9bd1049f1a5449252e36f8b114 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Thu, 9 Feb 2023 16:16:20 +0800 Subject: [PATCH 26/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E6=96=B9=E5=BC=8F=E5=88=9B=E5=BB=BA=20playbook=20?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E4=B8=BA=E7=A9=BA=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/api/playbook.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index 43e609345..ba2275206 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -37,22 +37,19 @@ class PlaybookViewSet(OrgBulkModelViewSet): def perform_create(self, serializer): instance = serializer.save() - if instance.create_method == 'blank': - dest_path = os.path.join(settings.DATA_DIR, "ops", "playbook", instance.id.__str__()) - os.makedirs(dest_path) - with open(os.path.join(dest_path, 'main.yml'), 'w') as f: - f.write('## write your playbook here') - - if instance.create_method == 'upload': + if 'multipart/form-data' in self.request.headers['Content-Type']: src_path = os.path.join(settings.MEDIA_ROOT, instance.path.name) dest_path = os.path.join(settings.DATA_DIR, "ops", "playbook", instance.id.__str__()) unzip_playbook(src_path, dest_path) - valid_entry = ('main.yml', 'main.yaml', 'main') - for f in os.listdir(dest_path): - if f in valid_entry: - return - os.remove(dest_path) - raise PlaybookNoValidEntry + if 'main.yml' not in os.listdir(dest_path): + raise PlaybookNoValidEntry + + else: + if instance.create_method == 'blank': + dest_path = os.path.join(settings.DATA_DIR, "ops", "playbook", instance.id.__str__()) + os.makedirs(dest_path) + with open(os.path.join(dest_path, 'main.yml'), 'w') as f: + f.write('## write your playbook here') class PlaybookFileBrowserAPIView(APIView): From b3e609dc9b44fc6c3279767d5056d9e251051ac3 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Thu, 9 Feb 2023 16:25:10 +0800 Subject: [PATCH 27/44] =?UTF-8?q?fix:=20=E8=A7=A3=E5=86=B3=E4=B8=8D?= =?UTF-8?q?=E5=90=8C=E7=94=A8=E6=88=B7=E4=BD=BF=E7=94=A8=E8=BF=9C=E7=A8=8B?= =?UTF-8?q?=E5=BA=94=E7=94=A8Navicat=E7=99=BB=E5=BD=95Oracle=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/applets/navicat/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/terminal/applets/navicat/app.py b/apps/terminal/applets/navicat/app.py index f7447007b..8f2b779da 100644 --- a/apps/terminal/applets/navicat/app.py +++ b/apps/terminal/applets/navicat/app.py @@ -1,3 +1,4 @@ +import os import sys import time @@ -172,7 +173,8 @@ class AppletApplication(BaseApplication): def run(self): self.launch() app = Application(backend='uia') - app.start(self.path) + work_dir = os.path.dirname(self.path) + app.start(self.path, work_dir=work_dir) self.pid = app.process # 检测是否为试用版本 From 015ac4fbb656a755756414537d0c2582595ac61c Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:03:58 +0800 Subject: [PATCH 28/44] =?UTF-8?q?fix:=20=E7=A6=81=E7=94=A8omnidb=20?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=20=20sqlserver=20(#9480)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aaron3S From 5401b1cdf2c9a0ceeff462613981b4dc0bddf9da Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:17:35 +0800 Subject: [PATCH 29/44] perf: audit dashboard (#9486) Co-authored-by: feng <1304903146@qq.com> --- apps/audits/api.py | 10 ++++++---- apps/jumpserver/api.py | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apps/audits/api.py b/apps/audits/api.py index a8c178ac2..2a1409f95 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -129,10 +129,12 @@ class PasswordChangeLogViewSet(ListModelMixin, JMSGenericViewSet): ordering = ['-datetime'] def get_queryset(self): - users = current_org.get_members() - queryset = super().get_queryset().filter( - user__in=[user.__str__() for user in users] - ) + queryset = super().get_queryset() + if not current_org.is_root(): + users = current_org.get_members() + queryset = queryset.filter( + user__in=[str(user) for user in users] + ) return queryset # Todo: 看看怎么搞 diff --git a/apps/jumpserver/api.py b/apps/jumpserver/api.py index a3e0dc7ac..045c208c3 100644 --- a/apps/jumpserver/api.py +++ b/apps/jumpserver/api.py @@ -19,7 +19,7 @@ from orgs.utils import current_org from ops.const import JobStatus from ops.models import Job, JobExecution from common.utils import lazyproperty -from audits.models import UserLoginLog, PasswordChangeLog, OperateLog +from audits.models import UserLoginLog, PasswordChangeLog, OperateLog, FTPLog from audits.const import LoginStatusChoices from common.utils.timezone import local_now, local_zero_hour from orgs.caches import OrgResourceStatisticsCache @@ -38,13 +38,13 @@ class DateTimeMixin: def days(self): query_params = self.request.query_params count = query_params.get('days') - count = int(count) if count else 0 + count = int(count) if count else 1 return count @property def days_to_datetime(self): days = self.days - if days == 0: + if days == 1: t = local_zero_hour() else: t = local_now() - timezone.timedelta(days=days) @@ -109,7 +109,7 @@ class DateTimeMixin: @lazyproperty def ftp_logs_queryset(self): t = self.days_to_datetime - queryset = OperateLog.objects.filter(datetime__gte=t) + queryset = FTPLog.objects.filter(date_start__gte=t) queryset = self.get_logs_queryset(queryset, 'user') return queryset @@ -297,7 +297,7 @@ class DatesLoginMetricMixin: @lazyproperty def user_login_amount(self): - return self.login_logs_queryset.values('username').distinct().count() + return self.login_logs_queryset.values('username').count() @lazyproperty def operate_logs_amount(self): From 5d13f1e357e2211b42250eaa82b529b0e48bba4e Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Thu, 9 Feb 2023 17:00:54 +0800 Subject: [PATCH 30/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20Job=20?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E4=B8=8D=E6=98=BE=E7=A4=BA=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/api/job.py | 6 +++++- apps/ops/models/job.py | 2 +- apps/ops/serializers/job.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index 30bd0ee18..108e812e1 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -3,6 +3,7 @@ from rest_framework.views import APIView from django.shortcuts import get_object_or_404 from rest_framework.response import Response +from ops.const import Types from ops.models import Job, JobExecution from ops.serializers.job import JobSerializer, JobExecutionSerializer @@ -66,6 +67,8 @@ class JobExecutionViewSet(OrgBulkModelViewSet): def perform_create(self, serializer): instance = serializer.save() instance.job_version = instance.job.version + instance.material = instance.job.material + instance.type = Types[instance.job.type].value instance.creator = self.request.user instance.save() task = run_ops_job_execution.delay(instance.id) @@ -123,6 +126,7 @@ class FrequentUsernames(APIView): permission_classes = () def get(self, request, **kwargs): - top_accounts = Account.objects.exclude(username='root').exclude(username__startswith='jms_').values('username').annotate( + top_accounts = Account.objects.exclude(username='root').exclude(username__startswith='jms_').values( + 'username').annotate( total=Count('username')).order_by('total')[:5] return Response(data=top_accounts) diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py index 0af043580..dd248e83d 100644 --- a/apps/ops/models/job.py +++ b/apps/ops/models/job.py @@ -92,7 +92,7 @@ class Job(JMSOrgBaseModel, PeriodTaskModelMixin): return "{}:{}:{}".format(self.org.name, self.creator.name, self.playbook.name) def create_execution(self): - return self.executions.create(job_version=self.version, material=self.material, job_type=Types[self.type].label) + return self.executions.create(job_version=self.version, material=self.material, job_type=Types[self.type].value) class Meta: verbose_name = _("Job") diff --git a/apps/ops/serializers/job.py b/apps/ops/serializers/job.py index f5e4ee5d6..13afe6d6c 100644 --- a/apps/ops/serializers/job.py +++ b/apps/ops/serializers/job.py @@ -59,7 +59,7 @@ class JobExecutionSerializer(BulkOrgResourceModelSerializer): model = JobExecution read_only_fields = ["id", "task_id", "timedelta", "time_cost", 'is_finished', 'date_start', 'date_finished', - 'date_created', 'is_success', 'task_id', 'job_type', + 'date_created', 'is_success', 'job_type', 'summary', 'material'] fields = read_only_fields + [ "job", "parameters", "creator" From a87ff2d88035d5d91c8859ba0bcdbf4289831840 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Thu, 9 Feb 2023 17:44:35 +0800 Subject: [PATCH 31/44] =?UTF-8?q?feat:=20=E4=B8=BA=E6=89=80=E6=9C=89api?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=90=9C=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/api/adhoc.py | 2 ++ apps/ops/api/celery.py | 1 + apps/ops/api/job.py | 2 ++ apps/ops/api/playbook.py | 1 + 4 files changed, 6 insertions(+) diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index aa7b890c5..d7759d88e 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -13,10 +13,12 @@ __all__ = [ class AdHocViewSet(OrgBulkModelViewSet): serializer_class = AdHocSerializer permission_classes = () + search_fields = ('name', 'comment') model = AdHoc def allow_bulk_destroy(self, qs, filtered): return True + def get_queryset(self): queryset = super().get_queryset() return queryset.filter(creator=self.request.user) diff --git a/apps/ops/api/celery.py b/apps/ops/api/celery.py index 0fcc725ad..af54f824a 100644 --- a/apps/ops/api/celery.py +++ b/apps/ops/api/celery.py @@ -118,6 +118,7 @@ class CeleryTaskExecutionViewSet(CommonApiMixin, viewsets.ModelViewSet): serializer_class = CeleryTaskExecutionSerializer http_method_names = ('get', 'post', 'head', 'options',) queryset = CeleryTaskExecution.objects.all() + search_fields = ('name',) def get_queryset(self): task_id = self.request.query_params.get('task_id') diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index 108e812e1..3d1cf255c 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -26,6 +26,7 @@ def set_task_to_serializer_data(serializer, task): class JobViewSet(OrgBulkModelViewSet): serializer_class = JobSerializer permission_classes = () + search_fields = ('name', 'comment') model = Job def allow_bulk_destroy(self, qs, filtered): @@ -63,6 +64,7 @@ class JobExecutionViewSet(OrgBulkModelViewSet): http_method_names = ('get', 'post', 'head', 'options',) permission_classes = () model = JobExecution + search_fields = ('material',) def perform_create(self, serializer): instance = serializer.save() diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index ba2275206..44e7bbb19 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -26,6 +26,7 @@ class PlaybookViewSet(OrgBulkModelViewSet): serializer_class = PlaybookSerializer permission_classes = () model = Playbook + search_fields = ('name', 'comment') def allow_bulk_destroy(self, qs, filtered): return True From b81416d97358f40c629cdf69f1c3e8573785ce9c Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 19:02:45 +0800 Subject: [PATCH 32/44] =?UTF-8?q?perf:=20playbook=20=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E5=88=A0=E9=99=A4=20(#9490)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: feng <1304903146@qq.com> --- apps/ops/api/playbook.py | 3 --- apps/ops/urls/api_urls.py | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index 44e7bbb19..d25f2cb90 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -28,9 +28,6 @@ class PlaybookViewSet(OrgBulkModelViewSet): model = Playbook search_fields = ('name', 'comment') - def allow_bulk_destroy(self, qs, filtered): - return True - def get_queryset(self): queryset = super().get_queryset() queryset = queryset.filter(creator=self.request.user) diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index e637d84fb..051ca7894 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -12,10 +12,10 @@ app_name = "ops" router = DefaultRouter() bulk_router = BulkRouter() -router.register(r'adhocs', api.AdHocViewSet, 'adhoc') -router.register(r'playbooks', api.PlaybookViewSet, 'playbook') -router.register(r'jobs', api.JobViewSet, 'job') -router.register(r'job-executions', api.JobExecutionViewSet, 'job-execution') +bulk_router.register(r'adhocs', api.AdHocViewSet, 'adhoc') +bulk_router.register(r'playbooks', api.PlaybookViewSet, 'playbook') +bulk_router.register(r'jobs', api.JobViewSet, 'job') +bulk_router.register(r'job-executions', api.JobExecutionViewSet, 'job-execution') router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task') From 48067415ef5a3558baa0a550659195424b65c0d4 Mon Sep 17 00:00:00 2001 From: Bai Date: Thu, 9 Feb 2023 19:38:28 +0800 Subject: [PATCH 33/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=AC=AC?= =?UTF-8?q?=E4=B8=89=E6=96=B9=E7=94=A8=E6=88=B7=E7=99=BB=E5=BD=95=E5=A4=8D?= =?UTF-8?q?=E6=A0=B8=E6=97=B6=EF=BC=8C=E5=8F=AF=E4=BB=A5=E8=B7=B3=E8=BF=87?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/acls/serializers/login_acl.py | 2 +- apps/authentication/api/login_confirm.py | 1 + apps/authentication/middleware.py | 12 +++++++++++- apps/authentication/mixins.py | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/apps/acls/serializers/login_acl.py b/apps/acls/serializers/login_acl.py index 59b9cf0ad..fb826361a 100644 --- a/apps/acls/serializers/login_acl.py +++ b/apps/acls/serializers/login_acl.py @@ -22,7 +22,7 @@ class LoginACLSerializer(BulkModelSerializer): reviewers = ObjectRelatedField( queryset=User.objects, label=_("Reviewers"), many=True, required=False ) - action = LabeledChoiceField(choices=LoginACL.ActionChoices.choices) + action = LabeledChoiceField(choices=LoginACL.ActionChoices.choices, label=_('Action')) reviewers_amount = serializers.IntegerField( read_only=True, source="reviewers.count", label=_("Reviewers amount") ) diff --git a/apps/authentication/api/login_confirm.py b/apps/authentication/api/login_confirm.py index 866964677..d2e5eb14e 100644 --- a/apps/authentication/api/login_confirm.py +++ b/apps/authentication/api/login_confirm.py @@ -20,6 +20,7 @@ class TicketStatusApi(mixins.AuthMixin, APIView): try: self.check_user_login_confirm() self.request.session['auth_third_party_done'] = 1 + self.request.session.pop('auth_third_party_required', '') return Response({"msg": "ok"}) except errors.LoginConfirmOtherError as e: reason = e.msg diff --git a/apps/authentication/middleware.py b/apps/authentication/middleware.py index 8573b086b..d03798f94 100644 --- a/apps/authentication/middleware.py +++ b/apps/authentication/middleware.py @@ -62,6 +62,17 @@ class ThirdPartyLoginMiddleware(mixins.AuthMixin): return response if not request.session.get('auth_third_party_required'): return response + white_urls = [ + 'jsi18n/', '/static/', + 'login/guard', 'login/wait-confirm', + 'login-confirm-ticket/status', + 'settings/public/open', + 'core/auth/login', 'core/auth/logout' + ] + for url in white_urls: + if request.path.find(url) > -1: + return response + ip = get_request_ip(request) try: self.request = request @@ -89,7 +100,6 @@ class ThirdPartyLoginMiddleware(mixins.AuthMixin): guard_url = "%s?%s" % (guard_url, args) response = redirect(guard_url) finally: - request.session.pop('auth_third_party_required', '') return response diff --git a/apps/authentication/mixins.py b/apps/authentication/mixins.py index f381edecf..5355d79bd 100644 --- a/apps/authentication/mixins.py +++ b/apps/authentication/mixins.py @@ -369,7 +369,7 @@ class AuthACLMixin: def check_user_login_confirm(self): ticket = self.get_ticket() if not ticket: - raise errors.LoginConfirmOtherError('', "Not found") + raise errors.LoginConfirmOtherError('', "Not found", '') elif ticket.is_state(ticket.State.approved): self.request.session["auth_confirm_required"] = '' return From d0e56a17d6fe129c6b5f52cb06a6c4f185dddf1a Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 9 Feb 2023 18:37:15 +0800 Subject: [PATCH 34/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=AE=A1?= =?UTF-8?q?=E8=AE=A1=E6=97=A5=E5=BF=97=E4=BF=9D=E5=AD=98=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/audits/backends/db.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/audits/backends/db.py b/apps/audits/backends/db.py index b24efaaa4..aa60b3569 100644 --- a/apps/audits/backends/db.py +++ b/apps/audits/backends/db.py @@ -69,6 +69,12 @@ class OperateLogStore(object): before.update(op_before) after.update(op_after) else: + # 限制长度 128 OperateLog.resource.field.max_length + max_length = 128 + resource = kwargs.get('resource', '') + if isinstance(resource, str) and (len(resource) > max_length): + # 截取字符串 + kwargs.update({'resource': resource[:max_length]}) op_log = self.model(**kwargs) diff = self.convert_before_after_to_diff(before, after) From e36a64ae2c1a7cedcfea279f5b50df50e553cc21 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 9 Feb 2023 18:53:07 +0800 Subject: [PATCH 35/44] perf: update resource --- apps/audits/backends/db.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/audits/backends/db.py b/apps/audits/backends/db.py index aa60b3569..5acae53cc 100644 --- a/apps/audits/backends/db.py +++ b/apps/audits/backends/db.py @@ -69,12 +69,11 @@ class OperateLogStore(object): before.update(op_before) after.update(op_after) else: - # 限制长度 128 OperateLog.resource.field.max_length + # 限制长度 128 OperateLog.resource.field.max_length, 避免存储失败 max_length = 128 resource = kwargs.get('resource', '') - if isinstance(resource, str) and (len(resource) > max_length): - # 截取字符串 - kwargs.update({'resource': resource[:max_length]}) + if resource and isinstance(resource, str): + kwargs['resource'] = resource[:max_length] op_log = self.model(**kwargs) diff = self.convert_before_after_to_diff(before, after) From 1b1ae1145c18e46e748278a6e6848b30d3ac01be Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 20:05:26 +0800 Subject: [PATCH 36/44] =?UTF-8?q?fix:=20=E9=87=8D=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=89=8D=E6=A0=A1=E9=AA=8C=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E5=86=B2=E7=AA=81=20(#9488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aaron3S --- apps/ops/api/playbook.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index d25f2cb90..41edb0405 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -146,6 +146,8 @@ class PlaybookFileBrowserAPIView(APIView): if new_name: new_file_path = os.path.join(os.path.dirname(file_path), new_name) + if os.path.exists(new_file_path): + return Response({'msg': '{} already exists'.format(new_name)}, status=400) os.rename(file_path, new_file_path) file_path = new_file_path From 3c891ec313ceb216be7c503724807198e92e06d3 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 9 Feb 2023 20:11:12 +0800 Subject: [PATCH 37/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=9B=A0?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E6=8F=90=E4=BA=A4=E9=97=AE=E9=A2=98=E9=80=A0?= =?UTF-8?q?=E6=88=90=E7=9A=84=E4=BB=BB=E5=8A=A1=E6=97=A0=E6=B3=95=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E5=92=8C=E4=BB=BB=E5=8A=A1=E5=8E=86=E5=8F=B2=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E6=9F=A5=E7=9C=8B=20(#9493)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Aaron3S --- apps/ops/api/job.py | 2 ++ apps/ops/models/job.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index 3d1cf255c..0beaff278 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -1,4 +1,5 @@ from django.db.models import Count +from django.db.transaction import atomic from rest_framework.views import APIView from django.shortcuts import get_object_or_404 from rest_framework.response import Response @@ -66,6 +67,7 @@ class JobExecutionViewSet(OrgBulkModelViewSet): model = JobExecution search_fields = ('material',) + @atomic def perform_create(self, serializer): instance = serializer.save() instance.job_version = instance.job.version diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py index dd248e83d..529e1adda 100644 --- a/apps/ops/models/job.py +++ b/apps/ops/models/job.py @@ -235,6 +235,8 @@ class JobExecution(JMSOrgBaseModel): @property def time_cost(self): + if not self.date_start: + return 0 if self.is_finished: return (self.date_finished - self.date_start).total_seconds() return (timezone.now() - self.date_start).total_seconds() From 7926f7d75e3fe2a1419f93fe84c5132af6641949 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Thu, 9 Feb 2023 11:20:54 +0800 Subject: [PATCH 38/44] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9OAuth2=E5=8D=8F?= =?UTF-8?q?=E8=AE=AE=E7=9A=84=E6=B3=A8=E9=94=80=E5=9C=B0=E5=9D=80=E4=B8=BA?= =?UTF-8?q?=E9=9D=9E=E5=BF=85=E5=A1=AB=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/backends/oauth2/views.py | 4 ++-- apps/settings/serializers/auth/oauth2.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/authentication/backends/oauth2/views.py b/apps/authentication/backends/oauth2/views.py index 056f16b31..ad95e29cd 100644 --- a/apps/authentication/backends/oauth2/views.py +++ b/apps/authentication/backends/oauth2/views.py @@ -86,9 +86,9 @@ class OAuth2EndSessionView(View): logger.debug(log_prompt.format('Log out the current user: {}'.format(request.user))) auth.logout(request) - if settings.AUTH_OAUTH2_LOGOUT_COMPLETELY: + next_url = settings.AUTH_OAUTH2_PROVIDER_END_SESSION_ENDPOINT + if settings.AUTH_OAUTH2_LOGOUT_COMPLETELY and next_url: logger.debug(log_prompt.format('Log out OAUTH2 platform user session synchronously')) - next_url = settings.AUTH_OAUTH2_PROVIDER_END_SESSION_ENDPOINT return HttpResponseRedirect(next_url) logger.debug(log_prompt.format('Redirect')) diff --git a/apps/settings/serializers/auth/oauth2.py b/apps/settings/serializers/auth/oauth2.py index a689243ef..46e279969 100644 --- a/apps/settings/serializers/auth/oauth2.py +++ b/apps/settings/serializers/auth/oauth2.py @@ -49,7 +49,7 @@ class OAuth2SettingSerializer(serializers.Serializer): required=True, max_length=1024, label=_('Provider userinfo endpoint') ) AUTH_OAUTH2_PROVIDER_END_SESSION_ENDPOINT = serializers.CharField( - required=False, max_length=1024, label=_('Provider end session endpoint') + required=False, allow_blank=True, max_length=1024, label=_('Provider end session endpoint') ) AUTH_OAUTH2_LOGOUT_COMPLETELY = serializers.BooleanField(required=False, label=_('Logout completely')) AUTH_OAUTH2_USER_ATTR_MAP = serializers.DictField( From c5b8ae6c9a338b7f2561878cfbe222c864c8a62a Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Thu, 9 Feb 2023 11:22:58 +0800 Subject: [PATCH 39/44] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=E5=8F=98?= =?UTF-8?q?=E9=87=8F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/backends/oauth2/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/authentication/backends/oauth2/views.py b/apps/authentication/backends/oauth2/views.py index ad95e29cd..e3b919fff 100644 --- a/apps/authentication/backends/oauth2/views.py +++ b/apps/authentication/backends/oauth2/views.py @@ -86,10 +86,10 @@ class OAuth2EndSessionView(View): logger.debug(log_prompt.format('Log out the current user: {}'.format(request.user))) auth.logout(request) - next_url = settings.AUTH_OAUTH2_PROVIDER_END_SESSION_ENDPOINT - if settings.AUTH_OAUTH2_LOGOUT_COMPLETELY and next_url: + logout_url = settings.AUTH_OAUTH2_PROVIDER_END_SESSION_ENDPOINT + if settings.AUTH_OAUTH2_LOGOUT_COMPLETELY and logout_url: logger.debug(log_prompt.format('Log out OAUTH2 platform user session synchronously')) - return HttpResponseRedirect(next_url) + return HttpResponseRedirect(logout_url) logger.debug(log_prompt.format('Redirect')) return HttpResponseRedirect(logout_url) From 4c96d6935ad7cc6ef6945c976f12cd1c0bf180f1 Mon Sep 17 00:00:00 2001 From: Bai Date: Fri, 10 Feb 2023 11:12:03 +0800 Subject: [PATCH 40/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=20push=5Fnow=20=E6=9D=83=E9=99=90=E7=9A=84=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E5=8F=AF=E4=BB=A5=E5=9C=A8=E5=88=9B=E5=BB=BA=E8=B5=84?= =?UTF-8?q?=E4=BA=A7=E3=80=81=E5=88=9B=E5=BB=BA=E8=B4=A6=E5=8F=B7=E4=B8=AD?= =?UTF-8?q?=E5=BC=80=E5=90=AF=E6=8E=A8=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/serializers/asset/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index 974762364..732fbe55d 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -76,6 +76,12 @@ class AssetAccountSerializer( 'secret': {'write_only': True}, } + def validate_push_now(self, value): + request = self.context['request'] + if not request.user.has_perms('assets.push_assetaccount'): + return False + return value + def validate_name(self, value): if not value: value = self.initial_data.get('username') From fcf20335e8d1123fe2d3d47724ce3ab2397764b3 Mon Sep 17 00:00:00 2001 From: Bai Date: Fri, 10 Feb 2023 11:57:58 +0800 Subject: [PATCH 41/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E8=A7=92=E8=89=B2=E6=9D=83=E9=99=90=E3=80=81=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E8=A7=92=E8=89=B2=E3=80=81=E7=94=A8=E6=88=B7=E7=BB=91?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=EF=BC=8C=E7=94=A8=E6=88=B7=E6=9D=83=E9=99=90?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=8D=B3=E4=BD=BF=E6=9B=B4=E6=96=B0=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/rbac/models/rolebinding.py | 8 ++++++++ apps/rbac/signal_handlers.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/apps/rbac/models/rolebinding.py b/apps/rbac/models/rolebinding.py index b7795a498..f8f21d2bf 100644 --- a/apps/rbac/models/rolebinding.py +++ b/apps/rbac/models/rolebinding.py @@ -3,6 +3,7 @@ from django.db import models from django.db.models import Q from django.conf import settings from django.core.exceptions import ValidationError +from django.db.models.signals import post_save from rest_framework.serializers import ValidationError from common.db.models import JMSBaseModel, CASCADE_SIGNAL_SKIP @@ -15,6 +16,13 @@ __all__ = ['RoleBinding', 'SystemRoleBinding', 'OrgRoleBinding'] class RoleBindingManager(models.Manager): + + def bulk_create(self, objs, batch_size=None, ignore_conflicts=False): + objs = super().bulk_create(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts) + for i in objs: + post_save.send(i.__class__, instance=i, created=True) + return objs + def get_queryset(self): queryset = super(RoleBindingManager, self).get_queryset() q = Q(scope=Scope.system, org__isnull=True) diff --git a/apps/rbac/signal_handlers.py b/apps/rbac/signal_handlers.py index 37eed870a..bdc1e4a53 100644 --- a/apps/rbac/signal_handlers.py +++ b/apps/rbac/signal_handlers.py @@ -1,8 +1,8 @@ from django.dispatch import receiver -from django.db.models.signals import post_migrate, post_save +from django.db.models.signals import post_migrate, post_save, m2m_changed, post_delete from django.apps import apps -from .models import SystemRole, OrgRole +from .models import SystemRole, OrgRole, OrgRoleBinding, SystemRoleBinding from .builtin import BuiltinRole @@ -21,7 +21,32 @@ def on_system_role_update(sender, instance, created, **kwargs): User.expire_users_rbac_perms_cache() +@receiver(m2m_changed, sender=SystemRole.permissions.through) +def on_system_role_permission_changed(sender, instance, action, **kwargs): + from users.models import User + User.expire_users_rbac_perms_cache() + + +@receiver([post_save, post_delete], sender=SystemRoleBinding) +def on_system_role_binding_update(sender, instance, created, **kwargs): + from users.models import User + User.expire_users_rbac_perms_cache() + + @receiver(post_save, sender=OrgRole) def on_org_role_update(sender, instance, created, **kwargs): from users.models import User User.expire_users_rbac_perms_cache() + + +@receiver(m2m_changed, sender=OrgRole.permissions.through) +def on_org_role_permission_changed(sender, instance, action, **kwargs): + from users.models import User + User.expire_users_rbac_perms_cache() + + +@receiver([post_save, post_delete], sender=OrgRoleBinding) +def on_org_role_binding_update(sender, instance, **kwargs): + print('>>>>>>>>>>>') + from users.models import User + User.expire_users_rbac_perms_cache() From 0c11a602a8529d4eea7ec18151424a8255c81bd8 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:07:20 +0800 Subject: [PATCH 42/44] perf: push account automation (#9497) Co-authored-by: feng <1304903146@qq.com> --- .../automations/change_secret/host/windows/main.yml | 9 ++++++++- apps/accounts/automations/change_secret/manager.py | 2 ++ apps/accounts/automations/gather_accounts/filter.py | 4 ++++ .../automations/gather_accounts/host/posix/main.yml | 9 +++++++-- apps/accounts/automations/push_account/manager.py | 1 + apps/assets/automations/base/manager.py | 2 ++ apps/ops/api/job.py | 2 +- 7 files changed, 25 insertions(+), 4 deletions(-) diff --git a/apps/accounts/automations/change_secret/host/windows/main.yml b/apps/accounts/automations/change_secret/host/windows/main.yml index 9ba28c075..66efb0801 100644 --- a/apps/accounts/automations/change_secret/host/windows/main.yml +++ b/apps/accounts/automations/change_secret/host/windows/main.yml @@ -8,11 +8,18 @@ # debug: # msg: "Username: {{ account.username }}, Password: {{ account.secret }}" + + - name: Get groups of a Windows user + ansible.windows.win_user: + name: "{{ jms_account.username }}" + register: user_info + - name: Change password ansible.windows.win_user: name: "{{ account.username }}" password: "{{ account.secret }}" - groups: "Administrators" + groups: "{{ user_info.groups[0].name }}" + groups_action: add update_password: always when: account.secret_type == "password" diff --git a/apps/accounts/automations/change_secret/manager.py b/apps/accounts/automations/change_secret/manager.py index 411506fe7..ea5641a18 100644 --- a/apps/accounts/automations/change_secret/manager.py +++ b/apps/accounts/automations/change_secret/manager.py @@ -22,6 +22,8 @@ logger = get_logger(__name__) class ChangeSecretManager(AccountBasePlaybookManager): + ansible_account_prefer = '' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.method_hosts_mapper = defaultdict(list) diff --git a/apps/accounts/automations/gather_accounts/filter.py b/apps/accounts/automations/gather_accounts/filter.py index ebaf6d9b1..f3c9e583d 100644 --- a/apps/accounts/automations/gather_accounts/filter.py +++ b/apps/accounts/automations/gather_accounts/filter.py @@ -30,6 +30,10 @@ class GatherAccountsFilter: result = {} for line in info: data = line.split('@') + if len(data) == 1: + result[line] = {} + continue + if len(data) != 3: continue username, address, dt = data diff --git a/apps/accounts/automations/gather_accounts/host/posix/main.yml b/apps/accounts/automations/gather_accounts/host/posix/main.yml index a64323f9d..910b1213d 100644 --- a/apps/accounts/automations/gather_accounts/host/posix/main.yml +++ b/apps/accounts/automations/gather_accounts/host/posix/main.yml @@ -4,8 +4,13 @@ - name: Gather posix account ansible.builtin.shell: cmd: > - users=$(getent passwd | grep -v nologin | grep -v shutdown | awk -F":" '{ print $1 }');for i in $users; - do last -w -F $i -1 | head -1 | grep -v ^$ | awk '{ print $1"@"$3"@"$5,$6,$7,$8 }';done + users=$(getent passwd | grep -v nologin | grep -v shutdown | awk -F":" '{ print $1 }');for i in $users; + do k=$(last -w -F $i -1 | head -1 | grep -v ^$ | awk '{ print $1"@"$3"@"$5,$6,$7,$8 }') + if [ -n "$k" ]; then + echo $k + else + echo $i + fi;done register: result - name: Define info by set_fact diff --git a/apps/accounts/automations/push_account/manager.py b/apps/accounts/automations/push_account/manager.py index 4eb047ed0..edcb2856b 100644 --- a/apps/accounts/automations/push_account/manager.py +++ b/apps/accounts/automations/push_account/manager.py @@ -12,6 +12,7 @@ logger = get_logger(__name__) class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager): + ansible_account_prefer = '' @classmethod def method_type(cls): diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 6247c85d7..e51bfcd83 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -25,6 +25,7 @@ class PlaybookCallback(DefaultCallback): class BasePlaybookManager: bulk_size = 100 ansible_account_policy = 'privileged_first' + ansible_account_prefer = 'root,Administrator' def __init__(self, execution): self.execution = execution @@ -123,6 +124,7 @@ class BasePlaybookManager: def generate_inventory(self, platformed_assets, inventory_path): inventory = JMSInventory( assets=platformed_assets, + account_prefer=self.ansible_account_prefer, account_policy=self.ansible_account_policy, host_callback=self.host_callback, ) diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index 0beaff278..cbb88360a 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -14,7 +14,7 @@ __all__ = ['JobViewSet', 'JobExecutionViewSet', 'JobRunVariableHelpAPIView', from ops.tasks import run_ops_job_execution from ops.variables import JMS_JOB_VARIABLE_HELP from orgs.mixins.api import OrgBulkModelViewSet -from orgs.utils import tmp_to_org, get_current_org_id, get_current_org +from orgs.utils import tmp_to_org, get_current_org from accounts.models import Account From e36506c3b8da0482a51a62e578f24d664e2a744d Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:37:55 +0800 Subject: [PATCH 43/44] perf: ansible log (#9499) Co-authored-by: feng <1304903146@qq.com> --- apps/assets/automations/base/manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index e51bfcd83..9c7ac4fd1 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -174,7 +174,7 @@ class BasePlaybookManager: pass def on_host_error(self, host, error, result): - pass + print('host error: {} -> {}'.format(host, error)) def on_runner_success(self, runner, cb): summary = cb.summary @@ -200,8 +200,11 @@ class BasePlaybookManager: runners = self.get_runners() if len(runners) > 1: print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) - else: + elif len(runners) == 1: print(">>> 开始执行任务\n") + else: + print("### 没有需要执行的任务\n") + return self.execution.date_start = timezone.now() for i, runner in enumerate(runners, start=1): From e647205c24bead2d8c44cb86c6902404bd2fe1d9 Mon Sep 17 00:00:00 2001 From: jiangweidong Date: Fri, 10 Feb 2023 15:03:21 +0800 Subject: [PATCH 44/44] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96applet-navicat?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=96=B9=E5=BC=8F=20(#9498)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 优化applet-navicat连接方式 * perf: 关闭许可证通知，不阻塞登录 --- apps/terminal/applets/navicat/app.py | 359 +++++++++++++++---------- apps/terminal/applets/navicat/const.py | 7 + 2 files changed, 224 insertions(+), 142 deletions(-) create mode 100644 apps/terminal/applets/navicat/const.py diff --git a/apps/terminal/applets/navicat/app.py b/apps/terminal/applets/navicat/app.py index 8f2b779da..17d3b1579 100644 --- a/apps/terminal/applets/navicat/app.py +++ b/apps/terminal/applets/navicat/app.py @@ -7,9 +7,11 @@ if sys.platform == 'win32': import win32api from pywinauto import Application - from pywinauto.controls.uia_controls import ( - EditWrapper, ComboBoxWrapper, ButtonWrapper - ) + from pywinauto.controls.uia_controls import ButtonWrapper + from pywinauto.keyboard import send_keys + +import const as c + from common import wait_pid, BaseApplication _default_path = r'C:\Program Files\PremiumSoft\Navicat Premium 16\navicat.exe' @@ -30,17 +32,16 @@ class AppletApplication(BaseApplication): self.app = None def clean_up(self): - protocol_mapping = { - 'mariadb': 'NavicatMARIADB', 'mongodb': 'NavicatMONGODB', - 'mysql': 'Navicat', 'oracle': 'NavicatORA', - 'sqlserver': 'NavicatMSSQL', 'postgresql': 'NavicatPG' - } - protocol_display = protocol_mapping.get(self.protocol, 'mysql') - sub_key = r'Software\PremiumSoft\%s\Servers' % protocol_display - try: - win32api.RegDeleteTree(winreg.HKEY_CURRENT_USER, sub_key) - except Exception as err: - print('Error: %s' % err) + protocols = ( + 'NavicatMARIADB', 'NavicatMONGODB', 'Navicat', + 'NavicatORA', 'NavicatMSSQL', 'NavicatPG' + ) + for p in protocols: + sub_key = r'Software\PremiumSoft\%s\Servers' % p + try: + win32api.RegDeleteTree(winreg.HKEY_CURRENT_USER, sub_key) + except Exception: + pass @staticmethod def launch(): @@ -51,135 +52,208 @@ class AppletApplication(BaseApplication): winreg.SetValueEx(key, 'AlreadyShowNavicatV16WelcomeScreen', 0, winreg.REG_DWORD, 1) # 禁止开启自动检查更新 winreg.SetValueEx(key, 'AutoCheckUpdate', 0, winreg.REG_DWORD, 0) + # 禁止弹出初始化界面 winreg.SetValueEx(key, 'ShareUsageData', 0, winreg.REG_DWORD, 0) except Exception as err: print('Launch error: %s' % err) - def _fill_to_mysql(self, app, menu, protocol_display='MySQL'): - menu.item_by_path('File->New Connection->%s' % protocol_display).click_input() - conn_window = app.window(best_match='Dialog').child_window(title_re='New Connection') + @staticmethod + def _exec_commands(commands): + for command in commands: + if command['type'] == 'key': + time.sleep(0.5) + send_keys(' '.join(command['commands'])) + elif command['type'] == 'action': + for f in command['commands']: + f() - name_ele = conn_window.child_window(best_match='Edit5') - EditWrapper(name_ele.element_info).set_edit_text(self.name) + def _action_not_remember_password(self): + conn_window = self.app.window(best_match='Dialog'). \ + child_window(title_re='New Connection') + remember_checkbox = conn_window.child_window(best_match='Save password') + remember_checkbox.click() - host_ele = conn_window.child_window(best_match='Edit4') - EditWrapper(host_ele.element_info).set_edit_text(self.host) + def _get_mysql_commands(self): + commands = [ + { + 'type': 'key', + 'commands': [ + '%f', c.DOWN, c.RIGHT, c.ENTER + ], + }, + { + 'type': 'key', + 'commands': [ + self.name, c.TAB, self.host, c.TAB, + str(self.port), c.TAB, self.username, + ] + }, + { + 'type': 'action', + 'commands': [ + self._action_not_remember_password + ] + }, + { + 'type': 'key', + 'commands': [c.ENTER] + } + ] + return commands - port_ele = conn_window.child_window(best_match='Edit2') - EditWrapper(port_ele.element_info).set_edit_text(self.port) + def _get_mariadb_commands(self): + commands = [ + { + 'type': 'key', + 'commands': [ + '%f', c.DOWN, c.RIGHT, c.DOWN * 5, c.ENTER, + ], + }, + { + 'type': 'key', + 'commands': [ + self.name, c.TAB, self.host, c.TAB, + str(self.port), c.TAB, self.username + ] + }, + { + 'type': 'action', + 'commands': [ + self._action_not_remember_password + ] + }, + { + 'type': 'key', + 'commands': [c.ENTER] + } + ] + return commands - username_ele = conn_window.child_window(best_match='Edit1') - EditWrapper(username_ele.element_info).set_edit_text(self.username) + def _get_mongodb_commands(self): + commands = [ + { + 'type': 'key', + 'commands': [ + '%f', c.DOWN, c.RIGHT, c.DOWN * 6, c.ENTER, + ], + }, + { + 'type': 'key', + 'commands': [ + self.name, c.TAB * 3, self.host, c.TAB, str(self.port), + c.TAB, c.DOWN, c.TAB, self.db, c.TAB, self.username, + ] + }, + { + 'type': 'action', + 'commands': [ + self._action_not_remember_password + ] + }, + { + 'type': 'key', + 'commands': [c.ENTER] + } + ] + return commands - password_ele = conn_window.child_window(best_match='Edit3') - EditWrapper(password_ele.element_info).set_edit_text(self.password) + def _get_postgresql_commands(self): + commands = [ + { + 'type': 'key', + 'commands': [ + '%f', c.DOWN, c.RIGHT, c.DOWN, c.ENTER, + ], + }, + { + 'type': 'key', + 'commands': [ + self.name, c.TAB, self.host, c.TAB, str(self.port), + c.TAB, self.db, c.TAB, self.username + ] + }, + { + 'type': 'action', + 'commands': [ + self._action_not_remember_password + ] + }, + { + 'type': 'key', + 'commands': [c.ENTER] + } + ] + return commands - def _fill_to_mariadb(self, app, menu): - self._fill_to_mysql(app, menu, 'MariaDB') - - def _fill_to_mongodb(self, app, menu): - menu.item_by_path('File->New Connection->MongoDB').click_input() - conn_window = app.window(best_match='Dialog').child_window(title_re='New Connection') - - auth_type_ele = conn_window.child_window(best_match='ComboBox2') - ComboBoxWrapper(auth_type_ele.element_info).select('Password') - - name_ele = conn_window.child_window(best_match='Edit5') - EditWrapper(name_ele.element_info).set_edit_text(self.name) - - host_ele = conn_window.child_window(best_match='Edit4') - EditWrapper(host_ele.element_info).set_edit_text(self.host) - - port_ele = conn_window.child_window(best_match='Edit2') - EditWrapper(port_ele.element_info).set_edit_text(self.port) - - db_ele = conn_window.child_window(best_match='Edit6') - EditWrapper(db_ele.element_info).set_edit_text(self.db) - - username_ele = conn_window.child_window(best_match='Edit1') - EditWrapper(username_ele.element_info).set_edit_text(self.username) - - password_ele = conn_window.child_window(best_match='Edit3') - EditWrapper(password_ele.element_info).set_edit_text(self.password) - - def _fill_to_postgresql(self, app, menu): - menu.item_by_path('File->New Connection->PostgreSQL').click_input() - conn_window = app.window(best_match='Dialog').child_window(title_re='New Connection') - - name_ele = conn_window.child_window(best_match='Edit6') - EditWrapper(name_ele.element_info).set_edit_text(self.name) - - host_ele = conn_window.child_window(best_match='Edit5') - EditWrapper(host_ele.element_info).set_edit_text(self.host) - - port_ele = conn_window.child_window(best_match='Edit2') - EditWrapper(port_ele.element_info).set_edit_text(self.port) - - db_ele = conn_window.child_window(best_match='Edit4') - EditWrapper(db_ele.element_info).set_edit_text(self.db) - - username_ele = conn_window.child_window(best_match='Edit1') - EditWrapper(username_ele.element_info).set_edit_text(self.username) - - password_ele = conn_window.child_window(best_match='Edit3') - EditWrapper(password_ele.element_info).set_edit_text(self.password) - - def _fill_to_sqlserver(self, app, menu): - menu.item_by_path('File->New Connection->SQL Server').click_input() - conn_window = app.window(best_match='Dialog').child_window(title_re='New Connection') - - name_ele = conn_window.child_window(best_match='Edit5') - EditWrapper(name_ele.element_info).set_edit_text(self.name) - - host_ele = conn_window.child_window(best_match='Edit4') - EditWrapper(host_ele.element_info).set_edit_text('%s,%s' % (self.host, self.port)) - - db_ele = conn_window.child_window(best_match='Edit3') - EditWrapper(db_ele.element_info).set_edit_text(self.db) - - username_ele = conn_window.child_window(best_match='Edit6') - EditWrapper(username_ele.element_info).set_edit_text(self.username) - - password_ele = conn_window.child_window(best_match='Edit2') - EditWrapper(password_ele.element_info).set_edit_text(self.password) - - def _fill_to_oracle(self, app, menu): - menu.item_by_path('File->New Connection->Oracle').click_input() - conn_window = app.window(best_match='Dialog').child_window(title_re='New Connection') - - name_ele = conn_window.child_window(best_match='Edit6') - EditWrapper(name_ele.element_info).set_edit_text(self.name) - - host_ele = conn_window.child_window(best_match='Edit5') - EditWrapper(host_ele.element_info).set_edit_text(self.host) - - port_ele = conn_window.child_window(best_match='Edit3') - EditWrapper(port_ele.element_info).set_edit_text(self.port) - - db_ele = conn_window.child_window(best_match='Edit2') - EditWrapper(db_ele.element_info).set_edit_text(self.db) - - username_ele = conn_window.child_window(best_match='Edit') - EditWrapper(username_ele.element_info).set_edit_text(self.username) - - password_ele = conn_window.child_window(best_match='Edit4') - EditWrapper(password_ele.element_info).set_edit_text(self.password) + def _get_sqlserver_commands(self): + commands = [ + { + 'type': 'key', + 'commands': [ + '%f', c.DOWN, c.RIGHT, c.DOWN * 4, c.ENTER, + ], + }, + { + 'type': 'key', + 'commands': [ + self.name, c.TAB, '%s,%s' % (self.host, self.port), + c.TAB * 2, self.db, c.TAB * 2, self.username + ] + }, + { + 'type': 'action', + 'commands': [ + self._action_not_remember_password + ] + }, + { + 'type': 'key', + 'commands': [c.ENTER] + } + ] + return commands + def _get_oracle_commands(self): + commands = [ + { + 'type': 'key', + 'commands': [ + '%f', c.DOWN, c.RIGHT, c.DOWN * 2, c.ENTER, + ], + }, + { + 'type': 'key', + 'commands': [ + self.name, c.TAB * 2, self.host, c.TAB, + str(self.port), c.TAB, self.db, c.TAB, c.TAB, self.username, + ] + }, + { + 'type': 'action', + 'commands': (self._action_not_remember_password,) + }, + { + 'type': 'key', + 'commands': [c.ENTER] + } + ] if self.privileged: - conn_window.child_window(best_match='Advanced', control_type='TabItem').click_input() - role_ele = conn_window.child_window(best_match='ComboBox2') - ComboBoxWrapper(role_ele.element_info).select('SYSDBA') + commands.insert(3, { + 'type': 'key', + 'commands': (c.TAB * 4, c.RIGHT, c.TAB * 3, c.DOWN) + }) + return commands def run(self): self.launch() - app = Application(backend='uia') + self.app = Application(backend='uia') work_dir = os.path.dirname(self.path) - app.start(self.path, work_dir=work_dir) - self.pid = app.process + self.app.start(self.path, work_dir=work_dir) + self.pid = self.app.process # 检测是否为试用版本 try: - trial_btn = app.top_window().child_window( + trial_btn = self.app.top_window().child_window( best_match='Trial', control_type='Button' ) ButtonWrapper(trial_btn.element_info).click() @@ -187,26 +261,27 @@ class AppletApplication(BaseApplication): except Exception: pass - menubar = app.window(best_match='Navicat Premium', control_type='Window') \ - .child_window(best_match='Menu', control_type='MenuBar') - - file = menubar.child_window(best_match='File', control_type='MenuItem') - file.click_input() - menubar.item_by_path('File->New Connection').click_input() - - # 根据协议选择动作 - action = getattr(self, '_fill_to_%s' % self.protocol, None) + # 根据协议获取相应操作命令 + action = getattr(self, '_get_%s_commands' % self.protocol, None) if action is None: raise ValueError('This protocol is not supported: %s' % self.protocol) - action(app, menubar) - - conn_window = app.window(best_match='Dialog').child_window(title_re='New Connection') - ok_btn = conn_window.child_window(best_match='OK', control_type='Button') - ok_btn.click() - - file.click_input() - menubar.item_by_path('File->Open Connection').click_input() - self.app = app + commands = action() + # 关闭掉桌面许可弹框 + commands.insert(0, {'type': 'key', 'commands': (c.ENTER,)}) + # 登录 + commands.extend([ + { + 'type': 'key', + 'commands': ( + '%f', c.DOWN * 5, c.ENTER + ) + }, + { + 'type': 'key', + 'commands': (self.password, c.ENTER) + } + ]) + self._exec_commands(commands) def wait(self): try: diff --git a/apps/terminal/applets/navicat/const.py b/apps/terminal/applets/navicat/const.py new file mode 100644 index 000000000..68b111845 --- /dev/null +++ b/apps/terminal/applets/navicat/const.py @@ -0,0 +1,7 @@ + +UP = '{UP}' +LEFT = '{LEFT}' +DOWN = '{DOWN}' +RIGHT = '{RIGHT}' +TAB = '{VK_TAB}' +ENTER = '{VK_RETURN}'