diff --git a/README.md b/README.md index 3a220c96f..cf2b97ec5 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Jumpserver采纳分布式架构,支持多机房跨区域部署,中心节点 我们提供了DEMO和截图可以让你快速了解Jumpserver -[DEMO](http://demo.jumpserver.org) +[DEMO](https://demo.jumpserver.org) [截图](http://docs.jumpserver.org/zh/docs/snapshot.html) ### SDK diff --git a/apps/__init__.py b/apps/__init__.py index 414812ccc..27c6e5e83 100644 --- a/apps/__init__.py +++ b/apps/__init__.py @@ -2,4 +2,4 @@ # -*- coding: utf-8 -*- # -__version__ = "1.4.4" +__version__ = "1.4.6" diff --git a/apps/assets/api/admin_user.py b/apps/assets/api/admin_user.py index 8d30ee9d9..263d669fd 100644 --- a/apps/assets/api/admin_user.py +++ b/apps/assets/api/admin_user.py @@ -14,6 +14,7 @@ # limitations under the License. from django.db import transaction +from django.shortcuts import get_object_or_404 from rest_framework import generics from rest_framework.response import Response from rest_framework_bulk import BulkModelViewSet @@ -24,13 +25,14 @@ from common.utils import get_logger from ..hands import IsOrgAdmin from ..models import AdminUser, Asset from .. import serializers -from ..tasks import test_admin_user_connectability_manual +from ..tasks import test_admin_user_connectivity_manual logger = get_logger(__file__) __all__ = [ 'AdminUserViewSet', 'ReplaceNodesAdminUserApi', 'AdminUserTestConnectiveApi', 'AdminUserAuthApi', + 'AdminUserAssetsListView', ] @@ -81,12 +83,29 @@ class ReplaceNodesAdminUserApi(generics.UpdateAPIView): class AdminUserTestConnectiveApi(generics.RetrieveAPIView): """ - Test asset admin user connectivity + Test asset admin user assets_connectivity """ queryset = AdminUser.objects.all() permission_classes = (IsOrgAdmin,) def retrieve(self, request, *args, **kwargs): admin_user = self.get_object() - task = test_admin_user_connectability_manual.delay(admin_user) + task = test_admin_user_connectivity_manual.delay(admin_user) return Response({"task": task.id}) + + +class AdminUserAssetsListView(generics.ListAPIView): + permission_classes = (IsOrgAdmin,) + serializer_class = serializers.AssetSimpleSerializer + pagination_class = LimitOffsetPagination + filter_fields = ("hostname", "ip") + http_method_names = ['get'] + search_fields = filter_fields + + def get_object(self): + pk = self.kwargs.get('pk') + return get_object_or_404(AdminUser, pk=pk) + + def get_queryset(self): + admin_user = self.get_object() + return admin_user.get_related_assets() diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index 92a1775d0..cd343b2a5 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -17,7 +17,7 @@ from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser from ..models import Asset, AdminUser, Node from .. import serializers from ..tasks import update_asset_hardware_info_manual, \ - test_asset_connectability_manual + test_asset_connectivity_manual from ..utils import LabelFilter @@ -41,40 +41,46 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet): pagination_class = LimitOffsetPagination permission_classes = (IsOrgAdminOrAppUser,) - def filter_node(self): + def filter_node(self, queryset): node_id = self.request.query_params.get("node_id") if not node_id: - return + return queryset node = get_object_or_404(Node, id=node_id) show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true') - if node.is_root(): - if show_current_asset: - self.queryset = self.queryset.filter( - Q(nodes=node_id) | Q(nodes__isnull=True) - ) - return - if show_current_asset: - self.queryset = self.queryset.filter(nodes=node) + if node.is_root() and show_current_asset: + queryset = queryset.filter( + Q(nodes=node_id) | Q(nodes__isnull=True) + ) + elif node.is_root() and not show_current_asset: + pass + elif not node.is_root() and show_current_asset: + queryset = queryset.filter(nodes=node) else: - self.queryset = self.queryset.filter( + queryset = queryset.filter( nodes__key__regex='^{}(:[0-9]+)*$'.format(node.key), ) + return queryset - def filter_admin_user_id(self): + def filter_admin_user_id(self, queryset): admin_user_id = self.request.query_params.get('admin_user_id') - if admin_user_id: - admin_user = get_object_or_404(AdminUser, id=admin_user_id) - self.queryset = self.queryset.filter(admin_user=admin_user) + if not admin_user_id: + return queryset + admin_user = get_object_or_404(AdminUser, id=admin_user_id) + queryset = queryset.filter(admin_user=admin_user) + return queryset + + def filter_queryset(self, queryset): + queryset = super().filter_queryset(queryset) + queryset = self.filter_node(queryset) + queryset = self.filter_admin_user_id(queryset) + return queryset def get_queryset(self): - self.queryset = super().get_queryset()\ - .prefetch_related('labels', 'nodes')\ - .select_related('admin_user') - self.filter_admin_user_id() - self.filter_node() - return self.queryset.distinct() + queryset = super().get_queryset().distinct() + queryset = self.get_serializer_class().setup_eager_loading(queryset) + return queryset class AssetListUpdateApi(IDInFilterMixin, ListBulkCreateUpdateDestroyAPIView): @@ -103,7 +109,7 @@ class AssetRefreshHardwareApi(generics.RetrieveAPIView): class AssetAdminUserTestApi(generics.RetrieveAPIView): """ - Test asset admin user connectivity + Test asset admin user assets_connectivity """ queryset = Asset.objects.all() permission_classes = (IsOrgAdmin,) @@ -111,7 +117,7 @@ class AssetAdminUserTestApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): asset_id = kwargs.get('pk') asset = get_object_or_404(Asset, pk=asset_id) - task = test_asset_connectability_manual.delay(asset) + task = test_asset_connectivity_manual.delay(asset) return Response({"task": task.id}) diff --git a/apps/assets/api/domain.py b/apps/assets/api/domain.py index 721c149b7..b9c4aa39b 100644 --- a/apps/assets/api/domain.py +++ b/apps/assets/api/domain.py @@ -9,7 +9,6 @@ from django.views.generic.detail import SingleObjectMixin from common.utils import get_logger from common.permissions import IsOrgAdmin, IsAppUser, IsOrgAdminOrAppUser from ..models import Domain, Gateway -from ..utils import test_gateway_connectability from .. import serializers @@ -54,7 +53,7 @@ class GatewayTestConnectionApi(SingleObjectMixin, APIView): def get(self, request, *args, **kwargs): self.object = self.get_object(Gateway.objects.all()) - ok, e = test_gateway_connectability(self.object) + ok, e = self.object.test_connective() if ok: return Response("ok") else: diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 831c85e8a..4fc030e1e 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -17,15 +17,14 @@ from rest_framework import generics, mixins, viewsets from rest_framework.serializers import ValidationError from rest_framework.views import APIView from rest_framework.response import Response -from rest_framework_bulk import BulkModelViewSet from django.utils.translation import ugettext_lazy as _ from django.shortcuts import get_object_or_404 -from django.db.models import Count from common.utils import get_logger, get_object_or_none +from common.tree import TreeNodeSerializer from ..hands import IsOrgAdmin from ..models import Node -from ..tasks import update_assets_hardware_info_util, test_asset_connectability_util +from ..tasks import update_assets_hardware_info_util, test_asset_connectivity_util from .. import serializers @@ -34,7 +33,8 @@ __all__ = [ 'NodeViewSet', 'NodeChildrenApi', 'NodeAssetsApi', 'NodeAddAssetsApi', 'NodeRemoveAssetsApi', 'NodeReplaceAssetsApi', 'NodeAddChildrenApi', 'RefreshNodeHardwareInfoApi', - 'TestNodeConnectiveApi' + 'TestNodeConnectiveApi', 'NodeListAsTreeApi', + 'NodeChildrenAsTreeApi', 'RefreshAssetsAmount', ] @@ -43,22 +43,89 @@ class NodeViewSet(viewsets.ModelViewSet): permission_classes = (IsOrgAdmin,) serializer_class = serializers.NodeSerializer - def perform_create(self, serializer): - child_key = Node.root().get_next_child_key() - serializer.validated_data["key"] = child_key - serializer.save() - def update(self, request, *args, **kwargs): - node = self.get_object() - if node.is_root(): - node_value = node.value - post_value = request.data.get('value') - if node_value != post_value: - return Response( - {"msg": _("You can't update the root node name")}, - status=400 - ) - return super().update(request, *args, **kwargs) +class NodeListAsTreeApi(generics.ListAPIView): + """ + 获取节点列表树 + [ + { + "id": "", + "name": "", + "pId": "", + "meta": "" + } + ] + """ + permission_classes = (IsOrgAdmin,) + serializer_class = TreeNodeSerializer + + def get_queryset(self): + queryset = [node.as_tree_node() for node in Node.objects.all()] + return queryset + + def filter_queryset(self, queryset): + if self.request.query_params.get('refresh', '0') == '1': + queryset = self.refresh_nodes(queryset) + return queryset + + @staticmethod + def refresh_nodes(queryset): + Node.expire_nodes_assets_amount() + Node.expire_nodes_full_value() + return queryset + + +class NodeChildrenAsTreeApi(generics.ListAPIView): + """ + 节点子节点作为树返回, + [ + { + "id": "", + "name": "", + "pId": "", + "meta": "" + } + ] + + """ + permission_classes = (IsOrgAdmin,) + serializer_class = TreeNodeSerializer + node = None + is_root = False + + def get_queryset(self): + node_key = self.request.query_params.get('key') + if node_key: + self.node = Node.objects.get(key=node_key) + queryset = self.node.get_children(with_self=False) + else: + self.is_root = True + self.node = Node.root() + queryset = list(self.node.get_children(with_self=True)) + nodes_invalid = Node.objects.exclude(key__startswith=self.node.key) + queryset.extend(list(nodes_invalid)) + queryset = [node.as_tree_node() for node in queryset] + return queryset + + def filter_assets(self, queryset): + include_assets = self.request.query_params.get('assets', '0') == '1' + if not include_assets: + return queryset + assets = self.node.get_assets() + for asset in assets: + queryset.append(asset.as_tree_node(self.node)) + return queryset + + def filter_queryset(self, queryset): + queryset = self.filter_assets(queryset) + queryset = self.filter_refresh_nodes(queryset) + return queryset + + def filter_refresh_nodes(self, queryset): + if self.request.query_params.get('refresh', '0') == '1': + Node.expire_nodes_assets_amount() + Node.expire_nodes_full_value() + return queryset class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): @@ -67,19 +134,13 @@ class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): serializer_class = serializers.NodeSerializer instance = None - def counter(self): - values = [ - child.value[child.value.rfind(' '):] - for child in self.get_object().get_children() - if child.value.startswith("新节点 ") - ] - values = [int(value) for value in values if value.strip().isdigit()] - count = max(values)+1 if values else 1 - return count + def get(self, request, *args, **kwargs): + return self.list(request, *args, **kwargs) def post(self, request, *args, **kwargs): + instance = self.get_object() if not request.data.get("value"): - request.data["value"] = _("New node {}").format(self.counter()) + request.data["value"] = instance.get_next_child_preset_name() return super().post(request, *args, **kwargs) def create(self, request, *args, **kwargs): @@ -91,15 +152,12 @@ class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): 'The same level node name cannot be the same' ) node = instance.create_child(value=value) - return Response( - {"id": node.id, "key": node.key, "value": node.value}, - status=201, - ) + return Response(self.serializer_class(instance=node).data, status=201) def get_object(self): pk = self.kwargs.get('pk') or self.request.query_params.get('id') if not pk: - node = None + node = Node.root() else: node = get_object_or_404(Node, pk=pk) return node @@ -107,7 +165,6 @@ class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): def get_queryset(self): queryset = [] query_all = self.request.query_params.get("all") - query_assets = self.request.query_params.get('assets') node = self.get_object() if node is None: @@ -120,23 +177,8 @@ class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView): else: children = node.get_children() queryset.extend(list(children)) - - if query_assets: - assets = node.get_assets() - for asset in assets: - node_fake = Node() - node_fake.assets__count = 0 - node_fake.id = asset.id - node_fake.is_node = False - node_fake.key = node.key + ':0' - node_fake.value = asset.hostname - queryset.append(node_fake) - queryset = sorted(queryset, key=lambda x: x.is_node, reverse=True) return queryset - def get(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) - class NodeAssetsApi(generics.ListAPIView): permission_classes = (IsOrgAdmin,) @@ -234,5 +276,14 @@ class TestNodeConnectiveApi(APIView): assets = node.assets.all() # task_name = _("测试节点下资产是否可连接: {}".format(node.name)) task_name = _("Test if the assets under the node are connectable: {}".format(node.name)) - task = test_asset_connectability_util.delay(assets, task_name=task_name) + task = test_asset_connectivity_util.delay(assets, task_name=task_name) return Response({"task": task.id}) + + +class RefreshAssetsAmount(APIView): + permission_classes = (IsOrgAdmin,) + model = Node + + def get(self, request, *args, **kwargs): + self.model.expire_nodes_assets_amount() + return Response("Ok") diff --git a/apps/assets/api/system_user.py b/apps/assets/api/system_user.py index 3c1d0b3bd..e66e4bfc9 100644 --- a/apps/assets/api/system_user.py +++ b/apps/assets/api/system_user.py @@ -24,8 +24,8 @@ from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser from ..models import SystemUser, Asset from .. import serializers from ..tasks import push_system_user_to_assets_manual, \ - test_system_user_connectability_manual, push_system_user_a_asset_manual, \ - test_system_user_connectability_a_asset + test_system_user_connectivity_manual, push_system_user_a_asset_manual, \ + test_system_user_connectivity_a_asset logger = get_logger(__file__) @@ -33,7 +33,7 @@ __all__ = [ 'SystemUserViewSet', 'SystemUserAuthInfoApi', 'SystemUserPushApi', 'SystemUserTestConnectiveApi', 'SystemUserAssetsListView', 'SystemUserPushToAssetApi', - 'SystemUserTestAssetConnectabilityApi', 'SystemUserCommandFilterRuleListApi', + 'SystemUserTestAssetConnectivityApi', 'SystemUserCommandFilterRuleListApi', ] @@ -93,15 +93,16 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView): def retrieve(self, request, *args, **kwargs): system_user = self.get_object() - task = test_system_user_connectability_manual.delay(system_user) + task = test_system_user_connectivity_manual.delay(system_user) return Response({"task": task.id}) class SystemUserAssetsListView(generics.ListAPIView): permission_classes = (IsOrgAdmin,) - serializer_class = serializers.AssetSerializer + serializer_class = serializers.AssetSimpleSerializer pagination_class = LimitOffsetPagination filter_fields = ("hostname", "ip") + http_method_names = ['get'] search_fields = filter_fields def get_object(self): @@ -125,7 +126,7 @@ class SystemUserPushToAssetApi(generics.RetrieveAPIView): return Response({"task": task.id}) -class SystemUserTestAssetConnectabilityApi(generics.RetrieveAPIView): +class SystemUserTestAssetConnectivityApi(generics.RetrieveAPIView): queryset = SystemUser.objects.all() permission_classes = (IsOrgAdmin,) @@ -133,7 +134,7 @@ class SystemUserTestAssetConnectabilityApi(generics.RetrieveAPIView): system_user = self.get_object() asset_id = self.kwargs.get('aid') asset = get_object_or_404(Asset, id=asset_id) - task = test_system_user_connectability_a_asset.delay(system_user, asset) + task = test_system_user_connectivity_a_asset.delay(system_user, asset) return Response({"task": task.id}) diff --git a/apps/assets/forms/asset.py b/apps/assets/forms/asset.py index 28c78e981..0a6b2f093 100644 --- a/apps/assets/forms/asset.py +++ b/apps/assets/forms/asset.py @@ -41,9 +41,6 @@ class AssetCreateForm(OrgModelForm): 'nodes': _("Node"), } help_texts = { - 'hostname': '* required', - 'ip': '* required', - 'port': '* required', 'admin_user': _( 'root or other NOPASSWD sudo privilege user existed in asset,' 'If asset is windows or other set any one, more see admin user left menu' @@ -80,10 +77,6 @@ class AssetUpdateForm(OrgModelForm): 'nodes': _("Node"), } help_texts = { - 'hostname': '* required', - 'ip': '* required', - 'port': '* required', - 'cluster': '* required', 'admin_user': _( 'root or other NOPASSWD sudo privilege user existed in asset,' 'If asset is windows or other set any one, more see admin user left menu' @@ -95,7 +88,7 @@ class AssetUpdateForm(OrgModelForm): class AssetBulkUpdateForm(OrgModelForm): assets = forms.ModelMultipleChoiceField( - required=True, help_text='* required', + required=True, label=_('Select assets'), queryset=Asset.objects.all(), widget=forms.SelectMultiple( attrs={ @@ -142,14 +135,14 @@ class AssetBulkUpdateForm(OrgModelForm): if k in changed_fields} assets = cleaned_data.pop('assets') labels = cleaned_data.pop('labels', []) - nodes = cleaned_data.pop('nodes') + nodes = cleaned_data.pop('nodes', None) assets = Asset.objects.filter(id__in=[asset.id for asset in assets]) assets.update(**cleaned_data) if labels: - for label in labels: - label.assets.add(*tuple(assets)) + for asset in assets: + asset.labels.set(labels) if nodes: - for node in nodes: - node.assets.add(*tuple(assets)) + for asset in assets: + asset.nodes.set(nodes) return assets diff --git a/apps/assets/forms/domain.py b/apps/assets/forms/domain.py index 66f490bad..25295782a 100644 --- a/apps/assets/forms/domain.py +++ b/apps/assets/forms/domain.py @@ -28,6 +28,15 @@ class DomainForm(forms.ModelForm): initial['assets'] = kwargs['instance'].assets.all() super().__init__(*args, **kwargs) + # 前端渲染优化, 防止过多资产 + assets_field = self.fields.get('assets') + if not self.data: + instance = kwargs.get('instance') + if instance: + assets_field.queryset = instance.assets.all() + else: + assets_field.queryset = Asset.objects.none() + def save(self, commit=True): instance = super().save(commit=commit) assets = self.cleaned_data['assets'] @@ -40,6 +49,8 @@ class GatewayForm(PasswordAndKeyAuthForm, OrgModelForm): super().__init__(*args, **kwargs) password_field = self.fields.get('password') password_field.help_text = _('Password should not contain special characters') + protocol_field = self.fields.get('protocol') + protocol_field.choices = [Gateway.PROTOCOL_CHOICES[0]] def save(self, commit=True): # Because we define custom field, so we need rewrite :method: `save` @@ -59,7 +70,3 @@ class GatewayForm(PasswordAndKeyAuthForm, OrgModelForm): 'name': forms.TextInput(attrs={'placeholder': _('Name')}), 'username': forms.TextInput(attrs={'placeholder': _('Username')}), } - help_texts = { - 'name': '* required', - 'username': '* required', - } diff --git a/apps/assets/forms/label.py b/apps/assets/forms/label.py index ebdc9384e..8a5a54e4a 100644 --- a/apps/assets/forms/label.py +++ b/apps/assets/forms/label.py @@ -26,6 +26,15 @@ class LabelForm(forms.ModelForm): initial['assets'] = kwargs['instance'].assets.all() super().__init__(*args, **kwargs) + # 前端渲染优化, 防止过多资产 + assets_field = self.fields.get('assets') + if not self.data: + instance = kwargs.get('instance') + if instance: + assets_field.queryset = instance.assets.all() + else: + assets_field.queryset = Asset.objects.none() + def save(self, commit=True): label = super().save(commit=commit) assets = self.cleaned_data['assets'] diff --git a/apps/assets/forms/user.py b/apps/assets/forms/user.py index f5c62a4ff..575d3e59c 100644 --- a/apps/assets/forms/user.py +++ b/apps/assets/forms/user.py @@ -80,10 +80,6 @@ class AdminUserForm(PasswordAndKeyAuthForm): 'name': forms.TextInput(attrs={'placeholder': _('Name')}), 'username': forms.TextInput(attrs={'placeholder': _('Username')}), } - help_texts = { - 'name': '* required', - 'username': '* required', - } class SystemUserForm(OrgModelForm, PasswordAndKeyAuthForm): @@ -99,8 +95,10 @@ class SystemUserForm(OrgModelForm, PasswordAndKeyAuthForm): auto_generate_key = self.cleaned_data.get('auto_generate_key', False) private_key, public_key = super().gen_keys() - if login_mode == SystemUser.MANUAL_LOGIN or \ - protocol in [SystemUser.RDP_PROTOCOL, SystemUser.TELNET_PROTOCOL]: + if login_mode == SystemUser.LOGIN_MANUAL or \ + protocol in [SystemUser.PROTOCOL_RDP, + SystemUser.PROTOCOL_TELNET, + SystemUser.PROTOCOL_VNC]: system_user.auto_push = 0 auto_generate_key = False system_user.save() @@ -120,17 +118,18 @@ class SystemUserForm(OrgModelForm, PasswordAndKeyAuthForm): if not self.instance and not auto_generate: super().validate_password_key() - def is_valid(self): - validated = super().is_valid() - username = self.cleaned_data.get('username') - login_mode = self.cleaned_data.get('login_mode') - if login_mode == SystemUser.AUTO_LOGIN and not username: - self.add_error( - "username", _('* Automatic login mode,' - ' must fill in the username.') - ) - return False - return validated + def clean_username(self): + username = self.data.get('username') + login_mode = self.data.get('login_mode') + protocol = self.data.get('protocol') + + if username: + return username + if login_mode == SystemUser.LOGIN_AUTO and \ + protocol != SystemUser.PROTOCOL_VNC: + msg = _('* Automatic login mode must fill in the username.') + raise forms.ValidationError(msg) + return username class Meta: model = SystemUser @@ -147,8 +146,6 @@ class SystemUserForm(OrgModelForm, PasswordAndKeyAuthForm): }), } help_texts = { - 'name': '* required', - 'username': '* required', 'auto_push': _('Auto push system user to asset'), 'priority': _('1-100, High level will be using login asset as default, ' 'if user was granted more than 2 system user'), diff --git a/apps/assets/migrations/0020_auto_20180816_1652.py b/apps/assets/migrations/0020_auto_20180816_1652.py index be5835781..8639c4022 100644 --- a/apps/assets/migrations/0020_auto_20180816_1652.py +++ b/apps/assets/migrations/0020_auto_20180816_1652.py @@ -13,36 +13,36 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='adminuser', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), migrations.AlterField( model_name='asset', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), migrations.AlterField( model_name='domain', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), migrations.AlterField( model_name='gateway', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), migrations.AlterField( model_name='label', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), migrations.AlterField( model_name='node', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), migrations.AlterField( model_name='systemuser', name='org_id', - field=models.CharField(blank=True, default='', max_length=36, verbose_name='Organization'), + field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'), ), ] diff --git a/apps/assets/migrations/0022_auto_20181012_1717.py b/apps/assets/migrations/0022_auto_20181012_1717.py index 0a7d18380..db470c599 100644 --- a/apps/assets/migrations/0022_auto_20181012_1717.py +++ b/apps/assets/migrations/0022_auto_20181012_1717.py @@ -16,7 +16,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='CommandFilter', fields=[ - ('org_id', models.CharField(blank=True, default='', max_length=36, verbose_name='Organization')), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), ('name', models.CharField(max_length=64, verbose_name='Name')), ('is_active', models.BooleanField(default=True, verbose_name='Is active')), @@ -32,7 +32,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='CommandFilterRule', fields=[ - ('org_id', models.CharField(blank=True, default='', max_length=36, verbose_name='Organization')), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), ('type', models.CharField(choices=[('regex', 'Regex'), ('command', 'Command')], default='command', max_length=16, verbose_name='Type')), ('priority', models.IntegerField(default=50, help_text='1-100, the lower will be match first', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(100)], verbose_name='Priority')), diff --git a/apps/assets/migrations/0024_auto_20181219_1614.py b/apps/assets/migrations/0024_auto_20181219_1614.py new file mode 100644 index 000000000..5e6a6c4e3 --- /dev/null +++ b/apps/assets/migrations/0024_auto_20181219_1614.py @@ -0,0 +1,23 @@ +# Generated by Django 2.1.4 on 2018-12-19 08:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0023_auto_20181016_1650'), + ] + + operations = [ + migrations.AlterField( + model_name='asset', + name='protocol', + field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('telnet', 'telnet (beta)'), ('vnc', 'vnc')], default='ssh', max_length=128, verbose_name='Protocol'), + ), + migrations.AlterField( + model_name='systemuser', + name='protocol', + field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('telnet', 'telnet (beta)'), ('vnc', 'vnc')], default='ssh', max_length=16, verbose_name='Protocol'), + ), + ] diff --git a/apps/assets/models/__init__.py b/apps/assets/models/__init__.py index 4a3b67469..c60830fba 100644 --- a/apps/assets/models/__init__.py +++ b/apps/assets/models/__init__.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# from .user import * from .label import Label from .cluster import * diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 5c8f0e2cb..9ab5aae86 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -13,7 +13,6 @@ from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django.core.cache import cache -from ..const import ASSET_ADMIN_CONN_CACHE_KEY from .user import AdminUser, SystemUser from orgs.mixins import OrgModelMixin, OrgManager @@ -60,78 +59,65 @@ class Asset(OrgModelMixin): ('Other', 'Other'), ) - SSH_PROTOCOL = 'ssh' - RDP_PROTOCOL = 'rdp' - TELNET_PROTOCOL = 'telnet' + PROTOCOL_SSH = 'ssh' + PROTOCOL_RDP = 'rdp' + PROTOCOL_TELNET = 'telnet' + PROTOCOL_VNC = 'vnc' PROTOCOL_CHOICES = ( - (SSH_PROTOCOL, 'ssh'), - (RDP_PROTOCOL, 'rdp'), - (TELNET_PROTOCOL, 'telnet (beta)'), + (PROTOCOL_SSH, 'ssh'), + (PROTOCOL_RDP, 'rdp'), + (PROTOCOL_TELNET, 'telnet (beta)'), + (PROTOCOL_VNC, 'vnc'), ) id = models.UUIDField(default=uuid.uuid4, primary_key=True) ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True) hostname = models.CharField(max_length=128, verbose_name=_('Hostname')) - protocol = models.CharField(max_length=128, default=SSH_PROTOCOL, choices=PROTOCOL_CHOICES, verbose_name=_('Protocol')) + protocol = models.CharField(max_length=128, default=PROTOCOL_SSH, choices=PROTOCOL_CHOICES, verbose_name=_('Protocol')) port = models.IntegerField(default=22, verbose_name=_('Port')) platform = models.CharField(max_length=128, choices=PLATFORM_CHOICES, default='Linux', verbose_name=_('Platform')) - domain = models.ForeignKey("assets.Domain", null=True, blank=True, - related_name='assets', verbose_name=_("Domain"), - on_delete=models.SET_NULL) - nodes = models.ManyToManyField('assets.Node', default=default_node, - related_name='assets', - verbose_name=_("Nodes")) + domain = models.ForeignKey("assets.Domain", null=True, blank=True, related_name='assets', verbose_name=_("Domain"), on_delete=models.SET_NULL) + nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets', verbose_name=_("Nodes")) is_active = models.BooleanField(default=True, verbose_name=_('Is active')) # Auth - admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT, - null=True, verbose_name=_("Admin user")) + admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT, null=True, verbose_name=_("Admin user")) # Some information public_ip = models.GenericIPAddressField(max_length=32, blank=True, null=True, verbose_name=_('Public IP')) number = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Asset number')) # Collect - vendor = models.CharField(max_length=64, null=True, blank=True, - verbose_name=_('Vendor')) - model = models.CharField(max_length=54, null=True, blank=True, - verbose_name=_('Model')) - sn = models.CharField(max_length=128, null=True, blank=True, - verbose_name=_('Serial number')) + vendor = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Vendor')) + model = models.CharField(max_length=54, null=True, blank=True, verbose_name=_('Model')) + sn = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Serial number')) - cpu_model = models.CharField(max_length=64, null=True, blank=True, - verbose_name=_('CPU model')) + cpu_model = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('CPU model')) cpu_count = models.IntegerField(null=True, verbose_name=_('CPU count')) cpu_cores = models.IntegerField(null=True, verbose_name=_('CPU cores')) cpu_vcpus = models.IntegerField(null=True, verbose_name=_('CPU vcpus')) - memory = models.CharField(max_length=64, null=True, blank=True, - verbose_name=_('Memory')) - disk_total = models.CharField(max_length=1024, null=True, blank=True, - verbose_name=_('Disk total')) - disk_info = models.CharField(max_length=1024, null=True, blank=True, - verbose_name=_('Disk info')) + memory = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Memory')) + disk_total = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk total')) + disk_info = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk info')) - os = models.CharField(max_length=128, null=True, blank=True, - verbose_name=_('OS')) - os_version = models.CharField(max_length=16, null=True, blank=True, - verbose_name=_('OS version')) - os_arch = models.CharField(max_length=16, blank=True, null=True, - verbose_name=_('OS arch')) - hostname_raw = models.CharField(max_length=128, blank=True, null=True, - verbose_name=_('Hostname raw')) + os = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('OS')) + os_version = models.CharField(max_length=16, null=True, blank=True, verbose_name=_('OS version')) + os_arch = models.CharField(max_length=16, blank=True, null=True, verbose_name=_('OS arch')) + hostname_raw = models.CharField(max_length=128, blank=True, null=True, verbose_name=_('Hostname raw')) - labels = models.ManyToManyField('assets.Label', blank=True, - related_name='assets', - verbose_name=_("Labels")) - created_by = models.CharField(max_length=32, null=True, blank=True, - verbose_name=_('Created by')) - date_created = models.DateTimeField(auto_now_add=True, null=True, - blank=True, - verbose_name=_('Date created')) - comment = models.TextField(max_length=128, default='', blank=True, - verbose_name=_('Comment')) + labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels")) + created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by')) + date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created')) + comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment')) objects = OrgManager.from_queryset(AssetQuerySet)() + CONNECTIVITY_CACHE_KEY = '_JMS_ASSET_CONNECTIVITY_{}' + UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3) + CONNECTIVITY_CHOICES = ( + (UNREACHABLE, _("Unreachable")), + (REACHABLE, _('Reachable')), + (UNKNOWN, _("Unknown")), + ) def __str__(self): return '{0.hostname}({0.ip})'.format(self) @@ -145,6 +131,13 @@ class Asset(OrgModelMixin): return True, '' return False, warning + def support_ansible(self): + if self.platform in ("Windows", "Windows2016", "Other"): + return False + if self.protocol != 'ssh': + return False + return True + def is_unixlike(self): if self.platform not in ("Windows", "Windows2016"): return True @@ -190,25 +183,17 @@ class Asset(OrgModelMixin): return '' @property - def is_connective(self): + def connectivity(self): if not self.is_unixlike(): - return True - val = cache.get(ASSET_ADMIN_CONN_CACHE_KEY.format(self.hostname)) - if val == 1: - return True - else: - return False + return self.UNKNOWN + key = self.CONNECTIVITY_CACHE_KEY.format(str(self.id)) + cached = cache.get(key, None) + return cached if cached is not None else self.UNKNOWN - def to_json(self): - info = { - 'id': self.id, - 'hostname': self.hostname, - 'ip': self.ip, - 'port': self.port, - } - if self.domain and self.domain.gateway_set.all(): - info["gateways"] = [d.id for d in self.domain.gateway_set.all()] - return info + @connectivity.setter + def connectivity(self, value): + key = self.CONNECTIVITY_CACHE_KEY.format(str(self.id)) + cache.set(key, value, 3600*2) def get_auth_info(self): if self.admin_user: @@ -229,11 +214,20 @@ class Asset(OrgModelMixin): fake_node.is_node = False return fake_node + def to_json(self): + info = { + 'id': self.id, + 'hostname': self.hostname, + 'ip': self.ip, + 'port': self.port, + } + if self.domain and self.domain.gateway_set.all(): + info["gateways"] = [d.id for d in self.domain.gateway_set.all()] + return info + def _to_secret_json(self): """ - Ansible use it create inventory, First using asset user, - otherwise using cluster admin user - + Ansible use it create inventory Todo: May be move to ops implements it """ data = self.to_json() @@ -248,6 +242,36 @@ class Asset(OrgModelMixin): }) return data + def as_tree_node(self, parent_node): + from common.tree import TreeNode + icon_skin = 'file' + if self.platform.lower() == 'windows': + icon_skin = 'windows' + elif self.platform.lower() == 'linux': + icon_skin = 'linux' + data = { + 'id': str(self.id), + 'name': self.hostname, + 'title': self.ip, + 'pId': parent_node.key, + 'isParent': False, + 'open': False, + 'iconSkin': icon_skin, + 'meta': { + 'type': 'asset', + 'asset': { + 'id': self.id, + 'hostname': self.hostname, + 'ip': self.ip, + 'port': self.port, + 'platform': self.platform, + 'protocol': self.protocol, + } + } + } + tree_node = TreeNode(**data) + return tree_node + class Meta: unique_together = [('org_id', 'hostname')] verbose_name = _("Asset") @@ -257,7 +281,8 @@ class Asset(OrgModelMixin): from random import seed, choice import forgery_py from django.db import IntegrityError - + from .node import Node + nodes = list(Node.objects.all()) seed() for i in range(count): ip = [str(i) for i in random.sample(range(255), 4)] @@ -268,6 +293,11 @@ class Asset(OrgModelMixin): created_by='Fake') try: asset.save() + if nodes and len(nodes) > 3: + _nodes = random.sample(nodes, 3) + else: + _nodes = [Node.default_node()] + asset.nodes.set(_nodes) asset.system_users = [choice(SystemUser.objects.all()) for i in range(3)] logger.debug('Generate fake asset : %s' % asset.ip) except IntegrityError: diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index b03010905..37e099e99 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -29,6 +29,13 @@ class AssetUser(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) + UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3) + CONNECTIVITY_CHOICES = ( + (UNREACHABLE, _("Unreachable")), + (REACHABLE, _('Reachable')), + (UNKNOWN, _("Unknown")), + ) + @property def password(self): if self._password: @@ -105,6 +112,9 @@ class AssetUser(OrgModelMixin): if update_fields: self.save(update_fields=update_fields) + def get_auth(self, asset=None): + pass + def clear_auth(self): self._password = '' self._private_key = '' diff --git a/apps/assets/models/cmd_filter.py b/apps/assets/models/cmd_filter.py index 170921318..ea2059d51 100644 --- a/apps/assets/models/cmd_filter.py +++ b/apps/assets/models/cmd_filter.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # import uuid +import re from django.db import models from django.core.validators import MinValueValidator, MaxValueValidator @@ -35,7 +36,7 @@ class CommandFilterRule(OrgModelMixin): (TYPE_COMMAND, _('Command')), ) - ACTION_DENY, ACTION_ALLOW = range(2) + ACTION_DENY, ACTION_ALLOW, ACTION_UNKNOWN = range(3) ACTION_CHOICES = ( (ACTION_DENY, _('Deny')), (ACTION_ALLOW, _('Allow')), @@ -53,8 +54,34 @@ class CommandFilterRule(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True) created_by = models.CharField(max_length=128, blank=True, default='', verbose_name=_('Created by')) + __pattern = None + class Meta: ordering = ('-priority', 'action') + @property + def _pattern(self): + if self.__pattern: + return self.__pattern + if self.type == 'command': + regex = [] + for cmd in self.content.split('\r\n'): + cmd = cmd.replace(' ', '\s+') + regex.append(r'\b{0}\b'.format(cmd)) + self.__pattern = re.compile(r'{}'.format('|'.join(regex))) + else: + self.__pattern = re.compile(r'{0}'.format(self.content)) + return self.__pattern + + def match(self, data): + found = self._pattern.search(data) + if not found: + return self.ACTION_UNKNOWN, '' + + if self.action == self.ACTION_ALLOW: + return self.ACTION_ALLOW, found.group() + else: + return self.ACTION_DENY, found.group() + def __str__(self): return '{} % {}'.format(self.type, self.content) diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index 049063144..7272a60fd 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -4,6 +4,8 @@ import uuid import random +import paramiko + from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -38,15 +40,15 @@ class Domain(OrgModelMixin): class Gateway(AssetUser): - SSH_PROTOCOL = 'ssh' - RDP_PROTOCOL = 'rdp' + PROTOCOL_SSH = 'ssh' + PROTOCOL_RDP = 'rdp' PROTOCOL_CHOICES = ( - (SSH_PROTOCOL, 'ssh'), - (RDP_PROTOCOL, 'rdp'), + (PROTOCOL_SSH, 'ssh'), + (PROTOCOL_RDP, 'rdp'), ) ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True) port = models.IntegerField(default=22, verbose_name=_('Port')) - protocol = models.CharField(choices=PROTOCOL_CHOICES, max_length=16, default=SSH_PROTOCOL, verbose_name=_("Protocol")) + protocol = models.CharField(choices=PROTOCOL_CHOICES, max_length=16, default=PROTOCOL_SSH, verbose_name=_("Protocol")) domain = models.ForeignKey(Domain, on_delete=models.CASCADE, verbose_name=_("Domain")) comment = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Comment")) is_active = models.BooleanField(default=True, verbose_name=_("Is active")) @@ -57,3 +59,37 @@ class Gateway(AssetUser): class Meta: unique_together = [('name', 'org_id')] verbose_name = _("Gateway") + + def test_connective(self): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + proxy = paramiko.SSHClient() + proxy.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + proxy.connect(self.ip, port=self.port, + username=self.username, + password=self.password, + pkey=self.private_key_obj) + except(paramiko.AuthenticationException, + paramiko.BadAuthenticationType, + paramiko.SSHException) as e: + return False, str(e) + + sock = proxy.get_transport().open_channel( + 'direct-tcpip', ('127.0.0.1', self.port), ('127.0.0.1', 0) + ) + + try: + client.connect("127.0.0.1", port=self.port, + username=self.username, + password=self.password, + key_filename=self.private_key_file, + sock=sock, + timeout=5) + except (paramiko.SSHException, paramiko.ssh_exception.SSHException, + paramiko.AuthenticationException, TimeoutError) as e: + return False, str(e) + finally: + client.close() + return True, None diff --git a/apps/assets/models/label.py b/apps/assets/models/label.py index abc71e694..458f3077d 100644 --- a/apps/assets/models/label.py +++ b/apps/assets/models/label.py @@ -17,7 +17,8 @@ class Label(OrgModelMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField(max_length=128, verbose_name=_("Name")) value = models.CharField(max_length=128, verbose_name=_("Value")) - category = models.CharField(max_length=128, choices=CATEGORY_CHOICES, default=USER_CATEGORY, verbose_name=_("Category")) + category = models.CharField(max_length=128, choices=CATEGORY_CHOICES, + default=USER_CATEGORY, verbose_name=_("Category")) is_active = models.BooleanField(default=True, verbose_name=_("Is active")) comment = models.TextField(blank=True, null=True, verbose_name=_("Comment")) date_created = models.DateTimeField( diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 3d5c50997..881b041d7 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -5,6 +5,7 @@ import uuid from django.db import models, transaction from django.db.models import Q from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext from django.core.cache import cache from orgs.mixins import OrgModelMixin @@ -22,7 +23,9 @@ class Node(OrgModelMixin): date_create = models.DateTimeField(auto_now_add=True) is_node = True - _full_value_cache_key_prefix = '_NODE_VALUE_{}' + _assets_amount = None + _full_value_cache_key = '_NODE_VALUE_{}' + _assets_amount_cache_key = '_NODE_ASSETS_AMOUNT_{}' class Meta: verbose_name = _("Node") @@ -49,30 +52,65 @@ class Node(OrgModelMixin): def name(self): return self.value + @property + def assets_amount(self): + """ + 获取节点下所有资产数量速度太慢,所以需要重写,使用cache等方案 + :return: + """ + if self._assets_amount is not None: + return self._assets_amount + cache_key = self._assets_amount_cache_key.format(self.key) + cached = cache.get(cache_key) + if cached is not None: + return cached + assets_amount = self.get_all_assets().count() + cache.set(cache_key, assets_amount, 3600) + return assets_amount + + @assets_amount.setter + def assets_amount(self, value): + self._assets_amount = value + + def expire_assets_amount(self): + ancestor_keys = self.get_ancestor_keys(with_self=True) + cache_keys = [self._assets_amount_cache_key.format(k) for k in ancestor_keys] + cache.delete_many(cache_keys) + + @classmethod + def expire_nodes_assets_amount(cls, nodes=None): + if nodes: + for node in nodes: + node.expire_assets_amount() + return + key = cls._assets_amount_cache_key.format('*') + cache.delete_pattern(key) + @property def full_value(self): - key = self._full_value_cache_key_prefix.format(self.key) + key = self._full_value_cache_key.format(self.key) cached = cache.get(key) if cached: return cached - value = self.get_full_value() - self.cache_full_value(value) - return value - - def get_full_value(self): - # ancestor = [a.value for a in self.get_ancestor(with_self=True)] if self.is_root(): return self.value parent_full_value = self.parent.full_value value = parent_full_value + ' / ' + self.value + key = self._full_value_cache_key.format(self.key) + cache.set(key, value, 3600) return value - def cache_full_value(self, value): - key = self._full_value_cache_key_prefix.format(self.key) - cache.set(key, value, 3600) - def expire_full_value(self): - key = self._full_value_cache_key_prefix.format(self.key) + key = self._full_value_cache_key.format(self.key) + cache.delete_pattern(key+'*') + + @classmethod + def expire_nodes_full_value(cls, nodes=None): + if nodes: + for node in nodes: + node.expire_full_value() + return + key = cls._full_value_cache_key.format('*') cache.delete_pattern(key+'*') @property @@ -85,6 +123,17 @@ class Node(OrgModelMixin): self.save() return "{}:{}".format(self.key, mark) + def get_next_child_preset_name(self): + name = ugettext("New node") + values = [ + child.value[child.value.rfind(' '):] + for child in self.get_children() + if child.value.startswith(name) + ] + values = [int(value) for value in values if value.strip().isdigit()] + count = max(values) + 1 if values else 1 + return '{} {}'.format(name, count) + def create_child(self, value): with transaction.atomic(): child_key = self.get_next_child_key() @@ -134,7 +183,7 @@ class Node(OrgModelMixin): pattern = r'^{0}$|^{0}:'.format(self.key) args = [] kwargs = {} - if self.is_default_node(): + if self.is_root(): args.append(Q(nodes__key__regex=pattern) | Q(nodes=None)) else: kwargs['nodes__key__regex'] = pattern @@ -182,17 +231,18 @@ class Node(OrgModelMixin): child.save() self.save() - def get_ancestor(self, with_self=False): - if self.is_root(): - root = self.__class__.root() - return [root] - _key = self.key.split(':') + def get_ancestor_keys(self, with_self=False): + parent_keys = [] + key_list = self.key.split(":") if not with_self: - _key.pop() - ancestor_keys = [] - for i in range(len(_key)): - ancestor_keys.append(':'.join(_key)) - _key.pop() + key_list.pop() + for i in range(len(key_list)): + parent_keys.append(":".join(key_list)) + key_list.pop() + return parent_keys + + def get_ancestor(self, with_self=False): + ancestor_keys = self.get_ancestor_keys(with_self=with_self) ancestor = self.__class__.objects.filter( key__in=ancestor_keys ).order_by('key') @@ -203,17 +253,14 @@ class Node(OrgModelMixin): # 如果使用current_org 在set_current_org时会死循环 _current_org = get_current_org() with transaction.atomic(): - if _current_org.is_root(): - key = '0' - elif _current_org.is_default(): - key = '1' - else: - set_current_org(Organization.root()) - org_nodes_roots = cls.objects.filter(key__regex=r'^[0-9]+$') - org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True) or ['1'] - key = max([int(k) for k in org_nodes_roots_keys]) - key = str(key + 1) if key != 0 else '2' - set_current_org(_current_org) + if not _current_org.is_real(): + return cls.default_node() + set_current_org(Organization.root()) + org_nodes_roots = cls.objects.filter(key__regex=r'^[0-9]+$') + org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True) or ['1'] + key = max([int(k) for k in org_nodes_roots_keys]) + key = str(key + 1) if key != 0 else '2' + set_current_org(_current_org) root = cls.objects.create(key=key, value=_current_org.name) return root @@ -230,9 +277,25 @@ class Node(OrgModelMixin): defaults = {'value': 'Default'} return cls.objects.get_or_create(defaults=defaults, key='1') - @classmethod - def get_tree_name_ref(cls): - pass + def as_tree_node(self): + from common.tree import TreeNode + from ..serializers import NodeSerializer + name = '{} ({})'.format(self.value, self.assets_amount) + node_serializer = NodeSerializer(instance=self) + data = { + 'id': self.key, + 'name': name, + 'title': name, + 'pId': self.parent_key, + 'isParent': True, + 'open': self.is_root(), + 'meta': { + 'node': node_serializer.data, + 'type': 'node' + } + } + tree_node = TreeNode(**data) + return tree_node @classmethod def generate_fake(cls, count=100): diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 918440081..1e661d631 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -14,7 +14,7 @@ from ..const import SYSTEM_USER_CONN_CACHE_KEY from .base import AssetUser -__all__ = ['AdminUser', 'SystemUser',] +__all__ = ['AdminUser', 'SystemUser'] logger = logging.getLogger(__name__) signer = get_signer() @@ -31,6 +31,7 @@ class AdminUser(AssetUser): become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4) become_user = models.CharField(default='root', max_length=64) _become_pass = models.CharField(default='', max_length=128) + CONNECTIVE_CACHE_KEY = '_JMS_ADMIN_USER_CONNECTIVE_{}' def __str__(self): return self.name @@ -67,6 +68,23 @@ class AdminUser(AssetUser): def assets_amount(self): return self.get_related_assets().count() + @property + def connectivity(self): + from .asset import Asset + assets = self.get_related_assets().values_list('id', 'hostname', flat=True) + data = { + 'unreachable': [], + 'reachable': [], + } + for asset_id, hostname in assets: + key = Asset.CONNECTIVITY_CACHE_KEY.format(str(self.id)) + value = cache.get(key, Asset.UNKNOWN) + if value == Asset.REACHABLE: + data['reachable'].append(hostname) + elif value == Asset.UNREACHABLE: + data['unreachable'].append(hostname) + return data + class Meta: ordering = ['name'] unique_together = [('name', 'org_id')] @@ -94,34 +112,36 @@ class AdminUser(AssetUser): class SystemUser(AssetUser): - SSH_PROTOCOL = 'ssh' - RDP_PROTOCOL = 'rdp' - TELNET_PROTOCOL = 'telnet' + PROTOCOL_SSH = 'ssh' + PROTOCOL_RDP = 'rdp' + PROTOCOL_TELNET = 'telnet' + PROTOCOL_VNC = 'vnc' PROTOCOL_CHOICES = ( - (SSH_PROTOCOL, 'ssh'), - (RDP_PROTOCOL, 'rdp'), - (TELNET_PROTOCOL, 'telnet (beta)'), + (PROTOCOL_SSH, 'ssh'), + (PROTOCOL_RDP, 'rdp'), + (PROTOCOL_TELNET, 'telnet (beta)'), + (PROTOCOL_VNC, 'vnc'), ) - AUTO_LOGIN = 'auto' - MANUAL_LOGIN = 'manual' + LOGIN_AUTO = 'auto' + LOGIN_MANUAL = 'manual' LOGIN_MODE_CHOICES = ( - (AUTO_LOGIN, _('Automatic login')), - (MANUAL_LOGIN, _('Manually login')) + (LOGIN_AUTO, _('Automatic login')), + (LOGIN_MANUAL, _('Manually login')) ) nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes")) assets = models.ManyToManyField('assets.Asset', blank=True, verbose_name=_("Assets")) - priority = models.IntegerField(default=20, verbose_name=_("Priority"), - validators=[MinValueValidator(1), MaxValueValidator(100)]) + priority = models.IntegerField(default=20, verbose_name=_("Priority"), validators=[MinValueValidator(1), MaxValueValidator(100)]) protocol = models.CharField(max_length=16, choices=PROTOCOL_CHOICES, default='ssh', verbose_name=_('Protocol')) auto_push = models.BooleanField(default=True, verbose_name=_('Auto push')) sudo = models.TextField(default='/bin/whoami', verbose_name=_('Sudo')) shell = models.CharField(max_length=64, default='/bin/bash', verbose_name=_('Shell')) - login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=AUTO_LOGIN, max_length=10, verbose_name=_('Login mode')) + login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode')) cmd_filters = models.ManyToManyField('CommandFilter', related_name='system_users', verbose_name=_("Command filter"), blank=True) - cache_key = "__SYSTEM_USER_CACHED_{}" + SYSTEM_USER_CACHE_KEY = "__SYSTEM_USER_CACHED_{}" + CONNECTIVE_CACHE_KEY = '_JMS_SYSTEM_USER_CONNECTIVE_{}' def __str__(self): return '{0.name}({0.username})'.format(self) @@ -136,34 +156,61 @@ class SystemUser(AssetUser): 'auto_push': self.auto_push, } - def get_assets(self): + def get_related_assets(self): assets = set(self.assets.all()) return assets @property - def assets_connective(self): - _result = cache.get(SYSTEM_USER_CONN_CACHE_KEY.format(self.name), {}) - return _result + def connectivity(self): + cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id)) + value = cache.get(cache_key, None) + if not value or 'unreachable' not in value: + return {'unreachable': [], 'reachable': []} + else: + return value + + @connectivity.setter + def connectivity(self, value): + data = self.connectivity + unreachable = data['unreachable'] + reachable = data['reachable'] + + for host in value.get('dark', {}).keys(): + if host not in unreachable: + unreachable.append(host) + if host in reachable: + reachable.remove(host) + for host in value.get('contacted'): + if host not in reachable: + reachable.append(host) + if host in unreachable: + unreachable.remove(host) + cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id)) + cache.set(cache_key, data, 3600) @property - def unreachable_assets(self): - return list(self.assets_connective.get('dark', {}).keys()) + def assets_unreachable(self): + return self.connectivity.get('unreachable') @property - def reachable_assets(self): - return self.assets_connective.get('contacted', []) + def assets_reachable(self): + return self.connectivity.get('reachable') + + @property + def login_mode_display(self): + return self.get_login_mode_display() def is_need_push(self): - if self.auto_push and self.protocol == self.__class__.SSH_PROTOCOL: + if self.auto_push and self.protocol == self.PROTOCOL_SSH: return True else: return False def set_cache(self): - cache.set(self.cache_key.format(self.id), self, 3600) + cache.set(self.SYSTEM_USER_CACHE_KEY.format(self.id), self, 3600) def expire_cache(self): - cache.delete(self.cache_key.format(self.id)) + cache.delete(self.SYSTEM_USER_CACHE_KEY.format(self.id)) @property def cmd_filter_rules(self): @@ -173,9 +220,18 @@ class SystemUser(AssetUser): ).distinct() return rules + def is_command_can_run(self, command): + for rule in self.cmd_filter_rules: + action, matched_cmd = rule.match(command) + if action == rule.ACTION_ALLOW: + return True, None + elif action == rule.ACTION_DENY: + return False, matched_cmd + return True, None + @classmethod def get_system_user_by_id_or_cached(cls, sid): - cached = cache.get(cls.cache_key.format(sid)) + cached = cache.get(cls.SYSTEM_USER_CACHE_KEY.format(sid)) if cached: return cached try: diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index dae9ab9af..9640aff7f 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -9,6 +9,7 @@ from .system_user import AssetSystemUserSerializer __all__ = [ 'AssetSerializer', 'AssetGrantedSerializer', 'MyAssetGrantedSerializer', + 'AssetAsNodeSerializer', 'AssetSimpleSerializer', ] @@ -22,14 +23,27 @@ class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer): fields = '__all__' validators = [] + @classmethod + def setup_eager_loading(cls, queryset): + """ Perform necessary eager loading of data. """ + queryset = queryset.prefetch_related('labels', 'nodes')\ + .select_related('admin_user') + return queryset + def get_field_names(self, declared_fields, info): fields = super().get_field_names(declared_fields, info) fields.extend([ - 'hardware_info', 'is_connective', 'org_name' + 'hardware_info', 'connectivity', 'org_name' ]) return fields +class AssetAsNodeSerializer(serializers.ModelSerializer): + class Meta: + model = Asset + fields = ['id', 'hostname', 'ip', 'port', 'platform', 'protocol'] + + class AssetGrantedSerializer(serializers.ModelSerializer): """ 被授权资产的数据结构 @@ -64,3 +78,9 @@ class MyAssetGrantedSerializer(AssetGrantedSerializer): "is_active", "system_users_join", "org_name", "os", "platform", "comment", "org_id", "protocol" ) + + +class AssetSimpleSerializer(serializers.ModelSerializer): + class Meta: + model = Asset + fields = ['id', 'hostname', 'port', 'ip', 'connectivity'] diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 034c29387..9cddf0c49 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -23,7 +23,6 @@ class DomainSerializer(serializers.ModelSerializer): class GatewaySerializer(serializers.ModelSerializer): - class Meta: model = Gateway fields = [ diff --git a/apps/assets/serializers/node.py b/apps/assets/serializers/node.py index a57da2cdc..79c573c60 100644 --- a/apps/assets/serializers/node.py +++ b/apps/assets/serializers/node.py @@ -8,85 +8,36 @@ from .asset import AssetGrantedSerializer __all__ = [ - 'NodeSerializer', "NodeGrantedSerializer", "NodeAddChildrenSerializer", + 'NodeSerializer', "NodeAddChildrenSerializer", "NodeAssetsSerializer", ] -class NodeGrantedSerializer(BulkSerializerMixin, serializers.ModelSerializer): - """ - 授权资产组 - """ - assets_granted = AssetGrantedSerializer(many=True, read_only=True) - assets_amount = serializers.SerializerMethodField() - parent = serializers.SerializerMethodField() - name = serializers.SerializerMethodField() - - class Meta: - model = Node - fields = [ - 'id', 'key', 'name', 'value', 'parent', - 'assets_granted', 'assets_amount', 'org_id', - ] - - @staticmethod - def get_assets_amount(obj): - return len(obj.assets_granted) - - @staticmethod - def get_name(obj): - return obj.name - - @staticmethod - def get_parent(obj): - return obj.parent.id - - class NodeSerializer(serializers.ModelSerializer): - assets_amount = serializers.SerializerMethodField() - tree_id = serializers.SerializerMethodField() - tree_parent = serializers.SerializerMethodField() + assets_amount = serializers.IntegerField(read_only=True) class Meta: model = Node fields = [ - 'id', 'key', 'value', 'assets_amount', - 'is_node', 'org_id', 'tree_id', 'tree_parent', + 'id', 'key', 'value', 'assets_amount', 'org_id', + ] + read_only_fields = [ + 'id', 'key', 'assets_amount', 'org_id', ] - list_serializer_class = BulkListSerializer - def validate(self, data): - value = data.get('value') + def validate_value(self, data): instance = self.instance if self.instance else Node.root() children = instance.parent.get_children().exclude(key=instance.key) values = [child.value for child in children] - if value in values: + if data in values: raise serializers.ValidationError( 'The same level node name cannot be the same' ) return data - @staticmethod - def get_assets_amount(obj): - return obj.get_all_assets().count() - - @staticmethod - def get_tree_id(obj): - return obj.key - - @staticmethod - def get_tree_parent(obj): - return obj.parent_key - - def get_fields(self): - fields = super().get_fields() - field = fields["key"] - field.required = False - return fields - class NodeAssetsSerializer(serializers.ModelSerializer): - assets = serializers.PrimaryKeyRelatedField(many=True, queryset = Asset.objects.all()) + assets = serializers.PrimaryKeyRelatedField(many=True, queryset=Asset.objects.all()) class Meta: model = Node @@ -95,3 +46,4 @@ class NodeAssetsSerializer(serializers.ModelSerializer): class NodeAddChildrenSerializer(serializers.Serializer): nodes = serializers.ListField() + diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index a295f245c..be1f594ec 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from ..models import SystemUser +from ..models import SystemUser, Asset from .base import AuthSerializer @@ -21,17 +21,17 @@ class SystemUserSerializer(serializers.ModelSerializer): def get_field_names(self, declared_fields, info): fields = super(SystemUserSerializer, self).get_field_names(declared_fields, info) fields.extend([ - 'get_login_mode_display', + 'login_mode_display', ]) return fields @staticmethod def get_unreachable_assets(obj): - return obj.unreachable_assets + return obj.assets_unreachable @staticmethod def get_reachable_assets(obj): - return obj.reachable_assets + return obj.assets_reachable def get_unreachable_amount(self, obj): return len(self.get_unreachable_assets(obj)) @@ -41,7 +41,7 @@ class SystemUserSerializer(serializers.ModelSerializer): @staticmethod def get_assets_amount(obj): - return len(obj.get_assets()) + return len(obj.get_related_assets()) class SystemUserAuthSerializer(AuthSerializer): @@ -75,4 +75,7 @@ class SystemUserSimpleSerializer(serializers.ModelSerializer): """ class Meta: model = SystemUser - fields = ('id', 'name', 'username') \ No newline at end of file + fields = ('id', 'name', 'username') + + + diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 71866c5d0..7d3b67f6c 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- # from collections import defaultdict -from django.db.models.signals import post_save, m2m_changed +from django.db.models.signals import post_save, m2m_changed, post_delete from django.dispatch import receiver from common.utils import get_logger from .models import Asset, SystemUser, Node from .tasks import update_assets_hardware_info_util, \ - test_asset_connectability_util, push_system_user_to_assets + test_asset_connectivity_util, push_system_user_to_assets logger = get_logger(__file__) @@ -19,8 +19,8 @@ def update_asset_hardware_info_on_created(asset): def test_asset_conn_on_created(asset): - logger.debug("Test asset `{}` connectability".format(asset)) - test_asset_connectability_util.delay([asset]) + logger.debug("Test asset `{}` connectivity".format(asset)) + test_asset_connectivity_util.delay([asset]) def set_asset_root_node(asset): @@ -35,6 +35,17 @@ def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): update_asset_hardware_info_on_created(instance) test_asset_conn_on_created(instance) + # 过期节点资产数量 + nodes = instance.nodes.all() + Node.expire_nodes_assets_amount(nodes) + + +@receiver(post_delete, sender=Asset, dispatch_uid="my_unique_identifier") +def on_asset_delete(sender, instance=None, **kwargs): + # 过期节点资产数量 + nodes = instance.nodes.all() + Node.expire_nodes_assets_amount(nodes) + @receiver(post_save, sender=SystemUser, dispatch_uid="my_unique_identifier") def on_system_user_update(sender, instance=None, created=True, **kwargs): @@ -58,15 +69,19 @@ def on_system_user_nodes_change(sender, instance=None, **kwargs): def on_system_user_assets_change(sender, instance=None, **kwargs): if instance and kwargs["action"] == "post_add": assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) - push_system_user_to_assets(instance, assets) + push_system_user_to_assets.delay(instance, assets) @receiver(m2m_changed, sender=Asset.nodes.through) def on_asset_node_changed(sender, instance=None, **kwargs): + logger.debug("Asset nodes change signal received") if isinstance(instance, Asset): - if kwargs['action'] == 'post_add': - logger.debug("Asset node change signal received") + if kwargs['action'] == 'pre_remove': nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + Node.expire_nodes_assets_amount(nodes) + if kwargs['action'] == 'post_add': + nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + Node.expire_nodes_assets_amount(nodes) system_users_assets = defaultdict(set) system_users = SystemUser.objects.filter(nodes__in=nodes) # 清理节点缓存 @@ -79,9 +94,11 @@ def on_asset_node_changed(sender, instance=None, **kwargs): @receiver(m2m_changed, sender=Asset.nodes.through) def on_node_assets_changed(sender, instance=None, **kwargs): if isinstance(instance, Node): + logger.debug("Node assets change signal {} received".format(instance)) + # 当节点和资产关系发生改变时,过期资产数量缓存 + instance.expire_assets_amount() assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) if kwargs['action'] == 'post_add': - logger.debug("Node assets change signal received") # 重新关联系统用户和资产的关系 system_users = SystemUser.objects.filter(nodes=instance) for system_user in system_users: diff --git a/apps/assets/tasks.py b/apps/assets/tasks.py index 57a4e7bec..ce5be0b6f 100644 --- a/apps/assets/tasks.py +++ b/apps/assets/tasks.py @@ -1,17 +1,16 @@ # ~*~ coding: utf-8 ~*~ import json import re +import time import os from celery import shared_task -from django.core.cache import cache from django.utils.translation import ugettext as _ +from django.core.cache import cache -from common.utils import get_object_or_none, capacity_convert, \ +from common.utils import capacity_convert, \ sum_capacity, encrypt_password, get_logger -from ops.celery.utils import register_as_period_task, after_app_shutdown_clean, \ - after_app_ready_start -from ops.celery import app as celery_app +from ops.celery.utils import register_as_period_task, after_app_shutdown_clean from .models import SystemUser, AdminUser, Asset from . import const @@ -20,34 +19,51 @@ from . import const FORKS = 10 TIMEOUT = 60 logger = get_logger(__file__) -CACHE_MAX_TIME = 60*60*60 +CACHE_MAX_TIME = 60*60*2 disk_pattern = re.compile(r'^hd|sd|xvd|vd') PERIOD_TASK = os.environ.get("PERIOD_TASK", "off") +def clean_hosts(assets): + clean_assets = [] + for asset in assets: + if not asset.is_active: + msg = _("Asset has been disabled, skipped: {}").format(asset) + logger.info(msg) + continue + if not asset.support_ansible(): + msg = _("Asset may not be support ansible, skipped: {}").format(asset) + logger.info(msg) + continue + clean_assets.append(asset) + if not clean_assets: + logger.info(_("No assets matched, stop task")) + return clean_assets + + @shared_task -def set_assets_hardware_info(result, **kwargs): +def set_assets_hardware_info(assets, result, **kwargs): """ Using ops task run result, to update asset info @shared_task must be exit, because we using it as a task callback, is must be a celery task also + :param assets: :param result: :param kwargs: {task_name: ""} :return: """ result_raw = result[0] assets_updated = [] - for hostname, info in result_raw.get('ok', {}).items(): + success_result = result_raw.get('ok', {}) + + for asset in assets: + hostname = asset.hostname + info = success_result.get(hostname, {}) info = info.get('setup', {}).get('ansible_facts', {}) if not info: - logger.error("Get asset info failed: {}".format(hostname)) + logger.error(_("Get asset info failed: {}").format(hostname)) continue - - asset = Asset.objects.get_object_by_fullname(hostname) - if not asset: - continue - ___vendor = info.get('ansible_system_vendor', 'Unknown') ___model = info.get('ansible_product_name', 'Unknown') ___sn = info.get('ansible_product_serial', 'Unknown') @@ -59,9 +75,12 @@ def set_assets_hardware_info(result, **kwargs): ___cpu_model = 'Unknown' ___cpu_model = ___cpu_model[:64] ___cpu_count = info.get('ansible_processor_count', 0) - ___cpu_cores = info.get('ansible_processor_cores', None) or len(info.get('ansible_processor', [])) + ___cpu_cores = info.get('ansible_processor_cores', None) or \ + len(info.get('ansible_processor', [])) ___cpu_vcpus = info.get('ansible_processor_vcpus', 0) - ___memory = '%s %s' % capacity_convert('{} MB'.format(info.get('ansible_memtotal_mb'))) + ___memory = '%s %s' % capacity_convert( + '{} MB'.format(info.get('ansible_memtotal_mb')) + ) disk_info = {} for dev, dev_info in info.get('ansible_devices', {}).items(): if disk_pattern.match(dev) and dev_info['removable'] == '0': @@ -94,34 +113,31 @@ def update_assets_hardware_info_util(assets, task_name=None): from ops.utils import update_or_create_ansible_task if task_name is None: task_name = _("Update some assets hardware info") - # task_name = _("更新资产硬件信息") tasks = const.UPDATE_ASSETS_HARDWARE_TASKS - hostname_list = [asset.fullname for asset in assets if asset.is_active and asset.is_unixlike()] - if not hostname_list: - logger.info("Not hosts get, may be asset is not active or not unixlike platform") + hosts = clean_hosts(assets) + if not hosts: return {} + created_by = str(assets[0].org_id) task, created = update_or_create_ansible_task( - task_name, hosts=hostname_list, tasks=tasks, pattern='all', - options=const.TASK_OPTIONS, run_as_admin=True, created_by='System', + task_name, hosts=hosts, tasks=tasks, created_by=created_by, + pattern='all', options=const.TASK_OPTIONS, run_as_admin=True, ) result = task.run() # Todo: may be somewhere using # Manual run callback function - set_assets_hardware_info(result) + set_assets_hardware_info(assets, result) return result @shared_task def update_asset_hardware_info_manual(asset): - task_name = _("Update asset hardware info") - # task_name = _("更新资产硬件信息") - return update_assets_hardware_info_util([asset], task_name=task_name) + task_name = _("Update asset hardware info: {}").format(asset.hostname) + return update_assets_hardware_info_util( + [asset], task_name=task_name + ) -@celery_app.task -@register_as_period_task(interval=3600) -@after_app_ready_start -@after_app_shutdown_clean +@shared_task def update_assets_hardware_info_period(): """ Update asset hardware period task @@ -131,126 +147,41 @@ def update_assets_hardware_info_period(): logger.debug("Period task disabled, update assets hardware info pass") return - from ops.utils import update_or_create_ansible_task - task_name = _("Update assets hardware info period") - # task_name = _("定期更新资产硬件信息") - hostname_list = [ - asset.fullname for asset in Asset.objects.all() - if asset.is_active and asset.is_unixlike() - ] - tasks = const.UPDATE_ASSETS_HARDWARE_TASKS - - # Only create, schedule by celery beat - update_or_create_ansible_task( - task_name, hosts=hostname_list, tasks=tasks, pattern='all', - options=const.TASK_OPTIONS, run_as_admin=True, created_by='System', - interval=60*60*24, is_periodic=True, callback=set_assets_hardware_info.name, - ) - ## ADMIN USER CONNECTIVE ## -@shared_task -def set_admin_user_connectability_info(result, **kwargs): - admin_user = kwargs.get("admin_user") - task_name = kwargs.get("task_name") - if admin_user is None and task_name is not None: - admin_user = task_name.split(":")[-1] - - raw, summary = result - cache_key = const.ADMIN_USER_CONN_CACHE_KEY.format(admin_user) - cache.set(cache_key, summary, CACHE_MAX_TIME) - - for i in summary.get('contacted', []): - asset_conn_cache_key = const.ASSET_ADMIN_CONN_CACHE_KEY.format(i) - cache.set(asset_conn_cache_key, 1, CACHE_MAX_TIME) - - for i, msg in summary.get('dark', {}).items(): - asset_conn_cache_key = const.ASSET_ADMIN_CONN_CACHE_KEY.format(i) - cache.set(asset_conn_cache_key, 0, CACHE_MAX_TIME) - logger.error(msg) - @shared_task -def test_admin_user_connectability_util(admin_user, task_name): - """ - Test asset admin user can connect or not. Using ansible api do that - :param admin_user: - :param task_name: - :return: - """ - from ops.utils import update_or_create_ansible_task - - assets = admin_user.get_related_assets() - hosts = [asset.fullname for asset in assets - if asset.is_active and asset.is_unixlike()] - if not hosts: - return - tasks = const.TEST_ADMIN_USER_CONN_TASKS - task, created = update_or_create_ansible_task( - task_name=task_name, hosts=hosts, tasks=tasks, pattern='all', - options=const.TASK_OPTIONS, run_as_admin=True, created_by='System', - ) - result = task.run() - set_admin_user_connectability_info(result, admin_user=admin_user.name) - return result - - -@celery_app.task -@register_as_period_task(interval=3600) -@after_app_ready_start -@after_app_shutdown_clean -def test_admin_user_connectability_period(): - """ - A period task that update the ansible task period - """ - if PERIOD_TASK != "on": - logger.debug("Period task disabled, test admin user connectability pass") - return - - admin_users = AdminUser.objects.all() - for admin_user in admin_users: - task_name = _("Test admin user connectability period: {}".format(admin_user.name)) - # task_name = _("定期测试管理账号可连接性: {}".format(admin_user.name)) - test_admin_user_connectability_util(admin_user, task_name) - - -@shared_task -def test_admin_user_connectability_manual(admin_user): - task_name = _("Test admin user connectability: {}").format(admin_user.name) - # task_name = _("测试管理行号可连接性: {}").format(admin_user.name) - return test_admin_user_connectability_util(admin_user, task_name) - - -@shared_task -def test_asset_connectability_util(assets, task_name=None): +def test_asset_connectivity_util(assets, task_name=None): from ops.utils import update_or_create_ansible_task if task_name is None: - task_name = _("Test assets connectability") - # task_name = _("测试资产可连接性") - hosts = [asset.fullname for asset in assets if asset.is_active and asset.is_unixlike()] + task_name = _("Test assets connectivity") + hosts = clean_hosts(assets) if not hosts: - logger.info("No hosts, passed") return {} tasks = const.TEST_ADMIN_USER_CONN_TASKS + created_by = assets[0].org_id task, created = update_or_create_ansible_task( task_name=task_name, hosts=hosts, tasks=tasks, pattern='all', - options=const.TASK_OPTIONS, run_as_admin=True, created_by='System', + options=const.TASK_OPTIONS, run_as_admin=True, created_by=created_by, ) result = task.run() summary = result[1] - for k in summary.get('dark'): - cache.set(const.ASSET_ADMIN_CONN_CACHE_KEY.format(k), 0, CACHE_MAX_TIME) - - for k in summary.get('contacted'): - cache.set(const.ASSET_ADMIN_CONN_CACHE_KEY.format(k), 1, CACHE_MAX_TIME) + for asset in assets: + if asset.hostname in summary.get('dark', {}): + asset.connectivity = asset.UNREACHABLE + elif asset.hostname in summary.get('contacted', []): + asset.connectivity = asset.REACHABLE + else: + asset.connectivity = asset.UNKNOWN return summary @shared_task -def test_asset_connectability_manual(asset): - summary = test_asset_connectability_util([asset]) +def test_asset_connectivity_manual(asset): + task_name = _("Test assets connectivity: {}").format(asset) + summary = test_asset_connectivity_util([asset], task_name=task_name) if summary.get('dark'): return False, summary['dark'] @@ -258,21 +189,56 @@ def test_asset_connectability_manual(asset): return True, "" +@shared_task +def test_admin_user_connectivity_util(admin_user, task_name): + """ + Test asset admin user can connect or not. Using ansible api do that + :param admin_user: + :param task_name: + :return: + """ + assets = admin_user.get_related_assets() + hosts = clean_hosts(assets) + if not hosts: + return {} + summary = test_asset_connectivity_util(hosts, task_name) + return summary + + +@shared_task +@register_as_period_task(interval=3600) +def test_admin_user_connectivity_period(): + """ + A period task that update the ansible task period + """ + key = '_JMS_TEST_ADMIN_USER_CONNECTIVITY_PERIOD' + prev_execute_time = cache.get(key) + if prev_execute_time: + logger.debug("Test admin user connectivity, less than 40 minutes, skip") + return + cache.set(key, 1, 60*40) + admin_users = AdminUser.objects.all() + for admin_user in admin_users: + task_name = _("Test admin user connectivity period: {}").format(admin_user.name) + test_admin_user_connectivity_util(admin_user, task_name) + + +@shared_task +def test_admin_user_connectivity_manual(admin_user): + task_name = _("Test admin user connectivity: {}").format(admin_user.name) + return test_admin_user_connectivity_util(admin_user, task_name) + + ## System user connective ## @shared_task -def set_system_user_connectablity_info(result, **kwargs): +def set_system_user_connectivity_info(system_user, result): summary = result[1] - task_name = kwargs.get("task_name") - system_user = kwargs.get("system_user") - if system_user is None: - system_user = task_name.split(":")[-1] - cache_key = const.SYSTEM_USER_CONN_CACHE_KEY.format(system_user) - cache.set(cache_key, summary, CACHE_MAX_TIME) + system_user.connectivity = summary @shared_task -def test_system_user_connectability_util(system_user, assets, task_name): +def test_system_user_connectivity_util(system_user, assets, task_name): """ Test system cant connect his assets or not. :param system_user: @@ -281,51 +247,45 @@ def test_system_user_connectability_util(system_user, assets, task_name): :return: """ from ops.utils import update_or_create_ansible_task - # assets = system_user.get_assets() - hosts = [asset.fullname for asset in assets if asset.is_active and asset.is_unixlike()] tasks = const.TEST_SYSTEM_USER_CONN_TASKS + hosts = clean_hosts(assets) if not hosts: - logger.info("No hosts, passed") return {} task, created = update_or_create_ansible_task( task_name, hosts=hosts, tasks=tasks, pattern='all', options=const.TASK_OPTIONS, - run_as=system_user.name, created_by="System", + run_as=system_user, created_by=system_user.org_id, ) result = task.run() - set_system_user_connectablity_info(result, system_user=system_user.name) + set_system_user_connectivity_info(system_user, result) return result @shared_task -def test_system_user_connectability_manual(system_user): - task_name = _("Test system user connectability: {}").format(system_user) - assets = system_user.get_assets() - return test_system_user_connectability_util(system_user, assets, task_name) +def test_system_user_connectivity_manual(system_user): + task_name = _("Test system user connectivity: {}").format(system_user) + assets = system_user.get_related_assets() + return test_system_user_connectivity_util(system_user, assets, task_name) @shared_task -def test_system_user_connectability_a_asset(system_user, asset): - task_name = _("Test system user connectability: {} => {}").format( +def test_system_user_connectivity_a_asset(system_user, asset): + task_name = _("Test system user connectivity: {} => {}").format( system_user, asset ) - return test_system_user_connectability_util(system_user, [asset], task_name) + return test_system_user_connectivity_util(system_user, [asset], task_name) @shared_task -@register_as_period_task(interval=3600) -@after_app_ready_start -@after_app_shutdown_clean -def test_system_user_connectability_period(): +def test_system_user_connectivity_period(): if PERIOD_TASK != "on": - logger.debug("Period task disabled, test system user connectability pass") + logger.debug("Period task disabled, test system user connectivity pass") return - system_users = SystemUser.objects.all() for system_user in system_users: - task_name = _("Test system user connectability period: {}".format(system_user)) - # task_name = _("定期测试系统用户可连接性: {}".format(system_user)) - test_system_user_connectability_util(system_user, task_name) + task_name = _("Test system user connectivity period: {}").format(system_user) + assets = system_user.get_related_assets() + test_system_user_connectivity_util(system_user, assets, task_name) #### Push system user tasks #### @@ -347,6 +307,24 @@ def get_push_system_user_tasks(system_user): ), } }) + tasks.extend([ + { + 'name': 'Check home dir exists', + 'action': { + 'module': 'stat', + 'args': 'path=/home/{}'.format(system_user.username) + }, + 'register': 'home_existed' + }, + { + 'name': "Set home dir permission", + 'action': { + 'module': 'file', + 'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username) + }, + 'when': 'home_existed.stat.exists == true' + } + ]) if system_user.public_key: tasks.append({ 'name': 'Set {} authorized key'.format(system_user.username), @@ -374,53 +352,57 @@ def get_push_system_user_tasks(system_user): @shared_task -def push_system_user_util(system_users, assets, task_name): +def push_system_user_util(system_user, assets, task_name): from ops.utils import update_or_create_ansible_task - tasks = [] - for system_user in system_users: - if not system_user.is_need_push(): - msg = "push system user `{}` passed, may be not auto push or ssh " \ - "protocol is not ssh".format(system_user.name) - logger.info(msg) - continue - tasks.extend(get_push_system_user_tasks(system_user)) + if not system_user.is_need_push(): + msg = _("Push system user task skip, auto push not enable or " + "protocol is not ssh: {}").format(system_user.name) + logger.info(msg) + return - if not tasks: - logger.info("Not tasks, passed") - return {} - - hosts = [asset.fullname for asset in assets if asset.is_active and asset.is_unixlike()] + tasks = get_push_system_user_tasks(system_user) + hosts = clean_hosts(assets) if not hosts: - logger.info("Not hosts, passed") return {} task, created = update_or_create_ansible_task( task_name=task_name, hosts=hosts, tasks=tasks, pattern='all', - options=const.TASK_OPTIONS, run_as_admin=True, created_by='System' + options=const.TASK_OPTIONS, run_as_admin=True, + created_by=system_user.org_id, ) return task.run() @shared_task def push_system_user_to_assets_manual(system_user): - assets = system_user.get_assets() - # task_name = "推送系统用户到入资产: {}".format(system_user.name) + assets = system_user.get_related_assets() task_name = _("Push system users to assets: {}").format(system_user.name) - return push_system_user_util([system_user], assets, task_name=task_name) + return push_system_user_util(system_user, assets, task_name=task_name) @shared_task def push_system_user_a_asset_manual(system_user, asset): task_name = _("Push system users to asset: {} => {}").format( - system_user.name, asset.fullname + system_user.name, asset ) - return push_system_user_util([system_user], [asset], task_name=task_name) + return push_system_user_util(system_user, [asset], task_name=task_name) @shared_task def push_system_user_to_assets(system_user, assets): - # task_name = _("推送系统用户到入资产: {}").format(system_user.name) task_name = _("Push system users to assets: {}").format(system_user.name) - return push_system_user_util.delay([system_user], assets, task_name) + return push_system_user_util(system_user, assets, task_name) + + +@shared_task +@after_app_shutdown_clean +def test_system_user_connectability_period(): + pass + + +@shared_task +@after_app_shutdown_clean +def test_admin_user_connectability_period(): + pass # @shared_task diff --git a/apps/assets/templates/assets/_asset_list_modal.html b/apps/assets/templates/assets/_asset_list_modal.html index ea8d59e49..d62dc7a59 100644 --- a/apps/assets/templates/assets/_asset_list_modal.html +++ b/apps/assets/templates/assets/_asset_list_modal.html @@ -57,6 +57,10 @@ {% endblock %} \ No newline at end of file diff --git a/apps/assets/templates/assets/domain_create_update.html b/apps/assets/templates/assets/domain_create_update.html index 3ad8724e9..7a31e3e88 100644 --- a/apps/assets/templates/assets/domain_create_update.html +++ b/apps/assets/templates/assets/domain_create_update.html @@ -24,17 +24,27 @@ {% block custom_foot_js %} diff --git a/apps/assets/templates/assets/system_user_asset.html b/apps/assets/templates/assets/system_user_asset.html index bf778d46f..9bcf6b5aa 100644 --- a/apps/assets/templates/assets/system_user_asset.html +++ b/apps/assets/templates/assets/system_user_asset.html @@ -136,7 +136,7 @@ {% block custom_foot_js %} {% endblock %} diff --git a/apps/ops/templates/ops/adhoc_history_detail.html b/apps/ops/templates/ops/adhoc_history_detail.html index b8a48d4e5..5091470d5 100644 --- a/apps/ops/templates/ops/adhoc_history_detail.html +++ b/apps/ops/templates/ops/adhoc_history_detail.html @@ -19,7 +19,7 @@ {% trans 'Run history detail' %}
+ | {% trans 'Hosts' %} | +{% trans 'User' %} | +{% trans 'Command' %} | +{% trans 'Run as' %} | +{% trans 'Output' %} | +{% trans 'Finished' %} | +{% trans 'Success' %} | +{% trans 'Date start' %} | + + + {% for object in object_list %} +
---|---|---|---|---|---|---|---|---|
{{ forloop.counter }} | ++ | {{ object.user.name }} | +{{ object.command| truncatechars:16 }} | +{{ object.run_as.username }} | +查看 | +{{ object.is_finished | state_show | safe }} | +{{ object.is_success | state_show | safe }} | +{{ object.date_start }} | +