Merge pull request #1171 from jumpserver/dev

Dev

apps/__init__.py

@@ -2,4 +2,4 @@
 # -*- coding: utf-8 -*-
 # 
 
-__version__ = "0.5.0"
+__version__ = "1.0.0"

apps/assets/api/__init__.py

@@ -3,3 +3,4 @@ from .asset import *
 from .label import *
 from .system_user import *
 from .node import *
+from .domain import *

apps/assets/api/admin_user.py

@@ -28,7 +28,8 @@ from ..tasks import test_admin_user_connectability_manual
 
 logger = get_logger(__file__)
 __all__ = [
-    'AdminUserViewSet', 'ReplaceNodesAdminUserApi', 'AdminUserTestConnectiveApi'
+    'AdminUserViewSet', 'ReplaceNodesAdminUserApi',
+    'AdminUserTestConnectiveApi', 'AdminUserAuthApi',
 ]
 
 
@@ -41,6 +42,12 @@ class AdminUserViewSet(IDInFilterMixin, BulkModelViewSet):
     permission_classes = (IsSuperUser,)
 
 
+class AdminUserAuthApi(generics.UpdateAPIView):
+    queryset = AdminUser.objects.all()
+    serializer_class = serializers.AdminUserAuthSerializer
+    permission_classes = (IsSuperUser,)
+
+
 class ReplaceNodesAdminUserApi(generics.UpdateAPIView):
     queryset = AdminUser.objects.all()
     serializer_class = serializers.ReplaceNodeAdminUserSerializer
@@ -72,5 +79,5 @@ class AdminUserTestConnectiveApi(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         admin_user = self.get_object()
-        test_admin_user_connectability_manual.delay(admin_user)
+        task = test_admin_user_connectability_manual.delay(admin_user)
-        return Response({"msg": "Task created"})
+        return Response({"task": task.id})

apps/assets/api/asset.py

@@ -87,12 +87,8 @@ class AssetRefreshHardwareApi(generics.RetrieveAPIView):
     def retrieve(self, request, *args, **kwargs):
         asset_id = kwargs.get('pk')
         asset = get_object_or_404(Asset, pk=asset_id)
-        summary = update_asset_hardware_info_manual(asset)[1]
+        task = update_asset_hardware_info_manual.delay(asset)
-        logger.debug("Refresh summary: {}".format(summary))
+        return Response({"task": task.id})
-        if summary.get('dark'):
-            return Response(summary['dark'].values(), status=501)
-        else:
-            return Response({"msg": "ok"})
 
 
 class AssetAdminUserTestApi(generics.RetrieveAPIView):
@@ -105,8 +101,5 @@ class AssetAdminUserTestApi(generics.RetrieveAPIView):
     def retrieve(self, request, *args, **kwargs):
         asset_id = kwargs.get('pk')
         asset = get_object_or_404(Asset, pk=asset_id)
-        ok, msg = test_asset_connectability_manual(asset)
+        task = test_asset_connectability_manual.delay(asset)
-        if ok:
+        return Response({"task": task.id})
-            return Response({"msg": "pong"})
-        else:
-            return Response({"error": msg}, status=502)

apps/assets/api/domain.py

@@ -0,0 +1,55 @@
+# ~*~ coding: utf-8 ~*~
+
+from rest_framework_bulk import BulkModelViewSet
+from rest_framework.views import APIView, Response
+from rest_framework.generics import RetrieveAPIView
+
+from django.views.generic.detail import SingleObjectMixin
+
+from common.utils import get_logger
+from ..hands import IsSuperUser, IsSuperUserOrAppUser
+from ..models import Domain, Gateway
+from ..utils import test_gateway_connectability
+from .. import serializers
+
+
+logger = get_logger(__file__)
+__all__ = ['DomainViewSet', 'GatewayViewSet', "GatewayTestConnectionApi"]
+
+
+class DomainViewSet(BulkModelViewSet):
+    queryset = Domain.objects.all()
+    permission_classes = (IsSuperUser,)
+    serializer_class = serializers.DomainSerializer
+
+    def get_serializer_class(self):
+        if self.request.query_params.get('gateway'):
+            return serializers.DomainWithGatewaySerializer
+        return super().get_serializer_class()
+
+    def get_permissions(self):
+        if self.request.query_params.get('gateway'):
+            self.permission_classes = (IsSuperUserOrAppUser,)
+        return super().get_permissions()
+
+
+class GatewayViewSet(BulkModelViewSet):
+    filter_fields = ("domain",)
+    search_fields = filter_fields
+    queryset = Gateway.objects.all()
+    permission_classes = (IsSuperUser,)
+    serializer_class = serializers.GatewaySerializer
+
+
+class GatewayTestConnectionApi(SingleObjectMixin, APIView):
+    permission_classes = (IsSuperUser,)
+    model = Gateway
+    object = None
+
+    def get(self, request, *args, **kwargs):
+        self.object = self.get_object(Gateway.objects.all())
+        ok, e = test_gateway_connectability(self.object)
+        if ok:
+            return Response("ok")
+        else:
+            return Response({"failed": e}, status=404)

apps/assets/api/node.py

@@ -130,10 +130,9 @@ class RefreshNodeHardwareInfoApi(APIView):
         node_id = kwargs.get('pk')
         node = get_object_or_404(self.model, id=node_id)
         assets = node.assets.all()
-        # task_name = _("Refresh node assets hardware info: {}".format(node.name))
         task_name = _("更新节点资产硬件信息: {}".format(node.name))
-        update_assets_hardware_info_util.delay(assets, task_name=task_name)
+        task = update_assets_hardware_info_util.delay(assets, task_name=task_name)
-        return Response({"msg": "Task created"})
+        return Response({"task": task.id})
 
 
 class TestNodeConnectiveApi(APIView):
@@ -145,6 +144,6 @@ class TestNodeConnectiveApi(APIView):
         node = get_object_or_404(self.model, id=node_id)
         assets = node.assets.all()
         task_name = _("测试节点下资产是否可连接: {}".format(node.name))
-        test_asset_connectability_util.delay(assets, task_name=task_name)
+        task = test_asset_connectability_util.delay(assets, task_name=task_name)
-        return Response({"msg": "Task created"})
+        return Response({"task": task.id})
 

apps/assets/api/system_user.py

@@ -48,15 +48,6 @@ class SystemUserAuthInfoApi(generics.RetrieveUpdateAPIView):
     permission_classes = (IsSuperUserOrAppUser,)
     serializer_class = serializers.SystemUserAuthSerializer
 
-    def update(self, request, *args, **kwargs):
-        password = request.data.pop("password", None)
-        private_key = request.data.pop("private_key", None)
-        instance = self.get_object()
-
-        if password or private_key:
-            instance.set_auth(password=password, private_key=private_key)
-        return super().update(request, *args, **kwargs)
-
 
 class SystemUserPushApi(generics.RetrieveAPIView):
     """
@@ -67,8 +58,8 @@ class SystemUserPushApi(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         system_user = self.get_object()
-        push_system_user_to_assets_manual.delay(system_user)
+        task = push_system_user_to_assets_manual.delay(system_user)
-        return Response({"msg": "Task created"})
+        return Response({"task": task.id})
 
 
 class SystemUserTestConnectiveApi(generics.RetrieveAPIView):
@@ -80,5 +71,5 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         system_user = self.get_object()
-        test_system_user_connectability_manual.delay(system_user)
+        task = test_system_user_connectability_manual.delay(system_user)
-        return Response({"msg": "Task created"})
+        return Response({"task": task.id})

apps/assets/forms/__init__.py

@@ -3,3 +3,4 @@
 from .asset import *
 from .label import *
 from .user import *
+from .domain import *

apps/assets/forms/asset.py

@@ -16,6 +16,7 @@ class AssetCreateForm(forms.ModelForm):
         fields = [
             'hostname', 'ip', 'public_ip', 'port',  'comment',
             'nodes', 'is_active', 'admin_user', 'labels', 'platform',
+            'domain',
 
         ]
         widgets = {
@@ -29,6 +30,9 @@ class AssetCreateForm(forms.ModelForm):
                 'class': 'select2', 'data-placeholder': _('Labels')
             }),
             'port': forms.TextInput(),
+            'domain': forms.Select(attrs={
+                'class': 'select2', 'data-placeholder': _('Domain')
+            }),
         }
         help_texts = {
             'hostname': '* required',
@@ -38,7 +42,8 @@ class AssetCreateForm(forms.ModelForm):
                 'root or other NOPASSWD sudo privilege user existed in asset,'
                 'If asset is windows or other set any one, more see admin user left menu'
             ),
-            'platform': _("* required Must set exact system platform, Windows, Linux ...")
+            'platform': _("* required Must set exact system platform, Windows, Linux ..."),
+            'domain': _("If your have some network not connect with each other, you can set domain")
         }
 
 
@@ -48,6 +53,7 @@ class AssetUpdateForm(forms.ModelForm):
         fields = [
             'hostname', 'ip', 'port', 'nodes',  'is_active', 'platform',
             'public_ip', 'number', 'comment', 'admin_user', 'labels',
+            'domain',
         ]
         widgets = {
             'nodes': forms.SelectMultiple(attrs={
@@ -60,6 +66,9 @@ class AssetUpdateForm(forms.ModelForm):
                 'class': 'select2', 'data-placeholder': _('Labels')
             }),
             'port': forms.TextInput(),
+            'domain': forms.Select(attrs={
+                'class': 'select2', 'data-placeholder': _('Domain')
+            }),
         }
         help_texts = {
             'hostname': '* required',
@@ -70,7 +79,8 @@ class AssetUpdateForm(forms.ModelForm):
                 'root or other NOPASSWD sudo privilege user existed in asset,'
                 'If asset is windows or other set any one, more see admin user left menu'
             ),
-            'platform': _("* required Must set exact system platform, Windows, Linux ...")
+            'platform': _("* required Must set exact system platform, Windows, Linux ..."),
+            'domain': _("If your have some network not connect with each other, you can set domain")
         }
 
 

apps/assets/forms/domain.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+from django import forms
+from django.utils.translation import gettext_lazy as _
+
+from ..models import Domain, Asset, Gateway
+from .user import PasswordAndKeyAuthForm
+
+__all__ = ['DomainForm', 'GatewayForm']
+
+
+class DomainForm(forms.ModelForm):
+    assets = forms.ModelMultipleChoiceField(
+        queryset=Asset.objects.all(), label=_('Asset'), required=False,
+        widget=forms.SelectMultiple(
+            attrs={'class': 'select2', 'data-placeholder': _('Select assets')}
+        )
+    )
+
+    class Meta:
+        model = Domain
+        fields = ['name', 'comment', 'assets']
+
+    def __init__(self, *args, **kwargs):
+        if kwargs.get('instance', None):
+            initial = kwargs.get('initial', {})
+            initial['assets'] = kwargs['instance'].assets.all()
+        super().__init__(*args, **kwargs)
+
+    def save(self, commit=True):
+        instance = super().save(commit=commit)
+        assets = self.cleaned_data['assets']
+        instance.assets.set(assets)
+        return instance
+
+
+class GatewayForm(PasswordAndKeyAuthForm):
+
+    def save(self, commit=True):
+        # Because we define custom field, so we need rewrite :method: `save`
+        instance = super().save()
+        password = self.cleaned_data.get('password')
+        private_key, public_key = super().gen_keys()
+        instance.set_auth(password=password, private_key=private_key)
+        return instance
+
+    class Meta:
+        model = Gateway
+        fields = [
+            'name', 'ip', 'port', 'username', 'protocol', 'domain', 'password',
+            'private_key_file',  'is_active', 'comment',
+        ]
+        widgets = {
+            'name': forms.TextInput(attrs={'placeholder': _('Name')}),
+            'username': forms.TextInput(attrs={'placeholder': _('Username')}),
+        }
+        help_texts = {
+            'name': '* required',
+            'username': '* required',
+        }

apps/assets/forms/user.py

@@ -8,7 +8,7 @@ from common.utils import validate_ssh_private_key, ssh_pubkey_gen, get_logger
 
 logger = get_logger(__file__)
 __all__ = [
-    'FileForm', 'SystemUserForm', 'AdminUserForm',
+    'FileForm', 'SystemUserForm', 'AdminUserForm', 'PasswordAndKeyAuthForm',
 ]
 
 
@@ -114,22 +114,15 @@ class SystemUserForm(PasswordAndKeyAuthForm):
         fields = [
             'name', 'username', 'protocol', 'auto_generate_key',
             'password', 'private_key_file', 'auto_push', 'sudo',
-            'comment', 'shell', 'nodes', 'priority',
+            'comment', 'shell', 'priority',
         ]
         widgets = {
             'name': forms.TextInput(attrs={'placeholder': _('Name')}),
             'username': forms.TextInput(attrs={'placeholder': _('Username')}),
-            'nodes': forms.SelectMultiple(
-                attrs={
-                    'class': 'select2',
-                    'data-placeholder': _('Nodes')
-                }
-            ),
         }
         help_texts = {
             'name': '* required',
             'username': '* required',
-            'nodes': _('If auto push checked, system user will be create at node assets'),
             'auto_push': _('Auto push system user to asset'),
             'priority': _('High level will be using login asset as default, if user was granted more than 2 system user'),
         }

apps/assets/models/__init__.py

@@ -5,6 +5,7 @@ from .user import AdminUser, SystemUser
 from .label import Label
 from .cluster import *
 from .group import *
+from .domain import *
 from .node import *
 from .asset import *
 from .utils import *

apps/assets/models/asset.py

@@ -4,14 +4,13 @@
 
 import uuid
 import logging
+import random
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.core.cache import cache
 
 from ..const import ASSET_ADMIN_CONN_CACHE_KEY
-from .cluster import Cluster
-from .group import AssetGroup
 from .user import AdminUser, SystemUser
 
 __all__ = ['Asset']
@@ -50,6 +49,7 @@ class Asset(models.Model):
     ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True)
     hostname = models.CharField(max_length=128, unique=True, verbose_name=_('Hostname'))
     port = models.IntegerField(default=22, verbose_name=_('Port'))
+    domain = models.ForeignKey("assets.Domain", null=True, blank=True, related_name='assets', verbose_name=_("Domain"), on_delete=models.SET_NULL)
     nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets', verbose_name=_("Nodes"))
     is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
 
@@ -96,7 +96,7 @@ class Asset(models.Model):
         return False, warning
 
     def is_unixlike(self):
-        if self.platform not in ("Windows", "Other"):
+        if self.platform not in ("Windows",):
             return True
         else:
             return False
@@ -122,12 +122,24 @@ class Asset(models.Model):
             return False
 
     def to_json(self):
-        return {
+        info = {
             'id': self.id,
             'hostname': self.hostname,
             'ip': self.ip,
             'port': self.port,
         }
+        if self.domain and self.domain.gateway_set.all():
+            info["gateways"] = [d.id for d in self.domain.gateway_set.all()]
+        return info
+
+    def get_auth_info(self):
+        if self.admin_user:
+            return {
+                'username': self.admin_user.username,
+                'password': self.admin_user.password,
+                'private_key': self.admin_user.private_key_file,
+                'become': self.admin_user.become_info,
+            }
 
     def _to_secret_json(self):
         """
@@ -168,9 +180,7 @@ class Asset(models.Model):
             try:
                 asset.save()
                 asset.system_users = [choice(SystemUser.objects.all()) for i in range(3)]
-                asset.groups = [choice(AssetGroup.objects.all()) for i in range(3)]
                 logger.debug('Generate fake asset : %s' % asset.ip)
             except IntegrityError:
                 print('Error continue')
                 continue
-

apps/assets/models/base.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+#
+import os
+import uuid
+from hashlib import md5
+
+import sshpubkeys
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from django.conf import settings
+
+from common.utils import get_signer, ssh_key_string_to_obj, ssh_key_gen
+from .utils import private_key_validator
+
+signer = get_signer()
+
+
+class AssetUser(models.Model):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    name = models.CharField(max_length=128, unique=True, verbose_name=_('Name'))
+    username = models.CharField(max_length=128, verbose_name=_('Username'))
+    _password = models.CharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
+    _private_key = models.TextField(max_length=4096, blank=True, null=True, verbose_name=_('SSH private key'), validators=[private_key_validator, ])
+    _public_key = models.TextField(max_length=4096, blank=True, verbose_name=_('SSH public key'))
+    comment = models.TextField(blank=True, verbose_name=_('Comment'))
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_updated = models.DateTimeField(auto_now=True)
+    created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by'))
+
+    @property
+    def password(self):
+        if self._password:
+            return signer.unsign(self._password)
+        else:
+            return None
+
+    @password.setter
+    def password(self, password_raw):
+        raise AttributeError("Using set_auth do that")
+        # self._password = signer.sign(password_raw)
+
+    @property
+    def private_key(self):
+        if self._private_key:
+            return signer.unsign(self._private_key)
+
+    @private_key.setter
+    def private_key(self, private_key_raw):
+        raise AttributeError("Using set_auth do that")
+        # self._private_key = signer.sign(private_key_raw)
+
+    @property
+    def private_key_obj(self):
+        if self._private_key:
+            key_str = signer.unsign(self._private_key)
+            return ssh_key_string_to_obj(key_str, password=self.password)
+        else:
+            return None
+
+    @property
+    def private_key_file(self):
+        if not self.private_key_obj:
+            return None
+        project_dir = settings.PROJECT_DIR
+        tmp_dir = os.path.join(project_dir, 'tmp')
+        key_str = signer.unsign(self._private_key)
+        key_name = '.' + md5(key_str.encode('utf-8')).hexdigest()
+        key_path = os.path.join(tmp_dir, key_name)
+        if not os.path.exists(key_path):
+            self.private_key_obj.write_private_key_file(key_path)
+            os.chmod(key_path, 0o400)
+        return key_path
+
+    @property
+    def public_key(self):
+        key = signer.unsign(self._public_key)
+        if key:
+            return key
+        else:
+            return None
+
+    @property
+    def public_key_obj(self):
+        if self.public_key:
+            try:
+                return sshpubkeys.SSHKey(self.public_key)
+            except TabError:
+                pass
+        return None
+
+    def set_auth(self, password=None, private_key=None, public_key=None):
+        update_fields = []
+        if password:
+            self._password = signer.sign(password)
+            update_fields.append('_password')
+        if private_key:
+            self._private_key = signer.sign(private_key)
+            update_fields.append('_private_key')
+        if public_key:
+            self._public_key = signer.sign(public_key)
+            update_fields.append('_public_key')
+
+        if update_fields:
+            self.save(update_fields=update_fields)
+
+    def auto_gen_auth(self):
+        password = str(uuid.uuid4())
+        private_key, public_key = ssh_key_gen(
+            username=self.name, password=password
+        )
+        self.set_auth(password=password,
+                      private_key=private_key,
+                      public_key=public_key)
+
+    def _to_secret_json(self):
+        """Push system user use it"""
+        return {
+            'name': self.name,
+            'username': self.username,
+            'password': self.password,
+            'public_key': self.public_key,
+            'private_key': self.private_key_file,
+        }
+
+    class Meta:
+        abstract = True

apps/assets/models/domain.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+
+import uuid
+import random
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from .base import AssetUser
+
+__all__ = ['Domain', 'Gateway']
+
+
+class Domain(models.Model):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    name = models.CharField(max_length=128, unique=True, verbose_name=_('Name'))
+    comment = models.TextField(blank=True, verbose_name=_('Comment'))
+    date_created = models.DateTimeField(auto_now_add=True, null=True,
+                                        verbose_name=_('Date created'))
+
+    def __str__(self):
+        return self.name
+
+    def has_gateway(self):
+        return self.gateway_set.filter(is_active=True).exists()
+
+    @property
+    def gateways(self):
+        return self.gateway_set.filter(is_active=True)
+
+    def random_gateway(self):
+        return random.choice(self.gateways)
+
+
+class Gateway(AssetUser):
+    SSH_PROTOCOL = 'ssh'
+    RDP_PROTOCOL = 'rdp'
+    PROTOCOL_CHOICES = (
+        (SSH_PROTOCOL, 'ssh'),
+        (RDP_PROTOCOL, 'rdp'),
+    )
+    ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True)
+    port = models.IntegerField(default=22, verbose_name=_('Port'))
+    protocol = models.CharField(choices=PROTOCOL_CHOICES, max_length=16, default=SSH_PROTOCOL, verbose_name=_("Protocol"))
+    domain = models.ForeignKey(Domain, verbose_name=_("Domain"))
+    comment = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Comment"))
+    is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
+
+    def __str__(self):
+        return self.name
+

apps/assets/models/user.py

@@ -2,20 +2,15 @@
 # -*- coding: utf-8 -*-
 #
 
-import os
 import logging
-import uuid
-from hashlib import md5
 
-import sshpubkeys
 from django.core.cache import cache
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from django.conf import settings
 
-from common.utils import get_signer, ssh_key_string_to_obj, ssh_key_gen
+from common.utils import get_signer
-from .utils import private_key_validator
 from ..const import SYSTEM_USER_CONN_CACHE_KEY
+from .base import AssetUser
 
 
 __all__ = ['AdminUser', 'SystemUser',]
@@ -23,117 +18,6 @@ logger = logging.getLogger(__name__)
 signer = get_signer()
 
 
-class AssetUser(models.Model):
-    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    name = models.CharField(max_length=128, unique=True, verbose_name=_('Name'))
-    username = models.CharField(max_length=128, verbose_name=_('Username'))
-    _password = models.CharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
-    _private_key = models.TextField(max_length=4096, blank=True, null=True, verbose_name=_('SSH private key'), validators=[private_key_validator, ])
-    _public_key = models.TextField(max_length=4096, blank=True, verbose_name=_('SSH public key'))
-    comment = models.TextField(blank=True, verbose_name=_('Comment'))
-    date_created = models.DateTimeField(auto_now_add=True)
-    date_updated = models.DateTimeField(auto_now=True)
-    created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by'))
-
-    @property
-    def password(self):
-        if self._password:
-            return signer.unsign(self._password)
-        else:
-            return None
-
-    @password.setter
-    def password(self, password_raw):
-        raise AttributeError("Using set_auth do that")
-        # self._password = signer.sign(password_raw)
-
-    @property
-    def private_key(self):
-        if self._private_key:
-            return signer.unsign(self._private_key)
-
-    @private_key.setter
-    def private_key(self, private_key_raw):
-        raise AttributeError("Using set_auth do that")
-        # self._private_key = signer.sign(private_key_raw)
-
-    @property
-    def private_key_obj(self):
-        if self._private_key:
-            key_str = signer.unsign(self._private_key)
-            return ssh_key_string_to_obj(key_str, password=self.password)
-        else:
-            return None
-
-    @property
-    def private_key_file(self):
-        if not self.private_key_obj:
-            return None
-        project_dir = settings.PROJECT_DIR
-        tmp_dir = os.path.join(project_dir, 'tmp')
-        key_str = signer.unsign(self._private_key)
-        key_name = '.' + md5(key_str.encode('utf-8')).hexdigest()
-        key_path = os.path.join(tmp_dir, key_name)
-        if not os.path.exists(key_path):
-            self.private_key_obj.write_private_key_file(key_path)
-            os.chmod(key_path, 0o400)
-        return key_path
-
-    @property
-    def public_key(self):
-        key = signer.unsign(self._public_key)
-        if key:
-            return key
-        else:
-            return None
-
-    @property
-    def public_key_obj(self):
-        if self.public_key:
-            try:
-                return sshpubkeys.SSHKey(self.public_key)
-            except TabError:
-                pass
-        return None
-
-    def set_auth(self, password=None, private_key=None, public_key=None):
-        update_fields = []
-        if password:
-            self._password = signer.sign(password)
-            update_fields.append('_password')
-        if private_key:
-            self._private_key = signer.sign(private_key)
-            update_fields.append('_private_key')
-        if public_key:
-            self._public_key = signer.sign(public_key)
-            update_fields.append('_public_key')
-
-        if update_fields:
-            self.save(update_fields=update_fields)
-
-    def auto_gen_auth(self):
-        password = str(uuid.uuid4())
-        private_key, public_key = ssh_key_gen(
-            username=self.name, password=password
-        )
-        self.set_auth(password=password,
-                      private_key=private_key,
-                      public_key=public_key)
-
-    def _to_secret_json(self):
-        """Push system user use it"""
-        return {
-            'name': self.name,
-            'username': self.username,
-            'password': self.password,
-            'public_key': self.public_key,
-            'private_key': self.private_key_file,
-        }
-
-    class Meta:
-        abstract = True
-
-
 class AdminUser(AssetUser):
     """
     A privileged user that ansible can use it to push system user and so on

apps/assets/models/utils.py

@@ -10,15 +10,15 @@ __all__ = ['init_model', 'generate_fake']
 
 
 def init_model():
-    from . import Cluster, SystemUser, AdminUser, AssetGroup, Asset
+    from . import SystemUser, AdminUser, Asset
-    for cls in [Cluster, SystemUser, AdminUser, AssetGroup, Asset]:
+    for cls in [SystemUser, AdminUser, Asset]:
         if hasattr(cls, 'initial'):
             cls.initial()
 
 
 def generate_fake():
-    from . import Cluster, SystemUser, AdminUser, AssetGroup, Asset
+    from . import SystemUser, AdminUser,  Asset
-    for cls in [Cluster, SystemUser, AdminUser, AssetGroup, Asset]:
+    for cls in [SystemUser, AdminUser, Asset]:
         if hasattr(cls, 'generate_fake'):
             cls.generate_fake()
 

apps/assets/serializers/__init__.py

@@ -6,3 +6,4 @@ from .admin_user import *
 from .label import *
 from .system_user import *
 from .node import *
+from .domain import *

apps/assets/serializers/admin_user.py

@@ -2,9 +2,12 @@
 #
 from django.core.cache import cache
 from rest_framework import serializers
+
 from ..models import Node, AdminUser
 from ..const import ADMIN_USER_CONN_CACHE_KEY
 
+from .base import AuthSerializer
+
 
 class AdminUserSerializer(serializers.ModelSerializer):
     """
@@ -18,6 +21,10 @@ class AdminUserSerializer(serializers.ModelSerializer):
         model = AdminUser
         fields = '__all__'
 
+    def get_field_names(self, declared_fields, info):
+        fields = super().get_field_names(declared_fields, info)
+        return [f for f in fields if not f.startswith('_')]
+
     @staticmethod
     def get_unreachable_amount(obj):
         data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name))
@@ -39,6 +46,13 @@ class AdminUserSerializer(serializers.ModelSerializer):
         return obj.assets_amount
 
 
+class AdminUserAuthSerializer(AuthSerializer):
+
+    class Meta:
+        model = AdminUser
+        fields = ['password', 'private_key']
+
+
 class ReplaceNodeAdminUserSerializer(serializers.ModelSerializer):
     """
     管理用户更新关联到的集群
@@ -50,3 +64,6 @@ class ReplaceNodeAdminUserSerializer(serializers.ModelSerializer):
     class Meta:
         model = AdminUser
         fields = ['id', 'nodes']
+
+
+

apps/assets/serializers/asset.py

@@ -38,7 +38,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer):
         model = Asset
         fields = (
             "id", "hostname", "ip", "port", "system_users_granted",
-            "is_active", "system_users_join", "os",
+            "is_active", "system_users_join", "os", 'domain',
             "platform", "comment"
         )
 

apps/assets/serializers/base.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+
+from rest_framework import serializers
+from common.utils import ssh_pubkey_gen
+
+
+class AuthSerializer(serializers.ModelSerializer):
+    password = serializers.CharField(required=False, allow_blank=True, allow_null=True, max_length=1024)
+    private_key = serializers.CharField(required=False, allow_blank=True, allow_null=True, max_length=4096)
+
+    def gen_keys(self, private_key=None, password=None):
+        if private_key is None:
+            return None, None
+        public_key = ssh_pubkey_gen(private_key=private_key, password=password)
+        return private_key, public_key
+
+    def save(self, **kwargs):
+        password = self.validated_data.pop('password', None) or None
+        private_key = self.validated_data.pop('private_key', None) or None
+        self.instance = super().save(**kwargs)
+        if password or private_key:
+            private_key, public_key = self.gen_keys(private_key, password)
+            self.instance.set_auth(password=password, private_key=private_key,
+                                   public_key=public_key)
+        return self.instance

apps/assets/serializers/cluster.py

@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-
-from rest_framework import serializers
-from common.mixins import BulkSerializerMixin
-from ..models import Asset, Cluster
-
-
-class ClusterUpdateAssetsSerializer(serializers.ModelSerializer):
-    """
-    集群更新资产数据结构
-    """
-    assets = serializers.PrimaryKeyRelatedField(many=True, queryset=Asset.objects.all())
-
-    class Meta:
-        model = Cluster
-        fields = ['id', 'assets']
-
-
-class ClusterSerializer(BulkSerializerMixin, serializers.ModelSerializer):
-    """
-    cluster
-    """
-    assets_amount = serializers.SerializerMethodField()
-    admin_user_name = serializers.SerializerMethodField()
-    assets = serializers.PrimaryKeyRelatedField(many=True, queryset=Asset.objects.all())
-    system_users = serializers.SerializerMethodField()
-
-    class Meta:
-        model = Cluster
-        fields = '__all__'
-
-    @staticmethod
-    def get_assets_amount(obj):
-        return obj.assets.count()
-
-    @staticmethod
-    def get_admin_user_name(obj):
-        try:
-            return obj.admin_user.name
-        except AttributeError:
-            return ''
-
-    @staticmethod
-    def get_system_users(obj):
-        return ', '.join(obj.name for obj in obj.systemuser_set.all())

apps/assets/serializers/domain.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#
+from rest_framework import serializers
+
+from ..models import Domain, Gateway
+
+
+class DomainSerializer(serializers.ModelSerializer):
+    asset_count = serializers.SerializerMethodField()
+    gateway_count = serializers.SerializerMethodField()
+
+    class Meta:
+        model = Domain
+        fields = '__all__'
+
+    @staticmethod
+    def get_asset_count(obj):
+        return obj.assets.count()
+
+    @staticmethod
+    def get_gateway_count(obj):
+        return obj.gateway_set.all().count()
+
+
+class GatewaySerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = Gateway
+        fields = [
+            'id', 'name', 'ip', 'port', 'protocol', 'username',
+            'domain', 'is_active', 'date_created', 'date_updated',
+            'created_by', 'comment',
+        ]
+
+
+class GatewayWithAuthSerializer(GatewaySerializer):
+    def get_field_names(self, declared_fields, info):
+        fields = super().get_field_names(declared_fields, info)
+        fields.extend(
+            ['password', 'private_key']
+        )
+        return fields
+
+
+class DomainWithGatewaySerializer(serializers.ModelSerializer):
+    gateways = GatewayWithAuthSerializer(many=True, read_only=True)
+
+    class Meta:
+        model = Domain
+        fields = '__all__'

apps/assets/serializers/system_user.py

@@ -1,6 +1,7 @@
 from rest_framework import serializers
 
 from ..models import SystemUser
+from .base import AuthSerializer
 
 
 class SystemUserSerializer(serializers.ModelSerializer):
@@ -36,12 +37,10 @@ class SystemUserSerializer(serializers.ModelSerializer):
         return len(obj.assets)
 
 
-class SystemUserAuthSerializer(serializers.ModelSerializer):
+class SystemUserAuthSerializer(AuthSerializer):
     """
     系统用户认证信息
     """
-    password = serializers.CharField(max_length=1024)
-    private_key = serializers.CharField(max_length=4096)
 
     class Meta:
         model = SystemUser

apps/assets/tasks.py

@@ -9,10 +9,11 @@ from django.utils.translation import ugettext as _
 
 from common.utils import get_object_or_none, capacity_convert, \
     sum_capacity, encrypt_password, get_logger
-from common.celery import register_as_period_task, after_app_shutdown_clean, \
+from ops.celery.utils import register_as_period_task, after_app_shutdown_clean, \
-    after_app_ready_start, app as celery_app
+    after_app_ready_start
+from ops.celery import app as celery_app
 
-from .models import SystemUser, AdminUser, Asset, Cluster
+from .models import SystemUser, AdminUser, Asset
 from . import const
 
 
@@ -214,7 +215,7 @@ def test_admin_user_connectability_period():
 def test_admin_user_connectability_manual(admin_user):
     # task_name = _("Test admin user connectability: {}").format(admin_user.name)
     task_name = _("测试管理行号可连接性: {}").format(admin_user.name)
-    return test_admin_user_connectability_util.delay(admin_user, task_name)
+    return test_admin_user_connectability_util(admin_user, task_name)
 
 
 @shared_task
@@ -394,10 +395,12 @@ def get_node_push_system_user_task_name(system_user, node):
     )
 
 
+@shared_task
 def push_system_user_to_node(system_user, node):
+    logger.info("Start push system user node: {} => {}".format(system_user.name, node.value))
     assets = node.get_all_assets()
     task_name = get_node_push_system_user_task_name(system_user, node)
-    push_system_user_util.delay([system_user], assets, task_name)
+    push_system_user_util([system_user], assets, task_name)
 
 
 @shared_task
@@ -438,3 +441,7 @@ def push_node_system_users_to_asset(node, assets):
 # def push_system_user_period():
 #     for system_user in SystemUser.objects.all():
 #         push_system_user_related_nodes(system_user)
+
+
+
+

apps/assets/templates/assets/admin_user_assets.html

@@ -121,14 +121,16 @@ $(document).ready(function () {
 })
 .on('click', '.btn-test-connective', function () {
     var the_url = "{% url 'api-assets:admin-user-connective' pk=admin_user.id %}";
-    var error = function (data) {
+    var success = function (data) {
-        alert(data)
+        var task_id = data.task;
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     };
     APIUpdateAttr({
         url: the_url,
-        error: error,
         method: 'GET',
-        success_message: "{% trans 'Task has been send, seen left asset status' %}"
+        success: success,
+        flash_message: false
     });
 })
 </script>

apps/assets/templates/assets/asset_create.html

@@ -19,6 +19,7 @@
     {% bootstrap_field form.port layout="horizontal" %}
     {% bootstrap_field form.platform layout="horizontal" %}
     {% bootstrap_field form.public_ip layout="horizontal" %}
+    {% bootstrap_field form.domain layout="horizontal" %}
 
     <div class="hr-line-dashed"></div>
     <h3>{% trans 'Auth' %}</h3>

apps/assets/templates/assets/asset_detail.html

@@ -270,15 +270,14 @@ function updateAssetNodes(nodes) {
 function refreshAssetHardware() {
     var the_url = "{% url 'api-assets:asset-refresh' pk=asset.id %}";
     var success = function(data) {
-        location.reload();
+        console.log(data);
-    };
+        var task_id = data.task;
-    var error = function (data) {
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        alert(data)
+        window.open(url, '', 'width=800,height=600')
     };
     APIUpdateAttr({
         url: the_url,
         success: success,
-        error: error,
         method: 'GET'
     });
 }
@@ -344,19 +343,20 @@ $(document).ready(function () {
     var redirect_url = "{% url 'assets:asset-list' %}";
     objectDelete($this, name, the_url, redirect_url);
 }).on('click', '#btn_refresh_asset', function () {
-    alert('关闭alert, 等待完成, 自动刷新页面');
     refreshAssetHardware()
 }).on('click', '#btn-test-is-alive', function () {
     var the_url = "{% url 'api-assets:asset-alive-test' pk=asset.id %}";
-    var error = function (data) {
+
-        alert(data)
+    var success = function(data) {
+        var task_id = data.task;
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     };
-    alert('关闭alert, 等待完成');
+
     APIUpdateAttr({
         url: the_url,
-        error: error,
         method: 'GET',
-        success_message: "{% trans "Reachable" %}"
+        success: success
     });
 })
 

apps/assets/templates/assets/asset_list.html

@@ -451,6 +451,14 @@ $(document).ready(function(){
 })
 .on('click', '#btn_asset_import', function () {
     var $form = $('#fm_asset_import');
+    var action = $form.attr("action");
+    var nodes = zTree.getSelectedNodes();
+    var current_node;
+    if (nodes && nodes.length ===1 ){
+        current_node = nodes[0];
+        action += "?node_id=" + current_node.id;
+        $form.attr("action", action)
+    }
     $form.find('.help-block').remove();
     function success (data) {
         if (data.valid === false) {
@@ -489,14 +497,17 @@ $(document).ready(function(){
     }
 
     var the_url = url.replace("{{ DEFAULT_PK }}", current_node.id);
-    function success() {
+    function success(data) {
         rMenu.css({"visibility" : "hidden"});
+        var task_id = data.task;
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     }
     APIUpdateAttr({
         url: the_url,
         method: "GET",
-        success_message: "更新硬件信息任务下发成功",
+        success: success,
-        success: success
+        flash_message: false
     });
 
 })
@@ -511,14 +522,17 @@ $(document).ready(function(){
     }
 
     var the_url = url.replace("{{ DEFAULT_PK }}", current_node.id);
-    function success() {
+    function success(data) {
         rMenu.css({"visibility" : "hidden"});
+        var task_id = data.task;
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     }
     APIUpdateAttr({
         url: the_url,
         method: "GET",
-        success_message: "测试可连接性任务下发成功",
+        success: success,
-        success: success
+        flash_message: false
     });
 })
 .on('click', '.btn_asset_delete', function () {
@@ -655,6 +669,7 @@ $(document).ready(function(){
         default:
             break;
     }
+    $(".ipt_check_all").prop("checked", false)
 });
 </script>
 

apps/assets/templates/assets/asset_update.html

@@ -24,6 +24,7 @@
     {% bootstrap_field form.port layout="horizontal" %}
     {% bootstrap_field form.platform layout="horizontal" %}
     {% bootstrap_field form.public_ip layout="horizontal" %}
+    {% bootstrap_field form.domain layout="horizontal" %}
 
     <div class="hr-line-dashed"></div>
     <h3>{% trans 'Auth' %}</h3>

apps/assets/templates/assets/domain_create_update.html

@@ -0,0 +1,31 @@
+{% extends '_base_create_update.html' %}
+{% load static %}
+{% load bootstrap3 %}
+{% load i18n %}
+
+{% block form %}
+<form id="groupForm" method="post" class="form-horizontal">
+    {% csrf_token %}
+    {% bootstrap_field form.name layout="horizontal" %}
+    {% bootstrap_field form.assets layout="horizontal" %}
+    {% bootstrap_field form.comment layout="horizontal" %}
+
+    <div class="hr-line-dashed"></div>
+    <div class="form-group">
+        <div class="col-sm-4 col-sm-offset-2">
+            <button class="btn btn-default" type="reset"> {% trans 'Reset' %}</button>
+            <button id="submit_button" class="btn btn-primary" type="submit">{% trans 'Submit' %}</button>
+        </div>
+    </div>
+</form>
+{% endblock %}
+
+{% block custom_foot_js %}
+<script type="text/javascript">
+    $(document).ready(function () {
+        $('.select2').select2({
+            closeOnSelect: false
+        });
+    });
+</script>
+{% endblock %}

apps/assets/templates/assets/domain_detail.html

@@ -0,0 +1,132 @@
+{% extends 'base.html' %}
+{% load static %}
+{% load i18n %}
+
+{% block custom_head_css_js %}
+    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
+    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
+{% endblock %}
+
+{% block content %}
+    <div class="wrapper wrapper-content animated fadeInRight">
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="ibox float-e-margins">
+                    <div class="panel-options">
+                        <ul class="nav nav-tabs">
+                            <li class="active">
+                                <a href="{% url 'assets:domain-detail' pk=object.id %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Detail' %} </a>
+                            </li>
+                            <li>
+                                <a href="{% url 'assets:domain-gateway-list' pk=object.id %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Gateway' %} </a>
+                            </li>
+                            <li class="pull-right">
+                                <a class="btn btn-outline btn-default" href="{% url 'assets:domain-update' pk=object.id %}"><i class="fa fa-edit"></i>{% trans 'Update' %}</a>
+                            </li>
+                            <li class="pull-right">
+                                <a class="btn btn-outline btn-danger btn-del">
+                                    <i class="fa fa-trash-o"></i>{% trans 'Delete' %}
+                                </a>
+                            </li>
+                        </ul>
+                    </div>
+                    <div class="tab-content">
+                        <div class="col-sm-9" style="padding-left: 0;">
+                            <div class="ibox float-e-margins">
+                                <div class="ibox-title">
+                                    <span class="label"><b>{{ object.name }}</b></span>
+                                    <div class="ibox-tools">
+                                        <a class="collapse-link">
+                                            <i class="fa fa-chevron-up"></i>
+                                        </a>
+                                        <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+                                            <i class="fa fa-wrench"></i>
+                                        </a>
+                                        <ul class="dropdown-menu dropdown-user">
+                                        </ul>
+                                        <a class="close-link">
+                                            <i class="fa fa-times"></i>
+                                        </a>
+                                    </div>
+                                </div>
+                                <div class="ibox-content">
+                                    <table class="table">
+                                        <tbody>
+                                        <tr class="no-borders-tr">
+                                            <td>{% trans 'Name' %}:</td>
+                                            <td><b>{{ object.name }}</b></td>
+                                        </tr>
+                                        <tr>
+                                            <td>{% trans 'Asset' %}:</td>
+                                            <td><b>{{ object.assets.count }}</b></td>
+                                        </tr>
+                                        <tr>
+                                            <td>{% trans 'Gateway' %}:</td>
+                                            <td><b>{{ object.gateway_set.count }}</b></td>
+                                        </tr>
+                                        <tr>
+                                            <td>{% trans 'Date created' %}:</td>
+                                            <td><b>{{ object.date_created }}</b></td>
+                                        </tr>
+                                        <tr>
+                                            <td>{% trans 'Created by' %}:</td>
+                                            <td><b>{{ object.created_by  }}</b></td>
+                                        </tr>
+                                        <tr>
+                                            <td>{% trans 'Comment' %}:</td>
+                                            <td><b>{{ object.comment }}</b></td>
+                                        </tr>
+                                        </tbody>
+                                    </table>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}
+{% block content_bottom_left %}{% endblock %}
+{% block custom_foot_js %}
+<script>
+function initTable() {
+    var options = {
+        ele: $('#domain_list_table'),
+        columnDefs: [
+            {targets: 1, createdCell: function (td, cellData, rowData) {
+                var detail_btn = '<a href="{% url "assets:domain-detail" pk=DEFAULT_PK %}">' + cellData + '</a>';
+                $(td).html(detail_btn.replace('{{ DEFAULT_PK }}', rowData.id));
+             }},
+
+            {targets: 5, createdCell: function (td, cellData, rowData) {
+                var update_btn = '<a href="{% url "assets:domain-update" pk=DEFAULT_PK %}" class="btn btn-xs btn-info">{% trans "Update" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                var del_btn = '<a class="btn btn-xs btn-danger m-l-xs btn-delete" data-uid="{{ DEFAULT_PK }}">{% trans "Delete" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                $(td).html(update_btn + del_btn)
+             }}
+        ],
+        ajax_url: '{% url "api-assets:domain-list" %}',
+        columns: [
+            {data: "id"}, {data: "name" }, {data: "asset_count" },
+            {data: "gateway_count" }, {data: "comment" }, {data: "id"}
+        ],
+        op_html: $('#actions').html()
+    };
+    jumpserver.initDataTable(options);
+}
+$(document).ready(function(){
+    initTable();
+})
+.on('click', '.btn-delete', function () {
+    var $this = $(this);
+    var $data_table = $('#domain_list_table').DataTable();
+    var name = $(this).closest("tr").find(":nth-child(2)").children('a').html();
+    var uid = $this.data('uid');
+    var the_url = '{% url "api-assets:domain-detail" pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', uid);
+    objectDelete($this, name, the_url);
+    setTimeout( function () {
+        $data_table.ajax.reload();
+    }, 3000);
+});
+</script>
+{% endblock %}

apps/assets/templates/assets/domain_gateway_list.html

@@ -0,0 +1,131 @@
+{% extends 'base.html' %}
+{% load static %}
+{% load i18n %}
+
+{% block custom_head_css_js %}
+    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
+    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
+{% endblock %}
+
+{% block content %}
+    <div class="wrapper wrapper-content animated fadeInRight">
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="ibox float-e-margins">
+                    <div class="panel-options">
+                        <ul class="nav nav-tabs">
+                            <li>
+                                <a href="{% url 'assets:domain-detail' pk=object.id %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Detail' %} </a>
+                            </li>
+                            <li class="active">
+                                <a href="{% url 'assets:domain-detail' pk=object.id %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Gateway' %} </a>
+                            </li>
+                        </ul>
+                    </div>
+                    <div class="tab-content">
+                        <div class="col-sm-12" style="padding-left: 0;">
+                            <div class="" id="content_start">
+                            </div>
+                            <div class="ibox float-e-margins">
+                                <div class="ibox-title">
+                                    <span style="float: left"><b>{% trans 'Gateway list' %}</b></span>
+                                    <div class="ibox-tools">
+                                        <a class="collapse-link">
+                                            <i class="fa fa-chevron-up"></i>
+                                        </a>
+                                        <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+                                            <i class="fa fa-wrench"></i>
+                                        </a>
+                                        <ul class="dropdown-menu dropdown-user">
+                                        </ul>
+                                        <a class="close-link">
+                                            <i class="fa fa-times"></i>
+                                        </a>
+                                    </div>
+                                </div>
+                                <div class="ibox-content">
+                                    <div class="uc pull-left  m-r-5">
+                                        <a href="{% url 'assets:domain-gateway-create' pk=object.id %}" class="btn btn-sm btn-primary"> {% trans "Create gateway" %} </a>
+                                    </div>
+                                    <table class="table table-striped table-bordered table-hover " id="domain_list_table" >
+                                        <thead>
+                                        <tr>
+                                            <th class="text-center">
+                                                <input type="checkbox" id="check_all" class="ipt_check_all" >
+                                            </th>
+                                            <th class="text-center">{% trans 'Name' %}</th>
+                                            <th class="text-center">{% trans 'IP' %}</th>
+                                            <th class="text-center">{% trans 'Port' %}</th>
+                                            <th class="text-center">{% trans 'Protocol' %}</th>
+                                            <th class="text-center">{% trans 'Username' %}</th>
+                                            <th class="text-center">{% trans 'Comment' %}</th>
+                                            <th class="text-center">{% trans 'Action' %}</th>
+                                        </tr>
+                                        </thead>
+                                        <tbody>
+                                        </tbody>
+                                    </table>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}
+{% block content_bottom_left %}{% endblock %}
+{% block custom_foot_js %}
+<script>
+function initTable() {
+    var options = {
+        ele: $('#domain_list_table'),
+        columnDefs: [
+            {targets: 1, createdCell: function (td, cellData, rowData) {
+                var detail_btn = '<a href="{% url "assets:domain-detail" pk=DEFAULT_PK %}">' + cellData + '</a>';
+                $(td).html(detail_btn.replace('{{ DEFAULT_PK }}', rowData.id));
+             }},
+
+            {targets: 7, createdCell: function (td, cellData, rowData) {
+                var update_btn = '<a href="{% url "assets:domain-gateway-update" pk=DEFAULT_PK %}" class="btn btn-xs btn-info">{% trans "Update" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                var del_btn = '<a class="btn btn-xs btn-danger m-l-xs btn-delete" data-uid="{{ DEFAULT_PK }}">{% trans "Delete" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                var test_btn = '<a class="btn btn-xs btn-warning m-l-xs btn-test" data-uid="{{ DEFAULT_PK }}">{% trans "Test connection" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                $(td).html(update_btn + test_btn + del_btn)
+             }}
+        ],
+        ajax_url: '{% url "api-assets:gateway-list" %}?domain={{ object.id }}',
+        columns: [
+            {data: "id"}, {data: "name" }, {data: 'ip'}, {data: 'port'},
+            {data: "protocol"}, {data: "username" }, {data: "comment" }, {data: "id"}
+        ],
+        op_html: $('#actions').html()
+    };
+    jumpserver.initDataTable(options);
+}
+$(document).ready(function(){
+    initTable();
+})
+.on('click', '.btn-delete', function () {
+    var $this = $(this);
+    var $data_table = $('#domain_list_table').DataTable();
+    var name = $(this).closest("tr").find(":nth-child(2)").children('a').html();
+    var uid = $this.data('uid');
+    var the_url = '{% url "api-assets:gateway-detail" pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', uid);
+    objectDelete($this, name, the_url);
+    setTimeout( function () {
+        $data_table.ajax.reload();
+    }, 3000);
+}).on('click', '.btn-test', function () {
+    var $this = $(this);
+    var uid = $this.data('uid');
+    var the_url = '{% url "api-assets:test-gateway-connective" pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', uid);
+    APIUpdateAttr({
+        url: the_url,
+        method: "GET",
+        success_message: "可连接",
+        fail_message: "连接失败"
+    })
+
+})
+</script>
+{% endblock %}

apps/assets/templates/assets/domain_list.html

@@ -0,0 +1,74 @@
+{% extends '_base_list.html' %}
+{% load i18n static %}
+{% block table_search %}{% endblock %}
+{% block table_container %}
+<div class="uc pull-left  m-r-5">
+    <a href="{% url 'assets:domain-create' %}" class="btn btn-sm btn-primary"> {% trans "Create domain" %} </a>
+</div>
+<table class="table table-striped table-bordered table-hover " id="domain_list_table" >
+    <thead>
+    <tr>
+        <th class="text-center">
+            <input type="checkbox" id="check_all" class="ipt_check_all" >
+        </th>
+        <th class="text-center">{% trans 'Name' %}</th>
+        <th class="text-center">{% trans 'Asset' %}</th>
+        <th class="text-center">{% trans 'Gateway' %}</th>
+        <th class="text-center">{% trans 'Comment' %}</th>
+        <th class="text-center">{% trans 'Action' %}</th>
+    </tr>
+    </thead>
+    <tbody>
+    </tbody>
+</table>
+{% endblock %}
+{% block content_bottom_left %}{% endblock %}
+{% block custom_foot_js %}
+<script>
+function initTable() {
+    var options = {
+        ele: $('#domain_list_table'),
+        columnDefs: [
+            {targets: 1, createdCell: function (td, cellData, rowData) {
+                var detail_btn = '<a href="{% url "assets:domain-detail" pk=DEFAULT_PK %}">' + cellData + '</a>';
+                $(td).html(detail_btn.replace('{{ DEFAULT_PK }}', rowData.id));
+             }},
+            {targets: 3, createdCell: function (td, cellData, rowData) {
+                var gateway_list_btn = '<a href="{% url "assets:domain-gateway-list" pk=DEFAULT_PK %}">' + cellData + '</a>';
+                gateway_list_btn = gateway_list_btn.replace("{{ DEFAULT_PK }}", rowData.id);
+                $(td).html(gateway_list_btn);
+             }},
+            {targets: 5, createdCell: function (td, cellData, rowData) {
+                var update_btn = '<a href="{% url "assets:domain-update" pk=DEFAULT_PK %}" class="btn btn-xs btn-info">{% trans "Update" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                var del_btn = '<a class="btn btn-xs btn-danger m-l-xs btn-delete" data-uid="{{ DEFAULT_PK }}">{% trans "Delete" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
+                $(td).html(update_btn + del_btn)
+             }}
+        ],
+        ajax_url: '{% url "api-assets:domain-list" %}',
+        columns: [
+            {data: "id"}, {data: "name" }, {data: "asset_count" },
+            {data: "gateway_count" }, {data: "comment" }, {data: "id"}
+        ],
+        op_html: $('#actions').html()
+    };
+    jumpserver.initDataTable(options);
+}
+$(document).ready(function(){
+    initTable();
+})
+.on('click', '.btn-delete', function () {
+    var $this = $(this);
+    var $data_table = $('#domain_list_table').DataTable();
+    var name = $(this).closest("tr").find(":nth-child(2)").children('a').html();
+    var uid = $this.data('uid');
+    var the_url = '{% url "api-assets:domain-detail" pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', uid);
+    objectDelete($this, name, the_url);
+    setTimeout( function () {
+        $data_table.ajax.reload();
+    }, 3000);
+});
+</script>
+{% endblock %}
+
+
+

apps/assets/templates/assets/gateway_create_update.html

@@ -0,0 +1,68 @@
+{% extends 'base.html' %}
+{% load i18n %}
+{% load static %}
+{% load bootstrap3 %}
+{% block custom_head_css_js %}
+    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
+    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
+{% endblock %}
+
+{% block content %}
+    <div class="wrapper wrapper-content animated fadeInRight">
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="ibox float-e-margins">
+                    <div class="ibox-title">
+                        <h5>{{ action }}</h5>
+                        <div class="ibox-tools">
+                            <a class="collapse-link">
+                                <i class="fa fa-chevron-up"></i>
+                            </a>
+                            <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+                                <i class="fa fa-wrench"></i>
+                            </a>
+                            <a class="close-link">
+                                <i class="fa fa-times"></i>
+                            </a>
+                        </div>
+                    </div>
+                    <div class="ibox-content">
+                        <form enctype="multipart/form-data" method="post" class="form-horizontal" action="" >
+                            {% csrf_token %}
+                            {% if form.non_field_errors %}
+                                <div class="alert alert-danger">
+                                    {{ form.non_field_errors }}
+                                </div>
+                            {% endif %}
+                            <h3>{% trans 'Basic' %}</h3>
+                            {% bootstrap_field form.name layout="horizontal" %}
+                            {% bootstrap_field form.ip layout="horizontal" %}
+                            {% bootstrap_field form.port layout="horizontal" %}
+                            {% bootstrap_field form.protocol layout="horizontal" %}
+                            {% bootstrap_field form.domain layout="horizontal" %}
+
+                            {% block auth %}
+                            <h3>{% trans 'Auth' %}</h3>
+                            <div class="auth-fields">
+                                {% bootstrap_field form.username layout="horizontal" %}
+                                {% bootstrap_field form.password layout="horizontal" %}
+                                {% bootstrap_field form.private_key_file layout="horizontal" %}
+                            </div>
+                            {% endblock %}
+
+                            <h3>{% trans 'Other' %}</h3>
+                            {% bootstrap_field form.is_active layout="horizontal" %}
+                            {% bootstrap_field form.comment layout="horizontal" %}
+                            <div class="form-group">
+                                <div class="col-sm-4 col-sm-offset-2">
+                                    <button class="btn btn-white" type="reset">{% trans 'Reset' %}</button>
+                                    <button id="submit_button" class="btn btn-primary" type="submit">{% trans 'Submit' %}</button>
+                                </div>
+                            </div>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}

apps/assets/templates/assets/system_user_detail.html

@@ -206,7 +206,7 @@
 {% endblock %}
 {% block custom_foot_js %}
 <script>
-function updateSystemUserCluster(nodes) {
+function updateSystemUserNode(nodes) {
     var the_url = "{% url 'api-assets:system-user-detail' pk=system_user.id %}";
     var body = {
         nodes: Object.assign([], nodes)
@@ -267,7 +267,7 @@ $(document).ready(function () {
     $.map(jumpserver.nodes_selected, function(value, index) {
         nodes.push(index);
     });
-    updateSystemUserCluster(nodes);
+    updateSystemUserNode(nodes);
 })
 .on('click', '.btn-remove-from-node', function() {
     var $this = $(this);
@@ -282,7 +282,7 @@ $(document).ready(function () {
     var nodes = $('.bdg_node').map(function () {
         return $(this).data('gid');
     }).get();
-    updateSystemUserCluster(nodes);
+    updateSystemUserNode(nodes);
 }).on('click', '.btn-del', function () {
     var $this = $(this);
     var name = "{{ system_user.name}}";
@@ -293,26 +293,30 @@ $(document).ready(function () {
 })
 .on('click', '.btn-push', function () {
     var the_url = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
-    var error = function (data) {
+    var success = function (data) {
-        alert(data)
+        var task_id = data.task;
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     };
     APIUpdateAttr({
         url: the_url,
-        error: error,
         method: 'GET',
-        success_message: "{% trans "Task has been send, Go to ops task list seen result" %}"
+        success: success,
+        flash_message: false
     });
 })
 .on('click', '.btn-test-connective', function () {
     var the_url = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
-    var error = function (data) {
+    var success = function (data) {
-        alert(data)
+        var task_id = data.task;
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     };
     APIUpdateAttr({
         url: the_url,
-        error: error,
         method: 'GET',
-        success_message: "{% trans "Task has been send, seen left assets status" %}"
+        success: success,
+        flash_message: false
     });
 })
 </script>

apps/assets/templates/assets/system_user_list.html

@@ -25,6 +25,7 @@
         </th>
         <th class="text-center">{% trans 'Name' %}</th>
         <th class="text-center">{% trans 'Username' %}</th>
+        <th class="text-center">{% trans 'Protocol' %}</th>
         <th class="text-center">{% trans 'Asset' %}</th>
         <th class="text-center">{% trans 'Reachable' %}</th>
         <th class="text-center">{% trans 'Unreachable' %}</th>
@@ -47,7 +48,7 @@ function initTable() {
                 var detail_btn = '<a href="{% url "assets:system-user-detail" pk=DEFAULT_PK %}">' + cellData + '</a>';
                 $(td).html(detail_btn.replace('{{ DEFAULT_PK }}', rowData.id));
             }},
-            {targets: 4, createdCell: function (td, cellData) {
+            {targets: 5, createdCell: function (td, cellData) {
                 var innerHtml = "";
                 if (cellData !== 0) {
                     innerHtml = "<span class='text-navy'>" + cellData + "</span>";
@@ -56,7 +57,7 @@ function initTable() {
                 }
                 $(td).html('<span href="javascript:void(0);" data-toggle="tooltip" title="' + cellData +'">' + innerHtml + '</span>');
             }},
-            {targets: 5, createdCell: function (td, cellData) {
+            {targets: 6, createdCell: function (td, cellData) {
                 var innerHtml = "";
                 if (cellData !== 0) {
                     innerHtml = "<span class='text-danger'>" + cellData + "</span>";
@@ -65,7 +66,7 @@ function initTable() {
                 }
                 $(td).html('<span href="javascript:void(0);" data-toggle="tooltip" title="' + cellData + '">' + innerHtml + '</span>');
             }},
-            {targets: 6, createdCell: function (td, cellData, rowData) {
+            {targets: 7, createdCell: function (td, cellData, rowData) {
                 var val = 0;
                 var innerHtml = "";
                 var total = rowData.assets_amount;
@@ -83,14 +84,14 @@ function initTable() {
                 $(td).html('<span href="javascript:void(0);" data-toggle="tooltip" title="' + cellData + '">' + innerHtml + '</span>');
 
             }},
-            {targets: 8, createdCell: function (td, cellData, rowData) {
+            {targets: 9, createdCell: function (td, cellData, rowData) {
                 var update_btn = '<a href="{% url "assets:system-user-update" pk=DEFAULT_PK %}" class="btn btn-xs m-l-xs btn-info">{% trans "Update" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
                 var del_btn = '<a class="btn btn-xs btn-danger m-l-xs btn_admin_user_delete" data-uid="{{ DEFAULT_PK }}">{% trans "Delete" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
                 $(td).html(update_btn + del_btn)
             }}],
         ajax_url: '{% url "api-assets:system-user-list" %}',
         columns: [
-            {data: "id" }, {data: "name" }, {data: "username" }, {data: "assets_amount" },
+            {data: "id" }, {data: "name" }, {data: "username" }, {data: "protocol"}, {data: "assets_amount" },
             {data: "reachable_amount"}, {data: "unreachable_amount"}, {data: "id"}, {data: "comment" }, {data: "id" }
         ],
         op_html: $('#actions').html()

apps/assets/urls/api_urls.py

@@ -7,13 +7,13 @@ app_name = 'assets'
 
 
 router = BulkRouter()
-# router.register(r'v1/groups', api.AssetGroupViewSet, 'asset-group')
 router.register(r'v1/assets', api.AssetViewSet, 'asset')
-# router.register(r'v1/clusters', api.ClusterViewSet, 'cluster')
 router.register(r'v1/admin-user', api.AdminUserViewSet, 'admin-user')
 router.register(r'v1/system-user', api.SystemUserViewSet, 'system-user')
 router.register(r'v1/labels', api.LabelViewSet, 'label')
 router.register(r'v1/nodes', api.NodeViewSet, 'node')
+router.register(r'v1/domain', api.DomainViewSet, 'domain')
+router.register(r'v1/gateway', api.GatewayViewSet, 'gateway')
 
 urlpatterns = [
     url(r'^v1/assets-bulk/$', api.AssetListUpdateApi.as_view(), name='asset-bulk-update'),
@@ -25,18 +25,10 @@ urlpatterns = [
         api.AssetAdminUserTestApi.as_view(), name='asset-alive-test'),
     url(r'^v1/assets/user-assets/$',
         api.UserAssetListView.as_view(), name='user-asset-list'),
-    # update the asset group, which add or delete the asset to the group
-    #url(r'^v1/groups/(?P<pk>[0-9a-zA-Z\-]{36})/assets/$',
-    #    api.GroupUpdateAssetsApi.as_view(), name='group-update-assets'),
-    #url(r'^v1/groups/(?P<pk>[0-9a-zA-Z\-]{36})/assets/add/$',
-    #    api.GroupAddAssetsApi.as_view(), name='group-add-assets'),
-    # update the Cluster, and add or delete the assets to the Cluster
-    #url(r'^v1/cluster/(?P<pk>[0-9a-zA-Z\-]{36})/assets/$',
-    #    api.ClusterAddAssetsApi.as_view(), name='cluster-add-assets'),
-    #url(r'^v1/cluster/(?P<pk>[0-9a-zA-Z\-]{36})/assets/connective/$',
-    #    api.ClusterTestAssetsAliveApi.as_view(), name='cluster-test-connective'),
     url(r'^v1/admin-user/(?P<pk>[0-9a-zA-Z\-]{36})/nodes/$',
         api.ReplaceNodesAdminUserApi.as_view(), name='replace-nodes-admin-user'),
+    url(r'^v1/admin-user/(?P<pk>[0-9a-zA-Z\-]{36})/auth/$',
+        api.AdminUserAuthApi.as_view(), name='admin-user-auth'),
     url(r'^v1/admin-user/(?P<pk>[0-9a-zA-Z\-]{36})/connective/$',
         api.AdminUserTestConnectiveApi.as_view(), name='admin-user-connective'),
     url(r'^v1/system-user/(?P<pk>[0-9a-zA-Z\-]{36})/push/$',
@@ -49,6 +41,8 @@ urlpatterns = [
     url(r'^v1/nodes/(?P<pk>[0-9a-zA-Z\-]{36})/assets/remove/$', api.NodeRemoveAssetsApi.as_view(), name='node-remove-assets'),
     url(r'^v1/nodes/(?P<pk>[0-9a-zA-Z\-]{36})/refresh-hardware-info/$', api.RefreshNodeHardwareInfoApi.as_view(), name='node-refresh-hardware-info'),
     url(r'^v1/nodes/(?P<pk>[0-9a-zA-Z\-]{36})/test-connective/$', api.TestNodeConnectiveApi.as_view(), name='node-test-connective'),
+
+    url(r'^v1/gateway/(?P<pk>[0-9a-zA-Z\-]{36})/test-connective/$', api.GatewayTestConnectionApi.as_view(), name='test-gateway-connective'),
 ]
 
 urlpatterns += router.urls

apps/assets/urls/views_urls.py

@@ -39,5 +39,15 @@ urlpatterns = [
     url(r'^label/create/$', views.LabelCreateView.as_view(), name='label-create'),
     url(r'^label/(?P<pk>[0-9a-zA-Z\-]{36})/update/$', views.LabelUpdateView.as_view(), name='label-update'),
     url(r'^label/(?P<pk>[0-9a-zA-Z\-]{36})/delete/$', views.LabelDeleteView.as_view(), name='label-delete'),
+
+    url(r'^domain/$', views.DomainListView.as_view(), name='domain-list'),
+    url(r'^domain/create/$', views.DomainCreateView.as_view(), name='domain-create'),
+    url(r'^domain/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.DomainDetailView.as_view(), name='domain-detail'),
+    url(r'^domain/(?P<pk>[0-9a-zA-Z\-]{36})/update/$', views.DomainUpdateView.as_view(), name='domain-update'),
+    url(r'^domain/(?P<pk>[0-9a-zA-Z\-]{36})/delete/$', views.DomainDeleteView.as_view(), name='domain-delete'),
+    url(r'^domain/(?P<pk>[0-9a-zA-Z\-]{36})/gateway/$', views.DomainGatewayListView.as_view(), name='domain-gateway-list'),
+
+    url(r'^domain/(?P<pk>[0-9a-zA-Z\-]{36})/gateway/create/$', views.DomainGatewayCreateView.as_view(), name='domain-gateway-create'),
+    url(r'^domain/gateway/(?P<pk>[0-9a-zA-Z\-]{36})/update/$', views.DomainGatewayUpdateView.as_view(), name='domain-gateway-update'),
 ]
 

apps/assets/utils.py

@@ -1,10 +1,7 @@
 # ~*~ coding: utf-8 ~*~
 #
-from collections import defaultdict
-from functools import reduce
-import operator
 
-from django.db.models import Q
+import paramiko
 
 from common.utils import get_object_or_none
 from .models import Asset, SystemUser, Label
@@ -44,5 +41,41 @@ class LabelFilter:
         return queryset
 
 
+def test_gateway_connectability(gateway):
+    """
+    Test system cant connect his assets or not.
+    :param gateway:
+    :return:
+    """
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
+    proxy_command = [
+        "ssh", "{}@{}".format(gateway.username, gateway.ip),
+        "-p", str(gateway.port), "-W", "127.0.0.1:{}".format(gateway.port),
+    ]
 
+    if gateway.password:
+        proxy_command.insert(0, "sshpass -p '{}'".format(gateway.password))
+    if gateway.private_key:
+        proxy_command.append("-i {}".format(gateway.private_key_file))
+
+    try:
+        sock = paramiko.ProxyCommand(" ".join(proxy_command))
+    except paramiko.ProxyCommandFailure as e:
+        return False, str(e)
+
+    try:
+        client.connect("127.0.0.1", port=gateway.port,
+                       username=gateway.username,
+                       password=gateway.password,
+                       key_filename=gateway.private_key_file,
+                       sock=sock,
+                       timeout=5
+                       )
+    except (paramiko.SSHException, paramiko.ssh_exception.SSHException,
+            paramiko.AuthenticationException, TimeoutError) as e:
+        return False, str(e)
+    finally:
+        client.close()
+    return True, None

apps/assets/views/__init__.py

@@ -3,3 +3,4 @@ from .asset import *
 from .system_user import *
 from .admin_user import *
 from .label import *
+from .domain import *

apps/assets/views/asset.py

@@ -9,6 +9,7 @@ import chardet
 from io import StringIO
 
 from django.conf import settings
+from django.db import transaction
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import TemplateView, ListView, View
 from django.views.generic.edit import CreateView, DeleteView, FormView, UpdateView
@@ -27,7 +28,7 @@ from common.mixins import JSONResponseMixin
 from common.utils import get_object_or_none, get_logger, is_uuid
 from common.const import create_success_msg, update_success_msg
 from .. import forms
-from ..models import Asset, AssetGroup, AdminUser, Cluster, SystemUser, Label, Node
+from ..models import Asset, AdminUser, SystemUser, Label, Node, Domain
 from ..hands import AdminUserRequiredMixin
 
 
@@ -244,6 +245,8 @@ class BulkImportAssetView(AdminUserRequiredMixin, JSONResponseMixin, FormView):
     form_class = forms.FileForm
 
     def form_valid(self, form):
+        node_id = self.request.GET.get("node_id")
+        node = get_object_or_none(Node, id=node_id) if node_id else Node.root()
         f = form.cleaned_data['file']
         det_result = chardet.detect(f.read())
         f.seek(0)  # reset file seek index
@@ -276,6 +279,7 @@ class BulkImportAssetView(AdminUserRequiredMixin, JSONResponseMixin, FormView):
             asset_dict = dict(zip(attr, row))
             id_ = asset_dict.pop('id', 0)
             for k, v in asset_dict.items():
+                v = v.strip()
                 if k == 'is_active':
                     v = True if v in ['TRUE', 1, 'true'] else False
                 elif k == 'admin_user':
@@ -285,8 +289,8 @@ class BulkImportAssetView(AdminUserRequiredMixin, JSONResponseMixin, FormView):
                         v = int(v)
                     except ValueError:
                         v = 0
-                else:
+                elif k == 'domain':
-                    continue
+                    v = get_object_or_none(Domain, name=v)
                 asset_dict[k] = v
 
             asset = get_object_or_none(Asset, id=id_) if is_uuid(id_) else None
@@ -294,7 +298,10 @@ class BulkImportAssetView(AdminUserRequiredMixin, JSONResponseMixin, FormView):
                 try:
                     if len(Asset.objects.filter(hostname=asset_dict.get('hostname'))):
                         raise Exception(_('already exists'))
+                    with transaction.atomic():
                         asset = Asset.objects.create(**asset_dict)
+                        if node:
+                            asset.nodes.set([node])
                         created.append(asset_dict['hostname'])
                         assets.append(asset)
                 except Exception as e:

apps/assets/views/domain.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+#
+
+from django.views.generic import TemplateView, CreateView, \
+    UpdateView, DeleteView, DetailView
+from django.views.generic.detail import SingleObjectMixin
+from django.utils.translation import ugettext_lazy as _
+from django.urls import reverse_lazy, reverse
+
+from common.mixins import AdminUserRequiredMixin
+from common.const import create_success_msg, update_success_msg
+from common.utils import get_object_or_none
+from ..models import Domain, Gateway
+from ..forms import DomainForm, GatewayForm
+
+
+__all__ = (
+    "DomainListView", "DomainCreateView", "DomainUpdateView",
+    "DomainDetailView", "DomainDeleteView", "DomainGatewayListView",
+    "DomainGatewayCreateView", 'DomainGatewayUpdateView',
+)
+
+
+class DomainListView(AdminUserRequiredMixin, TemplateView):
+    template_name = 'assets/domain_list.html'
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Domain list'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class DomainCreateView(AdminUserRequiredMixin, CreateView):
+    model = Domain
+    template_name = 'assets/domain_create_update.html'
+    form_class = DomainForm
+    success_url = reverse_lazy('assets:domain-list')
+    success_message = create_success_msg
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Create domain'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class DomainUpdateView(AdminUserRequiredMixin, UpdateView):
+    model = Domain
+    template_name = 'assets/domain_create_update.html'
+    form_class = DomainForm
+    success_url = reverse_lazy('assets:domain-list')
+    success_message = update_success_msg
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Update domain'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class DomainDetailView(AdminUserRequiredMixin, DetailView):
+    model = Domain
+    template_name = 'assets/domain_detail.html'
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Domain detail'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class DomainDeleteView(AdminUserRequiredMixin, DeleteView):
+    model = Domain
+    template_name = 'delete_confirm.html'
+    success_url = reverse_lazy('assets:domain-list')
+
+
+class DomainGatewayListView(AdminUserRequiredMixin, SingleObjectMixin, TemplateView):
+    template_name = 'assets/domain_gateway_list.html'
+    model = Domain
+    object = None
+
+    def get(self, request, *args, **kwargs):
+        self.object = self.get_object(queryset=self.model.objects.all())
+        return super().get(request, *args, **kwargs)
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Domain gateway list'),
+            'object': self.get_object()
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class DomainGatewayCreateView(AdminUserRequiredMixin, CreateView):
+    model = Gateway
+    template_name = 'assets/gateway_create_update.html'
+    form_class = GatewayForm
+    success_message = create_success_msg
+
+    def get_success_url(self):
+        domain = self.object.domain
+        return reverse('assets:domain-gateway-list', kwargs={"pk": domain.id})
+
+    def get_form(self, form_class=None):
+        form = super().get_form(form_class=form_class)
+        domain_id = self.kwargs.get("pk")
+        domain = get_object_or_none(Domain, id=domain_id)
+        if domain:
+            form['domain'].initial = domain
+        return form
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Create gateway'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class DomainGatewayUpdateView(AdminUserRequiredMixin, UpdateView):
+    model = Gateway
+    template_name = 'assets/gateway_create_update.html'
+    form_class = GatewayForm
+    success_message = update_success_msg
+
+    def get_success_url(self):
+        domain = self.object.domain
+        return reverse('assets:domain-gateway-list', kwargs={"pk": domain.id})
+
+    def form_valid(self, form):
+        response = super().form_valid(form)
+        print(form.cleaned_data)
+        return response
+
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Assets'),
+            'action': _('Update gateway'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)

apps/common/__init__.py

@@ -2,4 +2,4 @@ from __future__ import absolute_import
 
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
-from .celery import app as celery_app
+

apps/common/api.py

@@ -2,14 +2,13 @@
 #
 import json
 
-from rest_framework.views import APIView
+from rest_framework.views import Response, APIView
-from rest_framework.views import Response
 from ldap3 import Server, Connection
 from django.core.mail import get_connection, send_mail
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 
-from .permissions import IsSuperUser, IsAppUser
+from .permissions import IsSuperUser
 from .serializers import MailTestSerializer, LDAPTestSerializer
 
 
@@ -105,3 +104,6 @@ class DjangoSettingsAPI(APIView):
             if i.isupper():
                 configs[i] = str(getattr(settings, i))
         return Response(configs)
+
+
+

apps/common/const.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 
-from django.utils.translation import ugettext as _
+from django.utils.translation import ugettext_lazy as _
 
 create_success_msg = _("<b>%(name)s</b> was created successfully")
 update_success_msg = _("<b>%(name)s</b> was updated successfully")
+FILE_END_GUARD = ">>> Content End <<<"
+celery_task_pre_key = "CELERY_"

apps/common/models.py

@@ -79,3 +79,4 @@ class Setting(models.Model):
 
     class Meta:
         db_table = "settings"
+

apps/common/tasks.py

@@ -1,13 +1,13 @@
 from django.core.mail import send_mail
 from django.conf import settings
-from .celery import app
+from celery import shared_task
 from .utils import get_logger
 
 
 logger = get_logger(__file__)
 
 
-@app.task
+@shared_task
 def send_mail_async(*args, **kwargs):
     """ Using celery to send email async
 

apps/common/utils.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 import re
+import sys
 from collections import OrderedDict
 from six import string_types
 import base64
@@ -359,3 +360,21 @@ def is_uuid(s):
 def get_signer():
     signer = Signer(settings.SECRET_KEY)
     return signer
+
+
+class TeeObj:
+    origin_stdout = sys.stdout
+
+    def __init__(self, file_obj):
+        self.file_obj = file_obj
+
+    def write(self, msg):
+        self.origin_stdout.write(msg)
+        self.file_obj.write(msg.replace('*', ''))
+
+    def flush(self):
+        self.origin_stdout.flush()
+        self.file_obj.flush()
+
+    def close(self):
+        self.file_obj.close()

apps/common/views.py

@@ -1,12 +1,13 @@
-from django.views.generic import TemplateView
+
-from django.shortcuts import render, redirect
+from django.core.cache import cache
+from django.views.generic import TemplateView, View, DetailView
+from django.shortcuts import render, redirect, Http404, reverse
 from django.contrib import messages
 from django.utils.translation import ugettext as _
 from django.conf import settings
 
 from .forms import EmailSettingForm, LDAPSettingForm, BasicSettingForm, \
     TerminalSettingForm
-from .models import Setting
 from .mixins import AdminUserRequiredMixin
 from .signals import ldap_auth_enable
 
@@ -120,3 +121,4 @@ class TerminalSettingView(AdminUserRequiredMixin, TemplateView):
             context.update({"form": form})
             return render(request, self.template_name, context)
 
+

apps/jumpserver/settings.py

@@ -234,7 +234,7 @@ LOGGING = {
 
 # Internationalization
 # https://docs.djangoproject.com/en/1.10/topics/i18n/
-LANGUAGE_CODE = 'zh-cn'
+LANGUAGE_CODE = 'en'
 
 TIME_ZONE = 'Asia/Shanghai'
 

apps/ops/__init__.py

@@ -1 +1 @@
-
+from .celery import app as celery_app

apps/ops/ansible/callback.py

@@ -1,14 +1,18 @@
 # ~*~ coding: utf-8 ~*~
 
+import sys
+
 from ansible.plugins.callback import CallbackBase
 from ansible.plugins.callback.default import CallbackModule
 
+from .display import TeeObj
+
 
 class AdHocResultCallback(CallbackModule):
     """
     Task result Callback
     """
-    def __init__(self, display=None, options=None):
+    def __init__(self, display=None, options=None, file_obj=None):
         # result_raw example: {
         #   "ok": {"hostname": {"task_name": {}，...},..},
         #   "failed": {"hostname": {"task_name": {}..}, ..},
@@ -22,6 +26,8 @@ class AdHocResultCallback(CallbackModule):
         self.results_raw = dict(ok={}, failed={}, unreachable={}, skipped={})
         self.results_summary = dict(contacted=[], dark={})
         super().__init__()
+        if file_obj is not None:
+            sys.stdout = TeeObj(file_obj)
 
     def gather_result(self, t, res):
         self._clean_results(res._result, res._task.action)

apps/ops/ansible/display.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+#
+
+import sys
+
+
+class TeeObj:
+    origin_stdout = sys.stdout
+
+    def __init__(self, file_obj):
+        self.file_obj = file_obj
+
+    def write(self, msg):
+        self.origin_stdout.write(msg)
+        self.file_obj.write(msg.replace('*', ''))
+
+    def flush(self):
+        self.origin_stdout.flush()
+        self.file_obj.flush()

apps/ops/ansible/inventory.py

@@ -78,7 +78,7 @@ class BaseInventory(InventoryManager):
     variable_manager_class = VariableManager
     host_manager_class = BaseHost
 
-    def __init__(self, host_list=None):
+    def __init__(self, host_list=None, group_list=None):
         """
         用于生成动态构建Ansible Inventory. super().__init__ 会自动调用
         host_list: [{
@@ -97,11 +97,14 @@ class BaseInventory(InventoryManager):
             "vars": {},
           },
         ]
+        group_list: [
+          {"name: "", children: [""]},
+        ]
         :param host_list:
+        :param group_list
         """
-        if host_list is None:
+        self.host_list = host_list or []
-            host_list = []
+        self.group_list = group_list or []
-        self.host_list = host_list
         assert isinstance(host_list, list)
         self.loader = self.loader_class()
         self.variable_manager = self.variable_manager_class()
@@ -113,25 +116,40 @@ class BaseInventory(InventoryManager):
     def get_group(self, name):
         return self._inventory.groups.get(name, None)
 
-    def parse_sources(self, cache=False):
+    def get_or_create_group(self, name):
-        group_all = self.get_group('all')
+        group = self.get_group(name)
-        ungrouped = self.get_group('ungrouped')
+        if not group:
+            self.add_group(name)
+            return self.get_or_create_group(name)
+        else:
+            return group
 
+    def parse_groups(self):
+        for g in self.group_list:
+            parent = self.get_or_create_group(g.get("name"))
+            children = [self.get_or_create_group(n) for n in g.get('children', [])]
+            for child in children:
+                parent.add_child_group(child)
+
+    def parse_hosts(self):
+        group_all = self.get_or_create_group('all')
+        ungrouped = self.get_or_create_group('ungrouped')
         for host_data in self.host_list:
             host = self.host_manager_class(host_data=host_data)
             self.hosts[host_data['hostname']] = host
             groups_data = host_data.get('groups')
             if groups_data:
                 for group_name in groups_data:
-                    group = self.get_group(group_name)
+                    group = self.get_or_create_group(group_name)
-                    if group is None:
-                        self.add_group(group_name)
-                        group = self.get_group(group_name)
                     group.add_host(host)
             else:
                 ungrouped.add_host(host)
             group_all.add_host(host)
 
+    def parse_sources(self, cache=False):
+        self.parse_groups()
+        self.parse_hosts()
+
     def get_matched_hosts(self, pattern):
         return self.get_hosts(pattern)
 

apps/ops/ansible/runner.py

@@ -9,6 +9,7 @@ from ansible.parsing.dataloader import DataLoader
 from ansible.executor.playbook_executor import PlaybookExecutor
 from ansible.playbook.play import Play
 import ansible.constants as C
+from ansible.utils.display import Display
 
 from .callback import AdHocResultCallback, PlaybookResultCallBack, \
     CommandResultCallback
@@ -21,6 +22,13 @@ C.HOST_KEY_CHECKING = False
 logger = get_logger(__name__)
 
 
+class CustomDisplay(Display):
+    def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False):
+        pass
+
+display = CustomDisplay()
+
+
 Options = namedtuple('Options', [
     'listtags', 'listtasks', 'listhosts', 'syntax', 'connection',
     'module_path', 'forks', 'remote_user', 'private_key_file', 'timeout',
@@ -123,20 +131,22 @@ class AdHocRunner:
     ADHoc Runner接口
     """
     results_callback_class = AdHocResultCallback
+    results_callback = None
     loader_class = DataLoader
     variable_manager_class = VariableManager
-    options = get_default_options()
     default_options = get_default_options()
 
     def __init__(self, inventory, options=None):
-        if options:
+        self.options = self.update_options(options)
-            self.options = options
         self.inventory = inventory
         self.loader = DataLoader()
         self.variable_manager = VariableManager(
             loader=self.loader, inventory=self.inventory
         )
 
+    def get_result_callback(self, file_obj=None):
+        return self.__class__.results_callback_class(file_obj=file_obj)
+
     @staticmethod
     def check_module_args(module_name, module_args=''):
         if module_name in C.MODULE_REQUIRE_ARGS and not module_args:
@@ -160,19 +170,24 @@ class AdHocRunner:
             cleaned_tasks.append(task)
         return cleaned_tasks
 
-    def set_option(self, k, v):
+    def update_options(self, options):
-        kwargs = {k: v}
+        if options and isinstance(options, dict):
-        self.options = self.options._replace(**kwargs)
+            options = self.__class__.default_options._replace(**options)
+        else:
+            options = self.__class__.default_options
+        return options
 
-    def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no'):
+    def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no', file_obj=None):
         """
         :param tasks: [{'action': {'module': 'shell', 'args': 'ls'}, ...}, ]
         :param pattern: all, *, or others
         :param play_name: The play name
+        :param gather_facts:
+        :param file_obj: logging to file_obj
         :return:
         """
         self.check_pattern(pattern)
-        results_callback = self.results_callback_class()
+        self.results_callback = self.get_result_callback(file_obj)
         cleaned_tasks = self.clean_tasks(tasks)
 
         play_source = dict(
@@ -193,16 +208,16 @@ class AdHocRunner:
             variable_manager=self.variable_manager,
             loader=self.loader,
             options=self.options,
-            stdout_callback=results_callback,
+            stdout_callback=self.results_callback,
             passwords=self.options.passwords,
         )
-        logger.debug("Get inventory matched hosts: {}".format(
+        print("Get matched hosts: {}".format(
             self.inventory.get_matched_hosts(pattern)
         ))
 
         try:
             tqm.run(play)
-            return results_callback
+            return self.results_callback
         except Exception as e:
             raise AnsibleError(e)
         finally:

apps/ops/api.py

@@ -1,13 +1,17 @@
 # ~*~ coding: utf-8 ~*~
+import uuid
+import os
 
-
+from django.core.cache import cache
 from django.shortcuts import get_object_or_404
+from django.utils.translation import ugettext as _
 from rest_framework import viewsets, generics
 from rest_framework.views import Response
 
 from .hands import IsSuperUser
-from .models import Task, AdHoc, AdHocRunHistory
+from .models import Task, AdHoc, AdHocRunHistory, CeleryTask
-from .serializers import TaskSerializer, AdHocSerializer, AdHocRunHistorySerializer
+from .serializers import TaskSerializer, AdHocSerializer, \
+    AdHocRunHistorySerializer
 from .tasks import run_ansible_task
 
 
@@ -24,8 +28,8 @@ class TaskRun(generics.RetrieveAPIView):
 
     def retrieve(self, request, *args, **kwargs):
         task = self.get_object()
-        run_ansible_task.delay(str(task.id))
+        t = run_ansible_task.delay(str(task.id))
-        return Response({"msg": "start"})
+        return Response({"task": t.id})
 
 
 class AdHocViewSet(viewsets.ModelViewSet):
@@ -58,3 +62,30 @@ class AdHocRunHistorySet(viewsets.ModelViewSet):
             adhoc = get_object_or_404(AdHoc, id=adhoc_id)
             self.queryset = self.queryset.filter(adhoc=adhoc)
         return self.queryset
+
+
+class CeleryTaskLogApi(generics.RetrieveAPIView):
+    permission_classes = (IsSuperUser,)
+    buff_size = 1024 * 10
+    end = False
+    queryset = CeleryTask.objects.all()
+
+    def get(self, request, *args, **kwargs):
+        mark = request.query_params.get("mark") or str(uuid.uuid4())
+        task = super().get_object()
+        log_path = task.full_log_path
+
+        if not log_path or not os.path.isfile(log_path):
+            return Response({"data": _("Waiting ...")}, status=203)
+
+        with open(log_path, 'r') as f:
+            offset = cache.get(mark, 0)
+            f.seek(offset)
+            data = f.read(self.buff_size).replace('\n', '\r\n')
+            mark = str(uuid.uuid4())
+            cache.set(mark, f.tell(), 5)
+
+            if data == '' and task.is_finished():
+                self.end = True
+            return Response({"data": data, 'end': self.end, 'mark': mark})
+

apps/ops/apps.py

@@ -5,3 +5,7 @@ from django.apps import AppConfig
 
 class OpsConfig(AppConfig):
     name = 'ops'
+
+    def ready(self):
+        super().ready()
+        from .celery import signal_handler

apps/ops/celery/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'jumpserver.settings')
+
+from django.conf import settings
+
+app = Celery('jumpserver')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])

apps/ops/celery/const.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+#
+

apps/ops/celery/signal_handler.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+#
+import os
+import datetime
+import sys
+import time
+
+from django.conf import settings
+from django.utils import timezone
+from django.core.cache import cache
+from django.db import transaction
+from celery import subtask
+from celery.signals import worker_ready, worker_shutdown, task_prerun, \
+    task_postrun, after_task_publish
+from django_celery_beat.models import PeriodicTask
+
+from common.utils import get_logger, TeeObj, get_object_or_none
+from common.const import celery_task_pre_key
+from .utils import get_after_app_ready_tasks, get_after_app_shutdown_clean_tasks
+from ..models import CeleryTask
+
+logger = get_logger(__file__)
+
+
+@worker_ready.connect
+def on_app_ready(sender=None, headers=None, body=None, **kwargs):
+    if cache.get("CELERY_APP_READY", 0) == 1:
+        return
+    cache.set("CELERY_APP_READY", 1, 10)
+    logger.debug("App ready signal recv")
+    tasks = get_after_app_ready_tasks()
+    logger.debug("Start need start task: [{}]".format(
+        ", ".join(tasks))
+    )
+    for task in tasks:
+        subtask(task).delay()
+
+
+@worker_shutdown.connect
+def after_app_shutdown(sender=None, headers=None, body=None, **kwargs):
+    if cache.get("CELERY_APP_SHUTDOWN", 0) == 1:
+        return
+    cache.set("CELERY_APP_SHUTDOWN", 1, 10)
+    tasks = get_after_app_shutdown_clean_tasks()
+    logger.debug("App shutdown signal recv")
+    logger.debug("Clean need cleaned period tasks: [{}]".format(
+        ', '.join(tasks))
+    )
+    PeriodicTask.objects.filter(name__in=tasks).delete()
+
+
+@after_task_publish.connect
+def after_task_publish_signal_handler(sender, headers=None, **kwargs):
+    CeleryTask.objects.create(
+        id=headers["id"], status=CeleryTask.WAITING, name=headers["task"]
+    )
+    cache.set(headers["id"], True, 3600)
+
+
+@task_prerun.connect
+def pre_run_task_signal_handler(sender, task_id=None, task=None, **kwargs):
+    time.sleep(0.1)
+    for i in range(5):
+        if cache.get(task_id, False):
+            break
+        else:
+            time.sleep(0.1)
+            continue
+
+    t = get_object_or_none(CeleryTask, id=task_id)
+    if t is None:
+        logger.warn("Not get the task: {}".format(task_id))
+        return
+    now = datetime.datetime.now().strftime("%Y-%m-%d")
+    log_path = os.path.join(now, task_id + '.log')
+    full_path = os.path.join(CeleryTask.LOG_DIR, log_path)
+
+    if not os.path.exists(os.path.dirname(full_path)):
+        os.makedirs(os.path.dirname(full_path))
+    with transaction.atomic():
+        t.date_start = timezone.now()
+        t.status = CeleryTask.RUNNING
+        t.log_path = log_path
+        t.save()
+    f = open(full_path, 'w')
+    tee = TeeObj(f)
+    sys.stdout = tee
+    task.log_f = tee
+
+
+@task_postrun.connect
+def post_run_task_signal_handler(sender, task_id=None, task=None, **kwargs):
+    t = get_object_or_none(CeleryTask, id=task_id)
+    if t is None:
+        logger.warn("Not get the task: {}".format(task_id))
+        return
+    with transaction.atomic():
+        t.status = CeleryTask.FINISHED
+        t.date_finished = timezone.now()
+        t.save()
+    task.log_f.flush()
+    sys.stdout = task.log_f.origin_stdout
+    task.log_f.close()
+

apps/ops/celery/utils.py

@@ -1,33 +1,50 @@
-# ~*~ coding: utf-8 ~*~
+# -*- coding: utf-8 -*-
-
+#
-import os
 import json
 from functools import wraps
 
-from celery import Celery, subtask
-from celery.signals import worker_ready, worker_shutdown
 from django.db.utils import ProgrammingError, OperationalError
-
-from .utils import get_logger
-
-logger = get_logger(__file__)
-
-# set the default Django settings module for the 'celery' program.
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'jumpserver.settings')
-
-from django.conf import settings
 from django.core.cache import cache
+from django_celery_beat.models import PeriodicTask, IntervalSchedule, CrontabSchedule
 
-app = Celery('jumpserver')
 
-# Using a string here means the worker will not have to
+def add_register_period_task(name):
-# pickle the object when using Windows.
+    key = "__REGISTER_PERIODIC_TASKS"
-app.config_from_object('django.conf:settings', namespace='CELERY')
+    value = cache.get(key, [])
-app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])
+    value.append(name)
+    cache.set(key, value)
+
+
+def get_register_period_tasks():
+    key = "__REGISTER_PERIODIC_TASKS"
+    return cache.get(key, [])
+
+
+def add_after_app_shutdown_clean_task(name):
+    key = "__AFTER_APP_SHUTDOWN_CLEAN_TASKS"
+    value = cache.get(key, [])
+    value.append(name)
+    cache.set(key, value)
+
+
+def get_after_app_shutdown_clean_tasks():
+    key = "__AFTER_APP_SHUTDOWN_CLEAN_TASKS"
+    return cache.get(key, [])
+
+
+def add_after_app_ready_task(name):
+    key = "__AFTER_APP_READY_RUN_TASKS"
+    value = cache.get(key, [])
+    value.append(name)
+    cache.set(key, value)
+
+
+def get_after_app_ready_tasks():
+    key = "__AFTER_APP_READY_RUN_TASKS"
+    return cache.get(key, [])
 
 
 def create_or_update_celery_periodic_tasks(tasks):
-    from django_celery_beat.models import PeriodicTask, IntervalSchedule, CrontabSchedule
     """
     :param tasks: {
         'add-every-monday-morning': {
@@ -106,11 +123,6 @@ def delete_celery_periodic_task(task_name):
     PeriodicTask.objects.filter(name=task_name).delete()
 
 
-__REGISTER_PERIODIC_TASKS = []
-__AFTER_APP_SHUTDOWN_CLEAN_TASKS = []
-__AFTER_APP_READY_RUN_TASKS = []
-
-
 def register_as_period_task(crontab=None, interval=None):
     """
     Warning: Task must be have not any args and kwargs
@@ -128,7 +140,7 @@ def register_as_period_task(crontab=None, interval=None):
         # Because when this decorator run, the task was not created,
         # So we can't use func.name
         name = '{func.__module__}.{func.__name__}'.format(func=func)
-        if name not in __REGISTER_PERIODIC_TASKS:
+        if name not in get_register_period_tasks():
             create_or_update_celery_periodic_tasks({
                 name: {
                     'task': name,
@@ -138,7 +150,7 @@ def register_as_period_task(crontab=None, interval=None):
                     'enabled': True,
                 }
             })
-            __REGISTER_PERIODIC_TASKS.append(name)
+            add_register_period_task(name)
 
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -151,13 +163,12 @@ def after_app_ready_start(func):
     # Because when this decorator run, the task was not created,
     # So we can't use func.name
     name = '{func.__module__}.{func.__name__}'.format(func=func)
-    if name not in __AFTER_APP_READY_RUN_TASKS:
+    if name not in get_after_app_ready_tasks():
-        __AFTER_APP_READY_RUN_TASKS.append(name)
+        add_after_app_ready_task(name)
 
     @wraps(func)
     def decorate(*args, **kwargs):
         return func(*args, **kwargs)
-
     return decorate
 
 
@@ -165,37 +176,10 @@ def after_app_shutdown_clean(func):
     # Because when this decorator run, the task was not created,
     # So we can't use func.name
     name = '{func.__module__}.{func.__name__}'.format(func=func)
-    if name not in __AFTER_APP_READY_RUN_TASKS:
+    if name not in get_after_app_shutdown_clean_tasks():
-        __AFTER_APP_SHUTDOWN_CLEAN_TASKS.append(name)
+        add_after_app_shutdown_clean_task(name)
 
     @wraps(func)
     def decorate(*args, **kwargs):
         return func(*args, **kwargs)
-
     return decorate
-
-
-@worker_ready.connect
-def on_app_ready(sender=None, headers=None, body=None, **kwargs):
-    if cache.get("CELERY_APP_READY", 0) == 1:
-        return
-    cache.set("CELERY_APP_READY", 1, 10)
-    logger.debug("App ready signal recv")
-    logger.debug("Start need start task: [{}]".format(
-        ", ".join(__AFTER_APP_READY_RUN_TASKS))
-    )
-    for task in __AFTER_APP_READY_RUN_TASKS:
-        subtask(task).delay()
-
-
-@worker_shutdown.connect
-def after_app_shutdown(sender=None, headers=None, body=None, **kwargs):
-    if cache.get("CELERY_APP_SHUTDOWN", 0) == 1:
-        return
-    cache.set("CELERY_APP_SHUTDOWN", 1, 10)
-    from django_celery_beat.models import PeriodicTask
-    logger.debug("App shutdown signal recv")
-    logger.debug("Clean need cleaned period tasks: [{}]".format(
-        ', '.join(__AFTER_APP_SHUTDOWN_CLEAN_TASKS))
-    )
-    PeriodicTask.objects.filter(name__in=__AFTER_APP_SHUTDOWN_CLEAN_TASKS).delete()

apps/ops/inventory.py

@@ -15,20 +15,29 @@ class JMSInventory(BaseInventory):
     write you own manager, construct you inventory
     """
     def __init__(self, hostname_list, run_as_admin=False, run_as=None, become_info=None):
+        """
+        :param hostname_list: ["test1", ]
+        :param run_as_admin: True 是否使用管理用户去执行, 每台服务器的管理用户可能不同
+        :param run_as: 是否统一使用某个系统用户去执行
+        :param become_info: 是否become成某个用户去执行
+        """
         self.hostname_list = hostname_list
         self.using_admin = run_as_admin
         self.run_as = run_as
         self.become_info = become_info
 
         assets = self.get_jms_assets()
-        if run_as_admin:
+        host_list = []
-            host_list = [asset._to_secret_json() for asset in assets]
+
-        else:
+        for asset in assets:
-            host_list = [asset.to_json() for asset in assets]
+            info = self.convert_to_ansible(asset, run_as_admin=run_as_admin)
+            host_list.append(info)
+
         if run_as:
             run_user_info = self.get_run_user_info()
             for host in host_list:
                 host.update(run_user_info)
+
         if become_info:
             for host in host_list:
                 host.update(become_info)
@@ -38,9 +47,57 @@ class JMSInventory(BaseInventory):
         assets = get_assets_by_hostname_list(self.hostname_list)
         return assets
 
+    def convert_to_ansible(self, asset, run_as_admin=False):
+        info = {
+            'id': asset.id,
+            'hostname': asset.hostname,
+            'ip': asset.ip,
+            'port': asset.port,
+            'vars': dict(),
+            'groups': [],
+        }
+        if asset.domain and asset.domain.has_gateway():
+            info["vars"].update(self.make_proxy_command(asset))
+        if run_as_admin:
+            info.update(asset.get_auth_info())
+        for node in asset.nodes.all():
+            info["groups"].append(node.value)
+        for label in asset.labels.all():
+            info["vars"].update({
+                label.name: label.value
+            })
+            info["groups"].append("{}:{}".format(label.name, label.value))
+        if asset.domain:
+            info["vars"].update({
+                "domain": asset.domain.name,
+            })
+            info["groups"].append("domain_"+asset.domain.name)
+        return info
+
     def get_run_user_info(self):
         system_user = get_system_user_by_name(self.run_as)
         if not system_user:
             return {}
         else:
             return system_user._to_secret_json()
+
+    @staticmethod
+    def make_proxy_command(asset):
+        gateway = asset.domain.random_gateway()
+        proxy_command_list = [
+            "ssh", "-p", str(gateway.port),
+            "{}@{}".format(gateway.username, gateway.ip),
+            "-W", "%h:%p", "-q",
+        ]
+
+        if gateway.password:
+            proxy_command_list.insert(
+                0, "sshpass -p {}".format(gateway.password)
+            )
+        if gateway.private_key:
+            proxy_command_list.append("-i {}".format(gateway.private_key_file))
+
+        proxy_command = "'-o ProxyCommand={}'".format(
+            " ".join(proxy_command_list)
+        )
+        return {"ansible_ssh_common_args": proxy_command}

apps/ops/models/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+#
+
+from .adhoc import *
+from .celery import *

apps/ops/models/adhoc.py

@@ -2,18 +2,23 @@
 
 import json
 import uuid
-
+import os
 import time
+import datetime
+
+from celery import current_task
 from django.db import models
+from django.conf import settings
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask
+from django_celery_beat.models import PeriodicTask
 
 from common.utils import get_signer, get_logger
-from common.celery import delete_celery_periodic_task, create_or_update_celery_periodic_tasks, \
+from ..celery.utils import delete_celery_periodic_task, \
+    create_or_update_celery_periodic_tasks, \
     disable_celery_periodic_task
-from .ansible import AdHocRunner, AnsibleError
+from ..ansible import AdHocRunner, AnsibleError
-from .inventory import JMSInventory
+from ..inventory import JMSInventory
 
 __all__ = ["Task", "AdHoc", "AdHocRunHistory"]
 
@@ -85,7 +90,7 @@ class Task(models.Model):
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
-        from .tasks import run_ansible_task
+        from ..tasks import run_ansible_task
         super().save(
             force_insert=force_insert,  force_update=force_update,
             using=using, update_fields=update_fields,
@@ -206,10 +211,18 @@ class AdHoc(models.Model):
             return self._run_only()
 
     def _run_and_record(self):
-        history = AdHocRunHistory(adhoc=self, task=self.task)
+        try:
+            hid = current_task.request.id
+        except AttributeError:
+            hid = str(uuid.uuid4())
+        history = AdHocRunHistory(id=hid, adhoc=self, task=self.task)
         time_start = time.time()
         try:
+            date_start = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            print("{} Start task: {}\r\n".format(date_start, self.task.name))
             raw, summary = self._run_only()
+            date_end = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            print("\r\n{} Task finished".format(date_end))
             history.is_finished = True
             if summary.get('dark'):
                 history.is_success = False
@@ -221,17 +234,20 @@ class AdHoc(models.Model):
         except Exception as e:
             return {}, {"dark": {"all": str(e)}, "contacted": []}
         finally:
+            # f.close()
             history.date_finished = timezone.now()
             history.timedelta = time.time() - time_start
             history.save()
 
-    def _run_only(self):
+    def _run_only(self, file_obj=None):
-        runner = AdHocRunner(self.inventory)
+        runner = AdHocRunner(self.inventory, options=self.options)
-        for k, v in self.options.items():
-            runner.set_option(k, v)
-
         try:
-            result = runner.run(self.tasks, self.pattern, self.task.name)
+            result = runner.run(
+                self.tasks,
+                self.pattern,
+                self.task.name,
+                file_obj=file_obj,
+            )
             return result.results_raw, result.results_summary
         except AnsibleError as e:
             logger.warn("Failed run adhoc {}, {}".format(self.task.name, e))
@@ -316,6 +332,14 @@ class AdHocRunHistory(models.Model):
     def short_id(self):
         return str(self.id).split('-')[-1]
 
+    @property
+    def log_path(self):
+        dt = datetime.datetime.now().strftime('%Y-%m-%d')
+        log_dir = os.path.join(settings.PROJECT_DIR, 'data', 'ansible', dt)
+        if not os.path.exists(log_dir):
+            os.makedirs(log_dir)
+        return os.path.join(log_dir, str(self.id) + '.log')
+
     @property
     def result(self):
         if self._result:

apps/ops/models/celery.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+import uuid
+import os
+from django.conf import settings
+from django.db import models
+
+
+class CeleryTask(models.Model):
+    WAITING = "waiting"
+    RUNNING = "running"
+    FINISHED = "finished"
+    LOG_DIR = os.path.join(settings.PROJECT_DIR, 'data', 'celery')
+
+    STATUS_CHOICES = (
+        (WAITING, WAITING),
+        (RUNNING, RUNNING),
+        (FINISHED, FINISHED),
+    )
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4)
+    name = models.CharField(max_length=1024)
+    status = models.CharField(max_length=128, choices=STATUS_CHOICES)
+    log_path = models.CharField(max_length=256, blank=True, null=True)
+    date_published = models.DateTimeField(auto_now_add=True)
+    date_start = models.DateTimeField(null=True)
+    date_finished = models.DateTimeField(null=True)
+
+    def __str__(self):
+        return "{}: {}".format(self.name, self.id)
+
+    def is_finished(self):
+        return self.status == self.FINISHED
+
+    @property
+    def full_log_path(self):
+        if not self.log_path:
+            return None
+        return os.path.join(self.LOG_DIR, self.log_path)

apps/ops/tasks.py

@@ -12,14 +12,13 @@ def rerun_task():
 
 
 @shared_task
-def run_ansible_task(task_id, callback=None, **kwargs):
+def run_ansible_task(tid, callback=None, **kwargs):
     """
-    :param task_id: is the tasks serialized data
+    :param tid: is the tasks serialized data
     :param callback: callback function name
     :return:
     """
-
+    task = get_object_or_none(Task, id=tid)
-    task = get_object_or_none(Task, id=task_id)
     if task:
         result = task.run()
         if callback is not None:

apps/ops/templates/ops/adhoc_history.html

@@ -82,7 +82,8 @@ function initTable() {
         select: [],
         columnDefs: [
             {targets: 1, createdCell: function (td, cellData, rowData) {
-                $(td).html(cellData);
+                var d = new Date(cellData);
+                $(td).html(d);
              }},
             {targets: 2, createdCell: function (td, cellData) {
                 var total = "<span>" + cellData.total + "</span>";

apps/ops/templates/ops/adhoc_history_detail.html

@@ -18,6 +18,9 @@
                            <li class="active">
                                 <a href="{% url 'ops:adhoc-history-detail' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history detail' %} </a>
                             </li>
+                            <li>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Output' %} </a>
+                            </li>
                         </ul>
                     </div>
                     <div class="tab-content">

apps/ops/templates/ops/celery_task_log.html

@@ -0,0 +1,96 @@
+{% load static %}
+<head>
+    <title>term.js</title>
+    <script src="{% static 'js/jquery-2.1.1.js' %}"></script>
+    <style>
+      html {
+        background: #000;
+      }
+      h1 {
+        margin-bottom: 20px;
+        font: 20px/1.5 sans-serif;
+      }
+      .terminal {
+        float: left;
+        font-family: 'Monaco', 'Consolas', "DejaVu Sans Mono", "Liberation Mono", monospace;
+        font-size: 12px;
+        color: #f0f0f0;
+        background-color: #555;
+        padding: 20px 20px 20px;
+      }
+      .terminal-cursor {
+        color: #000;
+        background: #f0f0f0;
+      }
+    </style>
+</head>
+<div class="container">
+    <div id="term">
+    </div>
+</div>
+
+
+<script src="{% static 'js/term.js' %}"></script>
+<script>
+    var rowHeight = 1;
+    var colWidth = 1;
+    var mark = '';
+    var url = "{% url 'api-ops:celery-task-log' pk=object.id %}";
+    var term;
+    var end = false;
+    var error = false;
+    var interval = 200;
+
+    function calWinSize() {
+        var t = $('.terminal');
+        rowHeight = 1.00 * t.height() / 24;
+        colWidth = 1.00 * t.width() / 80;
+    }
+    function resize() {
+       var rows = Math.floor(window.innerHeight / rowHeight) - 2;
+       var cols = Math.floor(window.innerWidth / colWidth) - 10;
+       term.resize(cols, rows);
+    }
+    function requestAndWrite() {
+        if (!end) {
+            $.ajax({
+                url: url + '?mark=' + mark,
+                method: "GET",
+                contentType: "application/json; charset=utf-8"
+            }).done(function(data, textStatue, jqXHR) {
+                if (jqXHR.status === 203) {
+                    error = true;
+                    term.write('.');
+                    interval = 500;
+                }
+                if (jqXHR.status === 200){
+                    term.write(data.data);
+                    mark = data.mark;
+                    if (data.end){
+                        end = true
+                    }
+                }
+            })
+        }
+    }
+    $(document).ready(function () {
+        term = new Terminal({
+            cols: 80,
+            rows: 24,
+            useStyle: true,
+            screenKeys: false,
+            convertEol: false,
+            cursorBlink: false
+        });
+        term.open();
+        term.on('data', function (data) {
+            term.write(data.replace('\r', '\r\n'))
+        });
+        calWinSize();
+        resize();
+        $('.terminal').detach().appendTo('#term');
+        setInterval(function () {
+            requestAndWrite()
+        }, interval)
+    });
+</script>

apps/ops/templates/ops/task_adhoc.html

@@ -24,6 +24,9 @@
                             <li>
                                 <a href="{% url 'ops:task-history' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history' %} </a>
                             </li>
+                            <li>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.latest_history.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
+                            </li>
                         </ul>
                     </div>
                     <div class="tab-content">
@@ -105,6 +108,10 @@
                             $(td).html(cellData.user)
                         }
                     }},
+                    {targets: 6, createdCell: function (td, cellData) {
+                        var d = new Date(cellData);
+                        $(td).html(d.toLocaleString())
+                    }},
                     {targets: 7, createdCell: function (td, cellData, rowData) {
                        var detail_btn = '<a class="btn btn-xs btn-primary m-l-xs btn-run" href="{% url 'ops:adhoc-detail' pk=DEFAULT_PK %}">{% trans "Detail" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
                        if (cellData) {

apps/ops/templates/ops/task_detail.html

@@ -24,6 +24,9 @@
                             <li>
                                 <a href="{% url 'ops:task-history' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history' %} </a>
                             </li>
+                            <li>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.latest_history.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
+                            </li>
                         </ul>
                     </div>
                     <div class="tab-content">
@@ -160,6 +163,5 @@
             </div>
         </div>
     </div>
-    {% include 'users/_user_update_pk_modal.html' %}
 {% endblock %}
 

apps/ops/templates/ops/task_history.html

@@ -24,13 +24,16 @@
                             <li class="active">
                                 <a href="{% url 'ops:task-history' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history' %} </a>
                             </li>
+                            <li>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.latest_history.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
+                            </li>
                         </ul>
                     </div>
                     <div class="tab-content">
                         <div class="col-sm-12" style="padding-left: 0">
                             <div class="ibox float-e-margins">
                                 <div class="ibox-title">
-                                    <span style="float: left">{% trans 'History of ' %} <b>{{ object.task.name }}:{{ object.short_id }}</b></span>
+                                    <span style="float: left">{% trans 'History of ' %} <b>{{ object.name }}:{{ object.short_id }}</b></span>
                                     <div class="ibox-tools">
                                         <a class="collapse-link">
                                             <i class="fa fa-chevron-up"></i>
@@ -85,7 +88,8 @@ function initTable() {
         select: [],
         columnDefs: [
             {targets: 1, createdCell: function (td, cellData, rowData) {
-                $(td).html(cellData);
+                var d = new Date(cellData);
+                $(td).html(d.toLocaleString());
              }},
             {targets: 2, createdCell: function (td, cellData) {
                 var total = "<span>" + cellData.total + "</span>";

apps/ops/templates/ops/task_list.html

@@ -1,8 +1,9 @@
 {% extends '_base_list.html' %}
 {% load i18n %}
 {% load static %}
+
 {% block content_left_head %}
-    <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
+{#    <div class="uc pull-left m-r-5"><a class="btn btn-sm btn-primary btn-create-asset"> {% trans "Create task" %} </a></div>#}
 {% endblock %}
 
 
@@ -111,9 +112,10 @@ $(document).ready(function() {
     var error = function (data) {
         alert(data)
     };
-    var success = function () {
+    var success = function(data) {
-        alert("任务开始执行，重定向到任务详情页面，多刷新几次查看结果")
+        var task_id = data.task;
-        window.location = "{% url 'ops:task-detail' pk=DEFAULT_PK %}".replace('{{ DEFAULT_PK }}', uid);
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     };
     APIUpdateAttr({
         url: the_url,

apps/ops/urls/api_urls.py

@@ -15,6 +15,7 @@ router.register(r'v1/history', api.AdHocRunHistorySet, 'history')
 
 urlpatterns = [
     url(r'^v1/tasks/(?P<pk>[0-9a-zA-Z\-]{36})/run/$', api.TaskRun.as_view(), name='task-run'),
+    url(r'^v1/celery/task/(?P<pk>[0-9a-zA-Z\-]{36})/log/$', api.CeleryTaskLogApi.as_view(), name='celery-task-log'),
 ]
 
 urlpatterns += router.urls

apps/ops/urls/view_urls.py

@@ -18,4 +18,5 @@ urlpatterns = [
     url(r'^adhoc/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.AdHocDetailView.as_view(), name='adhoc-detail'),
     url(r'^adhoc/(?P<pk>[0-9a-zA-Z\-]{36})/history/$', views.AdHocHistoryView.as_view(), name='adhoc-history'),
     url(r'^adhoc/history/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.AdHocHistoryDetailView.as_view(), name='adhoc-history-detail'),
+    url(r'^celery/task/(?P<pk>[0-9a-zA-Z\-]{36})/log/$', views.CeleryTaskLogView.as_view(), name='celery-task-log'),
 ]

apps/ops/views.py

@@ -2,10 +2,10 @@
 
 from django.utils.translation import ugettext as _
 from django.conf import settings
-from django.views.generic import ListView, DetailView
+from django.views.generic import ListView, DetailView, TemplateView
 
 from common.mixins import DatetimeSearchMixin
-from .models import Task, AdHoc, AdHocRunHistory
+from .models import Task, AdHoc, AdHocRunHistory, CeleryTask
 from .hands import AdminUserRequiredMixin
 
 
@@ -119,3 +119,8 @@ class AdHocHistoryDetailView(AdminUserRequiredMixin, DetailView):
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)
+
+
+class CeleryTaskLogView(AdminUserRequiredMixin, DetailView):
+    template_name = 'ops/celery_task_log.html'
+    model = CeleryTask

apps/perms/hands.py

@@ -3,7 +3,7 @@
 
 from users.utils import AdminUserRequiredMixin
 from users.models import User, UserGroup
-from assets.models import Asset, AssetGroup, SystemUser, Node
+from assets.models import Asset, SystemUser, Node
 from assets.serializers import AssetGrantedSerializer, NodeGrantedSerializer, NodeSerializer
 
 

apps/static/js/jumpserver.js

@@ -157,7 +157,7 @@ function APIUpdateAttr(props) {
     props = props || {};
     var success_message = props.success_message || '更新成功!';
     var fail_message = props.fail_message || '更新时发生未知错误.';
-    var flash_message = true;
+    var flash_message = props.flash_message || true;
     if (props.flash_message === false){
         flash_message = false;
     }
@@ -205,6 +205,7 @@ function objectDelete(obj, name, url, redirectTo) {
             url: url,
             body: JSON.stringify(body),
             method: 'DELETE',
+            success_message: "删除成功",
             success: success,
             error: fail
         });

apps/templates/_nav.html

@@ -21,6 +21,7 @@
     </a>
     <ul class="nav nav-second-level">
         <li id="asset"><a href="{% url 'assets:asset-list' %}">{% trans 'Asset list' %}</a></li>
+        <li id="domain"><a href="{% url 'assets:domain-list' %}">{% trans 'Domain list' %}</a></li>
         <li id="admin-user"><a href="{% url 'assets:admin-user-list' %}">{% trans 'Admin user' %}</a></li>
         <li id="system-user"><a href="{% url 'assets:system-user-list' %}">{% trans 'System user' %}</a></li>
         <li id="label"><a href="{% url 'assets:label-list' %}">{% trans 'Labels' %}</a></li>

apps/terminal/backends/command/db.py

@@ -73,7 +73,7 @@ class CommandStore(CommandBase):
             session=session,
         )
         queryset = self.model.objects.filter(**filter_kwargs)
-        return [command.to_dict() for command in queryset]
+        return queryset
 
     def count(self, date_from=None, date_to=None,
               user=None, asset=None, system_user=None,

apps/terminal/backends/command/es.py

@@ -3,6 +3,7 @@
 
 from jms_es_sdk import ESStore
 from .base import CommandBase
+from .models import AbstractSessionCommand
 
 
 class CommandStore(CommandBase, ESStore):
@@ -25,7 +26,9 @@ class CommandStore(CommandBase, ESStore):
             user=user, asset=asset, system_user=system_user,
             input=input, session=session
         )
-        return [item["_source"] for item in data["hits"] if item]
+        return AbstractSessionCommand.from_multi_dict(
+            [item["_source"] for item in data["hits"] if item]
+        )
 
     def count(self, date_from=None, date_to=None,
               user=None, asset=None, system_user=None,

apps/terminal/backends/command/multi.py

@@ -10,9 +10,10 @@ class CommandStore(CommandBase):
 
     def filter(self, **kwargs):
         queryset = []
+
         for storage in self.storage_list:
             queryset.extend(storage.filter(**kwargs))
-        return sorted(queryset, key=lambda command: command["timestamp"], reverse=True)
+        return sorted(queryset, key=lambda command: command.timestamp, reverse=True)
 
     def count(self, **kwargs):
         amount = 0

apps/terminal/signals_handler.py

@@ -5,8 +5,6 @@ from django.core.cache import cache
 from django.db.utils import ProgrammingError, OperationalError
 
 from common.utils import get_logger
-from common.celery import after_app_ready_start, register_as_period_task, \
-    after_app_shutdown_clean
 from .const import ASSETS_CACHE_KEY, USERS_CACHE_KEY, SYSTEM_USER_CACHE_KEY
 
 RUNNING = False

apps/terminal/tasks.py

@@ -6,7 +6,7 @@ import datetime
 from celery import shared_task
 from django.utils import timezone
 
-from common.celery import register_as_period_task, after_app_ready_start, \
+from ops.celery.utils import register_as_period_task, after_app_ready_start, \
     after_app_shutdown_clean
 from .models import Status, Session
 

apps/users/api.py

@@ -28,7 +28,7 @@ class UserViewSet(IDInFilterMixin, BulkModelViewSet):
     queryset = User.objects.exclude(role="App")
     # queryset = User.objects.all().exclude(role="App").order_by("date_joined")
     serializer_class = UserSerializer
-    permission_classes = (IsSuperUser, IsAuthenticated)
+    permission_classes = (IsSuperUserOrAppUser, IsAuthenticated)
     filter_fields = ('username', 'email', 'name', 'id')
 
 

apps/users/forms.py

@@ -26,6 +26,11 @@ class UserCreateUpdateForm(forms.ModelForm):
         max_length=128, strip=False, required=False,
     )
     role = forms.ChoiceField(choices=role_choices, required=True, initial=User.ROLE_USER, label=_("Role"))
+    public_key = forms.CharField(
+        label=_('ssh public key'), max_length=5000, required=False,
+        widget=forms.Textarea(attrs={'placeholder': _('ssh-rsa AAAA...')}),
+        help_text=_('Paste user id_rsa.pub here.')
+    )
 
     class Meta:
         model = User
@@ -47,12 +52,28 @@ class UserCreateUpdateForm(forms.ModelForm):
             ),
         }
 
+    def clean_public_key(self):
+        public_key = self.cleaned_data['public_key']
+        if not public_key:
+            return public_key
+        if self.instance.public_key and public_key == self.instance.public_key:
+            msg = _('Public key should not be the same as your old one.')
+            raise forms.ValidationError(msg)
+
+        if not validate_ssh_public_key(public_key):
+            raise forms.ValidationError(_('Not a valid ssh public key'))
+        return public_key
+
     def save(self, commit=True):
         password = self.cleaned_data.get('password')
+        public_key = self.cleaned_data.get('public_key')
         user = super().save(commit=commit)
         if password:
             user.set_password(password)
             user.save()
+        if public_key:
+            user.public_key = public_key
+            user.save()
         return user
 
 
@@ -70,6 +91,9 @@ class UserProfileForm(forms.ModelForm):
         }
 
 
+UserProfileForm.verbose_name = _("Profile")
+
+
 class UserPasswordForm(forms.Form):
     old_password = forms.CharField(
         max_length=128, widget=forms.PasswordInput,
@@ -113,7 +137,7 @@ class UserPasswordForm(forms.Form):
 
 class UserPublicKeyForm(forms.Form):
     public_key = forms.CharField(
-        label=_('ssh public key'), max_length=5000,
+        label=_('ssh public key'), max_length=5000, required=False,
         widget=forms.Textarea(attrs={'placeholder': _('ssh-rsa AAAA...')}),
         help_text=_('Paste your id_rsa.pub here.')
     )
@@ -131,17 +155,21 @@ class UserPublicKeyForm(forms.Form):
             msg = _('Public key should not be the same as your old one.')
             raise forms.ValidationError(msg)
 
-        if not validate_ssh_public_key(public_key):
+        if public_key and not validate_ssh_public_key(public_key):
             raise forms.ValidationError(_('Not a valid ssh public key'))
         return public_key
 
     def save(self):
         public_key = self.cleaned_data['public_key']
+        if public_key:
             self.instance.public_key = public_key
             self.instance.save()
         return self.instance
 
 
+UserPublicKeyForm.verbose_name = _("Public key")
+
+
 class UserBulkUpdateForm(forms.ModelForm):
     users = forms.ModelMultipleChoiceField(
         required=True,

apps/users/models/user.py

@@ -115,7 +115,7 @@ class User(AbstractUser):
             import sshpubkeys
             try:
                 return sshpubkeys.SSHKey(self.public_key)
-            except TabError:
+            except (TabError, TypeError):
                 pass
         return PubKey()
 

apps/users/serializers.py

@@ -14,7 +14,7 @@ signer = get_signer()
 
 class UserSerializer(BulkSerializerMixin, serializers.ModelSerializer):
     groups_display = serializers.SerializerMethodField()
-    groups = serializers.PrimaryKeyRelatedField(many=True, queryset=UserGroup.objects.all())
+    groups = serializers.PrimaryKeyRelatedField(many=True, queryset=UserGroup.objects.all(), required=False)
 
     class Meta:
         model = User

apps/users/templates/users/first_login.html

@@ -45,10 +45,8 @@
                         {{ wizard.form.management_form }}
                         {% for form in wizard.form.forms %}
                             {% bootstrap_form form %}
-{#                            {{ form|bootstrap }}#}
                         {% endfor %}
                     {% else %}
-{#                        {{ wizard.form|bootstrap }}#}
                         {% bootstrap_form wizard.form %}
                     {% endif %}
                   </form>
@@ -56,8 +54,10 @@
                 <div class="actions clearfix">
                   <ul>
                     {% if wizard.steps.prev %}
-                    <li><a class="fl_goto" data-goto="{{ wizard.steps.first }}">{% trans "First step" %}</a></li>
+                    <li><a class="fl_goto" name="wizard_goto_step" data-goto="{{ wizard.steps.prev }}">{% trans "Previous" %}</a></li>
-                    <li><a class="fl_goto" name="wizard_goto_step" data-goto="{{ wizard.steps.prev }}">{% trans "Prev step" %}</a></li>
+                    {% endif %}
+                    {% if wizard.steps.next %}
+                        <li><a class="fl_goto" name="wizard_goto_step" data-goto="{{ wizard.steps.next }}">{% trans "Next" %}</a></li>
                     {% endif %}
                     <li><a id="fl_submit">{% trans "Submit" %}</a></li>
                   </ul>

apps/users/templates/users/user_detail.html

@@ -321,7 +321,7 @@ $(document).ready(function() {
         var the_url = '{% url "api-users:user-reset-password" pk=user_object.id %}';
         var body = {};
         var success = function() {
-            var msg = "{% trans "An e-mail has been sent to the user\'s mailbox." %}";
+            var msg = "{% trans "An e-mail has been sent to the user`s mailbox." %}";
             swal("{% trans 'Reset password' %}", msg, "success");
         };
         APIUpdateAttr({

apps/users/templates/users/user_profile.html

@@ -22,7 +22,7 @@
                         </ul>
                     </div>
                     <div class="tab-content">
-                        <div class="col-sm-7" style="padding-left: 0;">
+                        <div class="col-sm-8" style="padding-left: 0;">
                             <div class="ibox float-e-margins">
                                 <div class="ibox-title">
                                     <span class="label label-primary"><b>{{ user.name }}</b></span>
@@ -120,7 +120,7 @@
                             </div>
                         </div>
                     </div>
-                    <div class="col-sm-5" style="padding-left: 0;padding-right: 0">
+                    <div class="col-sm-4" style="padding-left: 0;padding-right: 0">
                         <div class="panel panel-primary">
                             <div class="panel-heading">
                                 <i class="fa fa-info-circle"></i> {% trans 'Quick modify' %}
@@ -129,18 +129,26 @@
                                 <table class="table">
                                     <tbody>
                                     <tr class="no-borders-tr">
-                                        <td>{% trans 'Reset password' %}:</td>
+                                        <td>{% trans 'Update password' %}:</td>
                                         <td>
                                             <span class="pull-right">
-                                                <a type="button" class="btn btn-primary btn-xs" style="width: 54px" href="{% url 'users:user-password-update' %}">{% trans 'Reset' %}</a>
+                                                <a type="button" class="btn btn-primary btn-xs" style="width: 54px" href="{% url 'users:user-password-update' %}">{% trans 'Update' %}</a>
                                             </span>
                                         </td>
                                     </tr>
                                     <tr>
-                                        <td>{% trans 'Reset SSH public key' %}:</td>
+                                        <td>{% trans 'Update SSH public key' %}:</td>
                                         <td>
                                             <span class="pull-right">
-                                                <a type="button" class="btn btn-primary btn-xs" style="width: 54px" href="{% url 'users:user-pubkey-update' %}">{% trans 'Reset' %}</a>
+                                                <a type="button" class="btn btn-primary btn-xs" style="width: 54px" href="{% url 'users:user-pubkey-update' %}">{% trans 'Update' %}</a>
+                                            </span>
+                                        </td>
+                                    </tr>
+                                    <tr>
+                                        <td>{% trans 'Reset public key and download' %}:</td>
+                                        <td>
+                                            <span class="pull-right">
+                                                <a type="button" class="btn btn-primary btn-xs btn-reset-pubkey" style="width: 54px">{% trans 'Reset' %}</a>
                                             </span>
                                         </td>
                                     </tr>
@@ -180,8 +188,11 @@ $(document).on('click', '#btn_update_pk', function() {
               $('#txt_pk').focus();
             }
         );
-    }
+    };
     APIUpdateAttr({ url: the_url, body: JSON.stringify(body), success: success, error: fail});
+}).on('click', '.btn-reset-pubkey', function () {
+    var the_url = '{% url "users:user-pubkey-generate" %}';
+    window.open(the_url, "_blank")
 })
 </script>
 {% endblock %}

apps/users/templates/users/user_pubkey_update.html

@@ -64,6 +64,12 @@
                                     <div class="hr-line-dashed"></div>
                                     <h3>{% trans 'Update public key' %}</h3>
                                     {% bootstrap_field form.public_key layout="horizontal" %}
+                                    <div class="form-group">
+                                        <label class="control-label col-sm-2 col-lg-2" style="padding-top: 0">{% trans 'Or reset by server' %}</label>
+                                        <div class=" col-sm-9 col-lg-9 ">
+                                            <a href="{% url 'users:user-pubkey-generate' %}">{% trans 'Reset' %}</a>
+                                        </div>
+                                    </div>
                                     <div class="hr-line-dashed"></div>
                                     <div class="form-group">
                                         <div class="col-sm-4 col-sm-offset-2">

apps/users/templates/users/user_update.html

@@ -5,4 +5,5 @@
 {% block password %}
     <h3>{% trans 'Auth' %}</h3>
     {% bootstrap_field form.password layout="horizontal" %}
+    {% bootstrap_field form.public_key layout="horizontal" %}
 {% endblock %}

apps/users/urls/views_urls.py

@@ -20,6 +20,7 @@ urlpatterns = [
     url(r'^profile/update/$', views.UserProfileUpdateView.as_view(), name='user-profile-update'),
     url(r'^profile/password/update/$', views.UserPasswordUpdateView.as_view(), name='user-password-update'),
     url(r'^profile/pubkey/update/$', views.UserPublicKeyUpdateView.as_view(), name='user-pubkey-update'),
+    url(r'^profile/pubkey/generate/$', views.UserPublicKeyGenerateView.as_view(), name='user-pubkey-generate'),
 
     # User view
     url(r'^user$', views.UserListView.as_view(), name='user-list'),

apps/users/utils.py

@@ -200,7 +200,7 @@ def get_ip_city(ip, timeout=10):
     url = 'http://int.dpool.sina.com.cn/iplookup/iplookup.php?ip=%s&format=json' % ip
     try:
         r = requests.get(url, timeout=timeout)
-    except requests.Timeout:
+    except:
         r = None
     city = 'Unknown'
     if r and r.status_code == 200:

apps/users/views/group.py

@@ -74,7 +74,7 @@ class UserGroupDetailView(AdminUserRequiredMixin, DetailView):
     template_name = 'users/user_group_detail.html'
 
     def get_context_data(self, **kwargs):
-        users = User.objects.exclude(id__in=self.object.users.all())
+        users = User.objects.exclude(id__in=self.object.users.all()).exclude(role=User.ROLE_APP)
         context = {
             'app': _('Users'),
             'action': _('User group detail'),