[Update] 优化可连接性

apps/assets/const.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-
+from django.utils.translation import ugettext_lazy as _
 
 UPDATE_ASSETS_HARDWARE_TASKS = [
    {
@@ -11,7 +11,6 @@ UPDATE_ASSETS_HARDWARE_TASKS = [
    }
 ]
 
-ADMIN_USER_CONN_CACHE_KEY = "ADMIN_USER_CONN_{}"
 TEST_ADMIN_USER_CONN_TASKS = [
     {
         "name": "ping",
@@ -49,7 +48,6 @@ TEST_WINDOWS_SYSTEM_USER_CONN_TASKS = [
     }
 ]
 
-ASSET_USER_CONN_CACHE_KEY = 'ASSET_USER_CONN_{}'
 TEST_ASSET_USER_CONN_TASKS = [
     {
         "name": "ping",
@@ -74,5 +72,10 @@ TASK_OPTIONS = {
 }
 
 CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX = '_KEY_ASSET_BULK_UPDATE_ID_{}'
-
+CONN_UNREACHABLE, CONN_REACHABLE, CONN_UNKNOWN = range(0, 3)
+CONNECTIVITY_CHOICES = (
+    (CONN_UNREACHABLE, _("Unreachable")),
+    (CONN_REACHABLE, _('Reachable')),
+    (CONN_UNKNOWN, _("Unknown")),
+)
 

apps/assets/models/asset.py

@@ -6,16 +6,14 @@ import uuid
 import logging
 import random
 from functools import reduce
-from collections import defaultdict
 
 from django.db import models
-from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
-from django.core.cache import cache
 from django.core.validators import MinValueValidator, MaxValueValidator
 
 from .user import AdminUser, SystemUser
 from orgs.mixins import OrgModelMixin, OrgManager
+from ..utils import Connectivity
 
 __all__ = ['Asset', 'Protocol']
 logger = logging.getLogger(__name__)
@@ -230,12 +228,12 @@ class Asset(OrgModelMixin):
     @property
     def connectivity(self):
         if not self.admin_user:
-            return self.UNKNOWN
+            return Connectivity.unknown()
-        return self.admin_user.get_connectivity_of(self)
+        return self.admin_user.get_asset_connectivity(self)
 
     @connectivity.setter
     def connectivity(self, value):
-        self.admin_user.set_connectivity_of(self, value)
+        self.admin_user.set_asset_connectivity(self, value)
 
     def get_auth_info(self):
         if not self.admin_user:

apps/assets/models/authbook.py

@@ -3,12 +3,10 @@
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from django.core.cache import cache
 
 from orgs.mixins import OrgManager
-
+from ..utils import Connectivity
 from .base import AssetUser
-from ..const import ASSET_USER_CONN_CACHE_KEY
 
 __all__ = ['AuthBook']
 
@@ -32,6 +30,7 @@ class AuthBook(AssetUser):
     backend = "db"
     # 用于system user和admin_user的动态设置
     _connectivity = None
+    CONN_CACHE_KEY = "ASSET_USER_CONN_{}"
 
     class Meta:
         verbose_name = _('AuthBook')
@@ -65,20 +64,17 @@ class AuthBook(AssetUser):
         self._set_version()
         self._set_latest()
 
-    @property
+    def get_related_assets(self):
-    def _conn_cache_key(self):
+        return [self.asset]
-        return ASSET_USER_CONN_CACHE_KEY.format(self.id)
+
+    def generate_id_with_asset(self, asset):
+        return self.id
 
     @property
     def connectivity(self):
         if self._connectivity:
             return self._connectivity
-        value = cache.get(self._conn_cache_key, self.UNKNOWN)
+        return self.get_asset_connectivity(self.asset)
-        return value
-
-    @connectivity.setter
-    def connectivity(self, value):
-        cache.set(self._conn_cache_key, value, 3600)
 
     @property
     def keyword(self):

apps/assets/models/base.py

@@ -14,8 +14,11 @@ from common.utils import (
     get_signer, ssh_key_string_to_obj, ssh_key_gen, get_logger
 )
 from common.validators import alphanumeric
+from common import fields
 from orgs.mixins import OrgModelMixin
 from .utils import private_key_validator
+from ..utils import Connectivity
+from .. import const
 
 signer = get_signer()
 
@@ -26,7 +29,7 @@ class AssetUser(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
     username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric])
-    _password = models.CharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
+    _password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
     _private_key = models.TextField(max_length=4096, blank=True, null=True, verbose_name=_('SSH private key'), validators=[private_key_validator, ])
     _public_key = models.TextField(max_length=4096, blank=True, verbose_name=_('SSH public key'))
     comment = models.TextField(blank=True, verbose_name=_('Comment'))
@@ -34,13 +37,8 @@ class AssetUser(OrgModelMixin):
     date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
     created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by'))
 
-    UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3)
+    CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_ASSET_CONNECTIVITY_{}"
-    CONNECTIVITY_CHOICES = (
+
-        (UNREACHABLE, _("Unreachable")),
-        (REACHABLE, _('Reachable')),
-        (UNKNOWN, _("Unknown")),
-    )
-    CONNECTIVITY_CACHE_KEY = "CONNECTIVITY_{}"
     _prefer = "system_user"
 
     @property
@@ -109,6 +107,10 @@ class AssetUser(OrgModelMixin):
                 pass
         return None
 
+    def get_related_assets(self):
+        assets = self.assets.all()
+        return assets
+
     def set_auth(self, password=None, private_key=None, public_key=None):
         update_fields = []
         if password:
@@ -124,17 +126,52 @@ class AssetUser(OrgModelMixin):
         if update_fields:
             self.save(update_fields=update_fields)
 
-    def get_auth(self, asset=None):
+    def set_connectivity(self, summary):
-        pass
+        unreachable = summary.get('dark', {}).keys()
+        reachable = summary.get('contacted', {}).keys()
 
-    def get_connectivity_of(self, asset):
+        for asset in self.get_related_assets():
-        i = self.generate_id_with_asset(asset)
+            if asset.hostname in unreachable:
-        key = self.CONNECTIVITY_CACHE_KEY.format(i)
+                self.set_asset_connectivity(asset, Connectivity.unreachable())
-        return cache.get(key)
+            elif asset.hostname in reachable:
+                self.set_asset_connectivity(asset, Connectivity.reachable())
+            else:
+                self.set_asset_connectivity(asset, Connectivity.unknown())
 
-    def set_connectivity_of(self, asset, c):
+    @property
+    def connectivity(self):
+        assets = self.get_related_assets()
+        data = {
+            'unreachable': [],
+            'reachable': [],
+            'unknown': [],
+        }
+        for asset in assets:
+            connectivity = self.get_asset_connectivity(asset)
+            if connectivity.is_reachable():
+                data["reachable"].append(asset.hostname)
+            elif connectivity.is_unreachable():
+                data["unreachable"].append(asset.hostname)
+            else:
+                data["unknown"].append(asset.hostname)
+        return data
+
+    @property
+    def connectivity_amount(self):
+        return {k: len(v) for k, v in self.connectivity.items()}
+
+    @property
+    def assets_amount(self):
+        return self.get_related_assets().count()
+
+    def get_asset_connectivity(self, asset):
         i = self.generate_id_with_asset(asset)
-        key = self.CONNECTIVITY_CACHE_KEY.format(i)
+        key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
+        return cache.get(key, const.CONN_UNKNOWN)
+
+    def set_asset_connectivity(self, asset, c):
+        i = self.generate_id_with_asset(asset)
+        key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
         cache.set(key, c, 3600)
 
     def load_specific_asset_auth(self, asset):
@@ -168,9 +205,10 @@ class AssetUser(OrgModelMixin):
         private_key, public_key = ssh_key_gen(
             username=self.username
         )
-        self.set_auth(password=password,
+        self.set_auth(
-                      private_key=private_key,
+            password=password, private_key=private_key,
-                      public_key=public_key)
+            public_key=public_key
+        )
 
     def auto_gen_auth_password(self):
         password = str(uuid.uuid4())
@@ -187,24 +225,18 @@ class AssetUser(OrgModelMixin):
         }
 
     def generate_id_with_asset(self, asset):
-        id_ = '{}_{}'.format(asset.id, self.id)
+        i = '{}_{}'.format(asset.id, self.id)
-        id_ = uuid.UUID(md5(id_.encode()).hexdigest())
+        i = uuid.UUID(md5(i.encode()).hexdigest())
-        return id_
+        return i
 
     def construct_to_authbook(self, asset):
-        from . import AuthBook
+        i = self.generate_id_with_asset(asset)
-        fields = [
+        self.id = i
-            'name', 'username', 'comment', 'org_id',
+        self.asset = asset
-            '_password', '_private_key', '_public_key',
+        self.version = 0
-            'date_created', 'date_updated', 'created_by'
+        self.is_latest = True
-        ]
+        return self
-        id_ = self.generate_id_with_asset(asset)
-        obj = AuthBook(id=id_, asset=asset, version=0, is_latest=True)
-        obj._connectivity = self.get_connectivity_of(asset)
-        for field in fields:
-            value = getattr(self, field)
-            setattr(obj, field, value)
-        return obj
 
     class Meta:
         abstract = True
+

apps/assets/models/user.py

@@ -4,13 +4,11 @@
 
 import logging
 
-from django.core.cache import cache
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.core.validators import MinValueValidator, MaxValueValidator
 
 from common.utils import get_signer
-from ..const import SYSTEM_USER_CONN_CACHE_KEY
 from .base import AssetUser
 
 
@@ -31,7 +29,7 @@ class AdminUser(AssetUser):
     become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4)
     become_user = models.CharField(default='root', max_length=64)
     _become_pass = models.CharField(default='', max_length=128)
-    CONNECTIVE_CACHE_KEY = '_JMS_ADMIN_USER_CONNECTIVE_{}'
+    CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}'
     _prefer = "admin_user"
 
     def __str__(self):
@@ -61,31 +59,6 @@ class AdminUser(AssetUser):
             info = None
         return info
 
-    def get_related_assets(self):
-        assets = self.assets.all()
-        return assets
-
-    @property
-    def assets_amount(self):
-        return self.get_related_assets().count()
-
-    @property
-    def connectivity(self):
-        from .asset import Asset
-        assets = self.get_related_assets().values_list('id', 'hostname', flat=True)
-        data = {
-            'unreachable': [],
-            'reachable': [],
-        }
-        for asset_id, hostname in assets:
-            key = Asset.CONNECTIVITY_CACHE_KEY.format(str(self.id))
-            value = cache.get(key, Asset.UNKNOWN)
-            if value == Asset.REACHABLE:
-                data['reachable'].append(hostname)
-            elif value == Asset.UNREACHABLE:
-                data['unreachable'].append(hostname)
-        return data
-
     class Meta:
         ordering = ['name']
         unique_together = [('name', 'org_id')]
@@ -141,9 +114,6 @@ class SystemUser(AssetUser):
     login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode'))
     cmd_filters = models.ManyToManyField('CommandFilter', related_name='system_users', verbose_name=_("Command filter"), blank=True)
 
-    SYSTEM_USER_CACHE_KEY = "__SYSTEM_USER_CACHED_{}"
-    CONNECTIVE_CACHE_KEY = '_JMS_SYSTEM_USER_CONNECTIVE_{}'
-
     def __str__(self):
         return '{0.name}({0.username})'.format(self)
 
@@ -157,49 +127,6 @@ class SystemUser(AssetUser):
             'auto_push': self.auto_push,
         }
 
-    def get_related_assets(self):
-        assets = set(self.assets.all())
-        return assets
-
-    @property
-    def connectivity(self):
-        cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id))
-        value = cache.get(cache_key, None)
-        if not value or 'unreachable' not in value:
-            return {'unreachable': [], 'reachable': []}
-        else:
-            return value
-
-    @connectivity.setter
-    def connectivity(self, value):
-        data = self.connectivity
-        unreachable = data['unreachable']
-        reachable = data['reachable']
-        assets = {asset.hostname: asset for asset in self.assets.all()}
-
-        for host in value.get('dark', {}).keys():
-            if host not in unreachable:
-                unreachable.append(host)
-            if host in reachable:
-                reachable.remove(host)
-            self.set_connectivity_of(assets.get(host), self.UNREACHABLE)
-        for host in value.get('contacted'):
-            if host not in reachable:
-                reachable.append(host)
-            if host in unreachable:
-                unreachable.remove(host)
-            self.set_connectivity_of(assets.get(host), self.REACHABLE)
-        cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id))
-        cache.set(cache_key, data, 3600)
-
-    @property
-    def assets_unreachable(self):
-        return self.connectivity.get('unreachable')
-
-    @property
-    def assets_reachable(self):
-        return self.connectivity.get('reachable')
-
     @property
     def login_mode_display(self):
         return self.get_login_mode_display()
@@ -210,12 +137,6 @@ class SystemUser(AssetUser):
         else:
             return False
 
-    def set_cache(self):
-        cache.set(self.SYSTEM_USER_CACHE_KEY.format(self.id), self, 3600)
-
-    def expire_cache(self):
-        cache.delete(self.SYSTEM_USER_CACHE_KEY.format(self.id))
-
     @property
     def cmd_filter_rules(self):
         from .cmd_filter import CommandFilterRule
@@ -233,18 +154,6 @@ class SystemUser(AssetUser):
                 return False, matched_cmd
         return True, None
 
-    @classmethod
-    def get_system_user_by_id_or_cached(cls, sid):
-        cached = cache.get(cls.SYSTEM_USER_CACHE_KEY.format(sid))
-        if cached:
-            return cached
-        try:
-            system_user = cls.objects.get(id=sid)
-            system_user.set_cache()
-            return system_user
-        except cls.DoesNotExist:
-            return None
-
     class Meta:
         ordering = ['name']
         unique_together = [('name', 'org_id')]

apps/assets/serializers/admin_user.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
 #
-from django.core.cache import cache
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
 
 from ..models import Node, AdminUser
-from ..const import ADMIN_USER_CONN_CACHE_KEY
 from orgs.mixins import BulkOrgResourceModelSerializer
 
 from .base import AuthSerializer
@@ -20,9 +18,6 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer):
     password = serializers.CharField(
         required=False, write_only=True, label=_('Password')
     )
-    unreachable_amount = serializers.SerializerMethodField(label=_('Unreachable'))
-    assets_amount = serializers.SerializerMethodField(label=_('Asset'))
-    reachable_amount = serializers.SerializerMethodField(label=_('Reachable'))
 
     class Meta:
         list_serializer_class = AdaptedBulkListSerializer
@@ -38,33 +33,14 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer):
             'date_created': {'label': _('Date created')},
             'date_updated': {'label': _('Date updated')},
             'become': {'read_only': True}, 'become_method': {'read_only': True},
-            'become_user': {'read_only': True}, 'created_by': {'read_only': True}
+            'become_user': {'read_only': True}, 'created_by': {'read_only': True},
+            'assets_amount': {'label', _('Asset')}
         }
 
     def get_field_names(self, declared_fields, info):
         fields = super().get_field_names(declared_fields, info)
         return [f for f in fields if not f.startswith('_')]
 
-    @staticmethod
-    def get_unreachable_amount(obj):
-        data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name))
-        if data:
-            return len(data.get('dark'))
-        else:
-            return 0
-
-    @staticmethod
-    def get_reachable_amount(obj):
-        data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name))
-        if data:
-            return len(data.get('contacted'))
-        else:
-            return 0
-
-    @staticmethod
-    def get_assets_amount(obj):
-        return obj.assets_amount
-
 
 class AdminUserAuthSerializer(AuthSerializer):
 

apps/assets/tasks.py

@@ -15,8 +15,9 @@ from ops.celery.decorator import (
     register_as_period_task, after_app_shutdown_clean_periodic
 )
 
-from .models import SystemUser, AdminUser, Asset
+from .models import SystemUser, AdminUser
 from . import const
+from .utils import Connectivity
 
 
 FORKS = 10
@@ -208,7 +209,7 @@ def test_asset_connectivity_util(assets, task_name=None):
             created_by=created_by,
         )
         result = task.run()
-        summary = result[1]
+        summary = result.get("summary", {})
         success = summary.get('success', False)
         contacted = summary.get('contacted', {})
         dark = summary.get('dark', {})
@@ -218,13 +219,12 @@ def test_asset_connectivity_util(assets, task_name=None):
         results_summary['dark'].update(dark)
 
     for asset in assets:
-        if asset.hostname in results_summary.get('dark', {}):
+        if asset.hostname in results_summary.get('dark', {}).keys():
-            asset.connectivity = asset.UNREACHABLE
+            asset.connectivity = Connectivity.unreachable()
-        elif asset.hostname in results_summary.get('contacted', []):
+        elif asset.hostname in results_summary.get('contacted', {}).keys():
-            asset.connectivity = asset.REACHABLE
+            asset.connectivity = Connectivity.reachable()
         else:
-            asset.connectivity = asset.UNKNOWN
+            asset.connectivity = Connectivity.unknown()
-
     return results_summary
 
 
@@ -286,10 +286,6 @@ def test_admin_user_connectivity_manual(admin_user):
 
 ##  System user connective ##
 
-@shared_task
-def set_system_user_connectivity_info(system_user, summary):
-    system_user.connectivity = summary
-
 
 @shared_task
 def test_system_user_connectivity_util(system_user, assets, task_name):
@@ -346,7 +342,7 @@ def test_system_user_connectivity_util(system_user, assets, task_name):
         results_summary['contacted'].update(contacted)
         results_summary['dark'].update(dark)
 
-    set_system_user_connectivity_info(system_user, results_summary)
+    system_user.set_connectivity(results_summary)
     return results_summary
 
 
@@ -584,6 +580,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False)
     """
     :param asset_user: <AuthBook>对象
     :param task_name:
+    :param run_as_admin:
     :return:
     """
     from ops.utils import update_or_create_ansible_task
@@ -607,7 +604,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False)
         kwargs["run_as"] = asset_user.username
     task, created = update_or_create_ansible_task(*args, **kwargs)
     result = task.run()
-    set_asset_user_connectivity_info(asset_user, result)
+    asset_user.set_connectivity(result.get("summary", {}))
 
 
 @shared_task

apps/assets/utils.py

@@ -1,8 +1,8 @@
 # ~*~ coding: utf-8 ~*~
 #
-import os
+from django.utils.translation import ugettext_lazy as _
-import paramiko
+from django.core.cache import cache
-from paramiko.ssh_exception import SSHException
+from django.utils import timezone
 
 from common.utils import get_object_or_none
 from .models import Asset, SystemUser, Label
@@ -45,3 +45,62 @@ class LabelFilter:
             for kwargs in conditions:
                 queryset = queryset.filter(**kwargs)
         return queryset
+
+
+class Connectivity:
+    UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3)
+    CONNECTIVITY_CHOICES = (
+        (UNREACHABLE, _("Unreachable")),
+        (REACHABLE, _('Reachable')),
+        (UNKNOWN, _("Unknown")),
+    )
+
+    value = UNKNOWN
+    datetime = timezone.now()
+
+    def __init__(self, value, datetime):
+        self.value = value
+        self.datetime = datetime
+
+    def display(self):
+        return dict(self.__class__.CONNECTIVITY_CHOICES).get(self.value)
+
+    def is_reachable(self):
+        return self.value == self.REACHABLE
+
+    def is_unreachable(self):
+        return self.value == self.UNREACHABLE
+
+    def is_unknown(self):
+        return self.value == self.UNKNOWN
+
+    @classmethod
+    def unreachable(cls):
+        return cls(cls.UNREACHABLE, timezone.now())
+
+    @classmethod
+    def reachable(cls):
+        return cls(cls.REACHABLE, timezone.now())
+
+    @classmethod
+    def unknown(cls):
+        return cls(cls.UNKNOWN, timezone.now())
+
+    @classmethod
+    def set(cls, key, value, ttl=0):
+        cache.set(key, value, ttl)
+
+    @classmethod
+    def get(cls, key):
+        return cache.get(key, cls.UNKNOWN)
+
+    @classmethod
+    def set_unreachable(cls, key, ttl=0):
+        cls.set(key, cls.unreachable(), ttl)
+
+    @classmethod
+    def set_reachable(cls, key, ttl=0):
+        cls.set(key, cls.reachable(), ttl)
+
+    def __eq__(self, other):
+        return self.value == other.value

apps/common/utils/common.py

@@ -293,3 +293,11 @@ class LocalProxy(object):
     __rdivmod__ = lambda x, o: x._get_current_object().__rdivmod__(o)
     __copy__ = lambda x: copy.copy(x._get_current_object())
     __deepcopy__ = lambda x, memo: copy.deepcopy(x._get_current_object(), memo)
+
+
+def random_string(length):
+    import string
+    import random
+    charset = string.ascii_letters + string.digits
+    s = [random.choice(charset) for i in range(length)]
+    return ''.join(s)

apps/terminal/api/session.py

@@ -47,9 +47,8 @@ class SessionViewSet(BulkModelViewSet):
         sid = serializer.validated_data["system_user"]
         # guacamole提交的是id
         if is_uuid(sid):
-            _system_user = SystemUser.get_system_user_by_id_or_cached(sid)
+            _system_user = get_object_or_404(SystemUser, id=sid)
-            if _system_user:
+            serializer.validated_data["system_user"] = _system_user.name
-                serializer.validated_data["system_user"] = _system_user.name
         return super().perform_create(serializer)