diff --git a/apps/terminal/api/component/status.py b/apps/terminal/api/component/status.py index 05f424283..4c4e31f27 100644 --- a/apps/terminal/api/component/status.py +++ b/apps/terminal/api/component/status.py @@ -21,7 +21,7 @@ __all__ = ['StatusViewSet', 'ComponentsMetricsAPIView'] class StatusViewSet(viewsets.ModelViewSet): queryset = Status.objects.all() - serializer_class = serializers.StatusSerializer + serializer_class = serializers.StatSerializer session_serializer_class = serializers.SessionSerializer task_serializer_class = serializers.TaskSerializer @@ -52,7 +52,7 @@ class StatusViewSet(viewsets.ModelViewSet): terminal_id = self.kwargs.get("terminal", None) if terminal_id: terminal = get_object_or_404(Terminal, id=terminal_id) - return terminal.status.all() + return terminal.status_set.all() return super().get_queryset() diff --git a/apps/terminal/api/component/storage.py b/apps/terminal/api/component/storage.py index d46b6f91f..c912e3131 100644 --- a/apps/terminal/api/component/storage.py +++ b/apps/terminal/api/component/storage.py @@ -61,7 +61,7 @@ class CommandStorageViewSet(BaseStorageViewSetMixin, viewsets.ModelViewSet): if not filterset.is_valid(): raise utils.translate_validation(filterset.errors) command_qs = filterset.qs - if storage.type == const.CommandStorageTypeChoices.es: + if storage.type == const.CommandStorageType.es: command_count = command_qs.count(limit_to_max_result_window=False) else: command_count = command_qs.count() diff --git a/apps/terminal/api/component/terminal.py b/apps/terminal/api/component/terminal.py index 872a8e2e3..3133db6eb 100644 --- a/apps/terminal/api/component/terminal.py +++ b/apps/terminal/api/component/terminal.py @@ -47,7 +47,7 @@ class TerminalViewSet(JMSBulkModelViewSet): s = self.request.query_params.get('status') if not s: return queryset - filtered_queryset_id = [str(q.id) for q in queryset if q.latest_status == s] + filtered_queryset_id = [str(q.id) for q in queryset if q.load == s] queryset = queryset.filter(id__in=filtered_queryset_id) return queryset diff --git a/apps/terminal/const.py b/apps/terminal/const.py index 91d6f5659..acea15238 100644 --- a/apps/terminal/const.py +++ b/apps/terminal/const.py @@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _ # -------------------------------- -class ReplayStorageTypeChoices(TextChoices): +class ReplayStorageType(TextChoices): null = 'null', 'Null', server = 'server', 'Server' s3 = 's3', 'S3' @@ -20,7 +20,7 @@ class ReplayStorageTypeChoices(TextChoices): cos = 'cos', 'COS' -class CommandStorageTypeChoices(TextChoices): +class CommandStorageType(TextChoices): null = 'null', 'Null', server = 'server', 'Server' es = 'es', 'Elasticsearch' @@ -29,7 +29,7 @@ class CommandStorageTypeChoices(TextChoices): # Component Status Choices # ------------------------ -class ComponentStatusChoices(TextChoices): +class ComponentLoad(TextChoices): critical = 'critical', _('Critical') high = 'high', _('High') normal = 'normal', _('Normal') @@ -40,7 +40,7 @@ class ComponentStatusChoices(TextChoices): return set(dict(cls.choices).keys()) -class TerminalTypeChoices(TextChoices): +class TerminalType(TextChoices): koko = 'koko', 'KoKo' guacamole = 'guacamole', 'Guacamole' omnidb = 'omnidb', 'OmniDB' diff --git a/apps/terminal/models/component/status.py b/apps/terminal/models/component/status.py index 863dea9dd..3da13d9f0 100644 --- a/apps/terminal/models/component/status.py +++ b/apps/terminal/models/component/status.py @@ -1,10 +1,6 @@ -from __future__ import unicode_literals - import uuid from django.db import models -from django.forms.models import model_to_dict -from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ from common.utils import get_logger @@ -22,56 +18,12 @@ class Status(models.Model): connections = models.IntegerField(verbose_name=_("Connections"), default=0) threads = models.IntegerField(verbose_name=_("Threads"), default=0) boot_time = models.FloatField(verbose_name=_("Boot Time"), default=0) - terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE, related_name='status') + terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE) date_created = models.DateTimeField(auto_now_add=True) - CACHE_KEY = 'TERMINAL_STATUS_{}' - class Meta: db_table = 'terminal_status' get_latest_by = 'date_created' verbose_name = _("Status") - def save_to_cache(self): - if not self.terminal: - return - key = self.CACHE_KEY.format(self.terminal.id) - data = model_to_dict(self) - cache.set(key, data, 60*3) - return data - - @classmethod - def get_terminal_latest_status(cls, terminal): - from ...utils import ComputeStatUtil - stat = cls.get_terminal_latest_stat(terminal) - return ComputeStatUtil.compute_component_status(stat) - - @classmethod - def get_terminal_latest_stat(cls, terminal): - key = cls.CACHE_KEY.format(terminal.id) - data = cache.get(key) - if not data: - return None - data.pop('terminal', None) - stat = cls(**data) - stat.terminal = terminal - stat.is_alive = terminal.is_alive - stat.keep_one_decimal_place() - return stat - - def keep_one_decimal_place(self): - keys = ['cpu_load', 'memory_used', 'disk_used'] - for key in keys: - value = getattr(self, key, 0) - if not isinstance(value, (int, float)): - continue - value = '%.1f' % value - setattr(self, key, float(value)) - - def save(self, force_insert=False, force_update=False, using=None, - update_fields=None): - self.terminal.set_alive(ttl=120) - return self.save_to_cache() - # return super().save() - diff --git a/apps/terminal/models/component/storage.py b/apps/terminal/models/component/storage.py index 7cab98058..11a4849a6 100644 --- a/apps/terminal/models/component/storage.py +++ b/apps/terminal/models/component/storage.py @@ -53,21 +53,21 @@ class CommonStorageModelMixin(models.Model): class CommandStorage(CommonStorageModelMixin, CommonModelMixin): type = models.CharField( - max_length=16, choices=const.CommandStorageTypeChoices.choices, - default=const.CommandStorageTypeChoices.server.value, verbose_name=_('Type'), + max_length=16, choices=const.CommandStorageType.choices, + default=const.CommandStorageType.server.value, verbose_name=_('Type'), ) @property def type_null(self): - return self.type == const.CommandStorageTypeChoices.null.value + return self.type == const.CommandStorageType.null.value @property def type_server(self): - return self.type == const.CommandStorageTypeChoices.server.value + return self.type == const.CommandStorageType.server.value @property def type_es(self): - return self.type == const.CommandStorageTypeChoices.es.value + return self.type == const.CommandStorageType.es.value @property def type_null_or_server(self): @@ -138,17 +138,17 @@ class CommandStorage(CommonStorageModelMixin, CommonModelMixin): class ReplayStorage(CommonStorageModelMixin, CommonModelMixin): type = models.CharField( - max_length=16, choices=const.ReplayStorageTypeChoices.choices, - default=const.ReplayStorageTypeChoices.server.value, verbose_name=_('Type') + max_length=16, choices=const.ReplayStorageType.choices, + default=const.ReplayStorageType.server.value, verbose_name=_('Type') ) @property def type_null(self): - return self.type == const.ReplayStorageTypeChoices.null.value + return self.type == const.ReplayStorageType.null.value @property def type_server(self): - return self.type == const.ReplayStorageTypeChoices.server.value + return self.type == const.ReplayStorageType.server.value @property def type_null_or_server(self): @@ -156,11 +156,11 @@ class ReplayStorage(CommonStorageModelMixin, CommonModelMixin): @property def type_swift(self): - return self.type == const.ReplayStorageTypeChoices.swift.value + return self.type == const.ReplayStorageType.swift.value @property def type_ceph(self): - return self.type == const.ReplayStorageTypeChoices.ceph.value + return self.type == const.ReplayStorageType.ceph.value @property def config(self): @@ -168,7 +168,7 @@ class ReplayStorage(CommonStorageModelMixin, CommonModelMixin): # add type config if self.type_ceph: - _type = const.ReplayStorageTypeChoices.s3.value + _type = const.ReplayStorageType.s3.value else: _type = self.type _config.update({'TYPE': _type}) diff --git a/apps/terminal/models/component/terminal.py b/apps/terminal/models/component/terminal.py index c24b0bd86..eb68915e4 100644 --- a/apps/terminal/models/component/terminal.py +++ b/apps/terminal/models/component/terminal.py @@ -1,16 +1,15 @@ import uuid +from django.utils import timezone from django.db import models from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ from django.conf import settings -from common.utils import get_logger +from common.utils import get_logger, lazyproperty from users.models import User from orgs.utils import tmp_to_root_org -from .status import Status -from terminal.const import TerminalTypeChoices as TypeChoices -from terminal.const import ComponentStatusChoices as StatusChoice +from terminal.const import TerminalType as TypeChoices, ComponentLoad as StatusChoice from ..session import Session @@ -18,42 +17,24 @@ logger = get_logger(__file__) class TerminalStatusMixin: - ALIVE_KEY = 'TERMINAL_ALIVE_{}' id: str + ALIVE_KEY = 'TERMINAL_ALIVE_{}' + status_set: models.Manager - @property - def latest_status(self): - return Status.get_terminal_latest_status(self) + @lazyproperty + def last_stat(self): + return self.status_set.order_by('date_created').last() - @property - def latest_status_display(self): - return self.latest_status.label - - @property - def latest_stat(self): - return Status.get_terminal_latest_stat(self) - - @property - def is_normal(self): - return self.latest_status == StatusChoice.normal - - @property - def is_high(self): - return self.latest_status == StatusChoice.high - - @property - def is_critical(self): - return self.latest_status == StatusChoice.critical + @lazyproperty + def load(self): + from ...utils import ComputeLoadUtil + return ComputeLoadUtil.compute_load(self.last_stat) @property def is_alive(self): - key = self.ALIVE_KEY.format(self.id) - # return self.latest_status != StatusChoice.offline - return cache.get(key, False) - - def set_alive(self, ttl=120): - key = self.ALIVE_KEY.format(self.id) - cache.set(key, True, ttl) + if not self.last_stat: + return False + return self.last_stat.date_created > timezone.now() - timezone.timedelta(seconds=120) class StorageMixin: diff --git a/apps/terminal/serializers/storage.py b/apps/terminal/serializers/storage.py index ff7d386de..cc4478c2d 100644 --- a/apps/terminal/serializers/storage.py +++ b/apps/terminal/serializers/storage.py @@ -118,13 +118,13 @@ class ReplayStorageTypeAzureSerializer(serializers.Serializer): # mapping replay_storage_type_serializer_classes_mapping = { - const.ReplayStorageTypeChoices.s3.value: ReplayStorageTypeS3Serializer, - const.ReplayStorageTypeChoices.ceph.value: ReplayStorageTypeCephSerializer, - const.ReplayStorageTypeChoices.swift.value: ReplayStorageTypeSwiftSerializer, - const.ReplayStorageTypeChoices.oss.value: ReplayStorageTypeOSSSerializer, - const.ReplayStorageTypeChoices.azure.value: ReplayStorageTypeAzureSerializer, - const.ReplayStorageTypeChoices.obs.value: ReplayStorageTypeOBSSerializer, - const.ReplayStorageTypeChoices.cos.value: ReplayStorageTypeCOSSerializer + const.ReplayStorageType.s3.value: ReplayStorageTypeS3Serializer, + const.ReplayStorageType.ceph.value: ReplayStorageTypeCephSerializer, + const.ReplayStorageType.swift.value: ReplayStorageTypeSwiftSerializer, + const.ReplayStorageType.oss.value: ReplayStorageTypeOSSSerializer, + const.ReplayStorageType.azure.value: ReplayStorageTypeAzureSerializer, + const.ReplayStorageType.obs.value: ReplayStorageTypeOBSSerializer, + const.ReplayStorageType.cos.value: ReplayStorageTypeCOSSerializer } @@ -172,7 +172,7 @@ class CommandStorageTypeESSerializer(serializers.Serializer): # mapping command_storage_type_serializer_classes_mapping = { - const.CommandStorageTypeChoices.es.value: CommandStorageTypeESSerializer + const.CommandStorageType.es.value: CommandStorageTypeESSerializer } diff --git a/apps/terminal/serializers/terminal.py b/apps/terminal/serializers/terminal.py index f0ac5a7c9..4b2e3614e 100644 --- a/apps/terminal/serializers/terminal.py +++ b/apps/terminal/serializers/terminal.py @@ -2,28 +2,26 @@ from rest_framework import serializers from django.utils.translation import ugettext_lazy as _ from common.drf.serializers import BulkModelSerializer -from common.utils import is_uuid -from common.utils import get_request_ip, pretty_string +from common.drf.fields import LabeledChoiceField +from common.utils import get_request_ip, pretty_string, is_uuid from users.serializers import ServiceAccountSerializer from .. import const - -from ..models import ( - Terminal, Status, Task, CommandStorage, ReplayStorage -) +from ..models import Terminal, Status, Task, CommandStorage, ReplayStorage -class StatusSerializer(serializers.ModelSerializer): +class StatSerializer(serializers.ModelSerializer): sessions = serializers.ListSerializer( - child=serializers.CharField(max_length=36), write_only=True + child=serializers.CharField(max_length=36), + write_only=True ) class Meta: + model = Status fields_mini = ['id'] fields_write_only = ['sessions', ] fields_small = fields_mini + fields_write_only + [ 'cpu_load', 'memory_used', 'disk_used', - 'session_online', - 'date_created' + 'session_online', 'date_created' ] fields_fk = ['terminal'] fields = fields_small + fields_fk @@ -32,30 +30,28 @@ class StatusSerializer(serializers.ModelSerializer): "memory_used": {'default': 0}, "disk_used": {'default': 0}, } - model = Status class TerminalSerializer(BulkModelSerializer): session_online = serializers.ReadOnlyField(source='get_online_session_count') is_alive = serializers.BooleanField(read_only=True) is_active = serializers.BooleanField(read_only=True, label='Is active') - status = serializers.ChoiceField( - read_only=True, choices=const.ComponentStatusChoices.choices, - source='latest_status', label=_('Load status') + load = LabeledChoiceField( + read_only=True, choices=const.ComponentLoad.choices, + label=_('Load status') ) - status_display = serializers.CharField(read_only=True, source='latest_status_display') - stat = StatusSerializer(read_only=True, source='latest_stat') + stat = StatSerializer(read_only=True, source='last_stat') class Meta: model = Terminal fields_mini = ['id', 'name'] fields_small = fields_mini + [ - 'type', 'remote_addr', 'http_port', 'ssh_port', - 'session_online', 'command_storage', 'replay_storage', - 'is_accepted', "is_active", 'is_alive', + 'type', 'remote_addr', 'session_online', + 'command_storage', 'replay_storage', + 'is_active', 'is_alive', 'date_created', 'comment', ] - fields_fk = ['status', 'status_display', 'stat'] + fields_fk = ['load', 'stat'] fields = fields_small + fields_fk read_only_fields = ['type', 'date_created'] extra_kwargs = { diff --git a/apps/terminal/startup.py b/apps/terminal/startup.py index 672d31830..1b4d7a7e8 100644 --- a/apps/terminal/startup.py +++ b/apps/terminal/startup.py @@ -9,8 +9,8 @@ from common.db.utils import close_old_connections from common.decorator import Singleton from common.utils import get_disk_usage, get_cpu_load, get_memory_usage, get_logger -from .serializers.terminal import TerminalRegistrationSerializer, StatusSerializer -from .const import TerminalTypeChoices +from .serializers.terminal import TerminalRegistrationSerializer, StatSerializer +from .const import TerminalType from .models import Terminal __all__ = ['CoreTerminal', 'CeleryTerminal'] @@ -51,16 +51,18 @@ class BaseTerminal(object): 'disk_used': get_disk_usage(path=settings.BASE_DIR), 'sessions': [], } - status_serializer = StatusSerializer(data=heartbeat_data) + status_serializer = StatSerializer(data=heartbeat_data) status_serializer.is_valid() status_serializer.validated_data.pop('sessions', None) terminal = self.get_or_register_terminal() status_serializer.validated_data['terminal'] = terminal try: - status_serializer.save() + status = status_serializer.save() + print("Save status ok: ", status) time.sleep(self.interval) except OperationalError: + print("Save status error, close old connections") close_old_connections() def get_or_register_terminal(self): @@ -90,8 +92,8 @@ class CoreTerminal(BaseTerminal): def __init__(self): super().__init__( - suffix_name=TerminalTypeChoices.core.label, - _type=TerminalTypeChoices.core.value + suffix_name=TerminalType.core.label, + _type=TerminalType.core.value ) @@ -99,6 +101,6 @@ class CoreTerminal(BaseTerminal): class CeleryTerminal(BaseTerminal): def __init__(self): super().__init__( - suffix_name=TerminalTypeChoices.celery.label, - _type=TerminalTypeChoices.celery.value + suffix_name=TerminalType.celery.label, + _type=TerminalType.celery.value ) diff --git a/apps/terminal/utils.py b/apps/terminal/utils.py index c41ce9bb3..53ef629f2 100644 --- a/apps/terminal/utils.py +++ b/apps/terminal/utils.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- # import os +import time from itertools import groupby, chain +from collections import defaultdict +from django.utils import timezone from django.conf import settings from django.core.files.storage import default_storage import jms_storage @@ -75,16 +78,16 @@ def get_session_replay_url(session): return local_path, url -class ComputeStatUtil: +class ComputeLoadUtil: # system status @staticmethod def _common_compute_system_status(value, thresholds): if thresholds[0] <= value <= thresholds[1]: - return const.ComponentStatusChoices.normal.value + return const.ComponentLoad.normal.value elif thresholds[1] < value <= thresholds[2]: - return const.ComponentStatusChoices.high.value + return const.ComponentLoad.high.value else: - return const.ComponentStatusChoices.critical.value + return const.ComponentLoad.critical.value @classmethod def _compute_system_stat_status(cls, stat): @@ -105,16 +108,16 @@ class ComputeStatUtil: return system_status @classmethod - def compute_component_status(cls, stat): - if not stat: - return const.ComponentStatusChoices.offline + def compute_load(cls, stat): + if not stat or time.time() - stat.date_created.timestamp() > 150: + return const.ComponentLoad.offline system_status_values = cls._compute_system_stat_status(stat).values() - if const.ComponentStatusChoices.critical in system_status_values: - return const.ComponentStatusChoices.critical - elif const.ComponentStatusChoices.high in system_status_values: - return const.ComponentStatusChoices.high + if const.ComponentLoad.critical in system_status_values: + return const.ComponentLoad.critical + elif const.ComponentLoad.high in system_status_values: + return const.ComponentLoad.high else: - return const.ComponentStatusChoices.normal + return const.ComponentLoad.normal class TypedComponentsStatusMetricsUtil(object): @@ -134,31 +137,15 @@ class TypedComponentsStatusMetricsUtil(object): def get_metrics(self): metrics = [] for _tp, components in self.grouped_components: - normal_count = high_count = critical_count = 0 - total_count = offline_count = session_online_total = 0 - + metric = { + 'normal': 0, 'high': 0, 'critical': 0, 'offline': 0, + 'total': 0, 'session_active': 0, 'type': _tp + } for component in components: - total_count += 1 - if not component.is_alive: - offline_count += 1 - continue - if component.is_normal: - normal_count += 1 - elif component.is_high: - high_count += 1 - else: - # critical - critical_count += 1 - session_online_total += component.get_online_session_count() - metrics.append({ - 'total': total_count, - 'normal': normal_count, - 'high': high_count, - 'critical': critical_count, - 'offline': offline_count, - 'session_active': session_online_total, - 'type': _tp, - }) + metric[component.load] += 1 + metric['total'] += 1 + metric['session_active'] += component.get_online_session_count() + metrics.append(metric) return metrics