diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index 47b3d7a04..db00cb6a2 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -48,6 +48,7 @@ INSTALLED_APPS = [ 'authentication.apps.AuthenticationConfig', # authentication 'applications.apps.ApplicationsConfig', 'tickets.apps.TicketsConfig', + 'system.apps.SystemConfig', 'jms_oidc_rp', 'rest_framework', 'rest_framework_swagger', diff --git a/apps/jumpserver/urls.py b/apps/jumpserver/urls.py index 624212e6f..dc74bcd14 100644 --- a/apps/jumpserver/urls.py +++ b/apps/jumpserver/urls.py @@ -23,6 +23,7 @@ api_v1 = [ path('common/', include('common.urls.api_urls', namespace='api-common')), path('applications/', include('applications.urls.api_urls', namespace='api-applications')), path('tickets/', include('tickets.urls.api_urls', namespace='api-tickets')), + path('system/', include('system.urls', namespace='api-system')), ] api_v2 = [ @@ -63,7 +64,7 @@ urlpatterns = [ # External apps url path('core/auth/captcha/', include('captcha.urls')), path('core/', include(app_view_patterns)), - path('ui/', views.UIView.as_view()) + path('ui/', views.UIView.as_view()), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \ diff --git a/apps/system/__init__.py b/apps/system/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/system/admin.py b/apps/system/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/apps/system/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/apps/system/api.py b/apps/system/api.py new file mode 100644 index 000000000..d1ebb369c --- /dev/null +++ b/apps/system/api.py @@ -0,0 +1,17 @@ +# ~*~ coding: utf-8 ~*~ +from common.permissions import IsOrgAdminOrAppUser +from common.drf.api import JMSBulkModelViewSet +from common.utils import get_logger +from . import serializers +from .models import Stat + +logger = get_logger(__name__) +__all__ = ['StatViewSet'] + + +class StatViewSet(JMSBulkModelViewSet): + queryset = Stat.objects.all() + filter_fields = ('id', 'key', 'value', 'component') + search_fields = filter_fields + permission_classes = (IsOrgAdminOrAppUser,) + serializer_class = serializers.StatSerializer diff --git a/apps/system/apps.py b/apps/system/apps.py new file mode 100644 index 000000000..5dc4d64bc --- /dev/null +++ b/apps/system/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class SystemConfig(AppConfig): + name = 'system' diff --git a/apps/system/migrations/0001_initial.py b/apps/system/migrations/0001_initial.py new file mode 100644 index 000000000..d4f7ad7a1 --- /dev/null +++ b/apps/system/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# Generated by Django 3.1 on 2020-11-25 06:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Stat', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('node', models.CharField(max_length=128)), + ('ip', models.GenericIPAddressField()), + ('component', models.CharField(choices=[('core', 'Core'), ('koko', 'KoKo'), ('guacamole', 'Guacamole'), ('omnidb', 'OmniDB')], max_length=16)), + ('key', models.CharField(db_index=True, max_length=16, verbose_name='Item key')), + ('value', models.FloatField()), + ('datetime', models.DateTimeField()), + ], + ), + ] diff --git a/apps/system/migrations/__init__.py b/apps/system/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/system/models.py b/apps/system/models.py new file mode 100644 index 000000000..924f60521 --- /dev/null +++ b/apps/system/models.py @@ -0,0 +1,68 @@ +import time + +from django.db import models +import psutil +from django.utils import timezone + +from django.utils.translation import ugettext_lazy as _ + + +class Stat(models.Model): + class Components(models.TextChoices): + core = 'core', 'Core' + koko = 'koko', 'KoKo' + guacamole = 'guacamole', 'Guacamole' + omnidb = 'omnidb', 'OmniDB' + + class Keys(models.TextChoices): + cpu_load_1 = 'cpu_load', 'CPU load' + memory_used_percent = 'memory_used_percent', _('Memory used percent') + disk_used_percent = 'disk_used_percent', _('Disk used percent') + session_active = 'session_active', _('Session active') + session_processed = 'session_processed', _('Session processed') + + node = models.CharField(max_length=128) + ip = models.GenericIPAddressField() + component = models.CharField(choices=Components.choices, max_length=16) + key = models.CharField(db_index=True, max_length=16, verbose_name=_('Item key')) + value = models.FloatField() + datetime = models.DateTimeField() + + def __str__(self): + return f'{self.key}:{self.value}' + + @staticmethod + def collect_local_stats(): + memory_percent = psutil.virtual_memory().percent + cpu_load = psutil.getloadavg() + cpu_load_1 = round(cpu_load[0], 2) + cpu_load_5 = round(cpu_load[1], 2) + cpu_load_15 = round(cpu_load[2], 2) + cpu_percent = psutil.cpu_percent() + stats = dict( + memory_percent=memory_percent, + cpu_load_1=cpu_load_1, + cpu_load_5=cpu_load_5, + cpu_load_15=cpu_load_15, + cpu_load=cpu_load_1, + cpu_percent=cpu_percent + ) + return stats + + @classmethod + def keep_collect_local_stats(cls): + data = { + 'node': 'core-01', + 'ip': '192.168.1.1', + 'component': 'core' + } + while True: + stats = cls.collect_local_stats() + data['datetime'] = timezone.now() + items = [] + for k, v in stats.items(): + data['key'] = k + data['value'] = v + items.append(cls(**data)) + cls.objects.bulk_create(items, ignore_conflicts=True) + time.sleep(60) diff --git a/apps/system/serializers.py b/apps/system/serializers.py new file mode 100644 index 000000000..e8ada1d15 --- /dev/null +++ b/apps/system/serializers.py @@ -0,0 +1,12 @@ +from common.drf.serializers import BulkModelSerializer + +from .models import Stat + + +class StatSerializer(BulkModelSerializer): + class Meta: + model = Stat + fields = ( + 'id', 'node', 'ip', 'component', + 'key', 'value', 'datetime', + ) diff --git a/apps/system/tests.py b/apps/system/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/apps/system/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/apps/system/urls.py b/apps/system/urls.py new file mode 100644 index 000000000..dee2db457 --- /dev/null +++ b/apps/system/urls.py @@ -0,0 +1,14 @@ +# coding:utf-8 +from rest_framework_bulk.routes import BulkRouter + +from . import api + +app_name = 'system' + +router = BulkRouter() +router.register(r'stats', api.StatViewSet, 'stat') + +urlpatterns = [ +] + +urlpatterns += router.urls diff --git a/apps/system/views.py b/apps/system/views.py new file mode 100644 index 000000000..91ea44a21 --- /dev/null +++ b/apps/system/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/utils/generate_fake_data/generate.py b/utils/generate_fake_data/generate.py index f504c8e55..c87a8dbe4 100644 --- a/utils/generate_fake_data/generate.py +++ b/utils/generate_fake_data/generate.py @@ -15,6 +15,7 @@ django.setup() from resources.assets import AssetsGenerator, NodesGenerator, SystemUsersGenerator, AdminUsersGenerator from resources.users import UserGroupGenerator, UserGenerator from resources.perms import AssetPermissionGenerator +from resources.system import StatGenerator resource_generator_mapper = { @@ -24,7 +25,8 @@ resource_generator_mapper = { 'admin_user': AdminUsersGenerator, 'user': UserGenerator, 'user_group': UserGroupGenerator, - 'asset_permission': AssetPermissionGenerator + 'asset_permission': AssetPermissionGenerator, + 'stat': StatGenerator } diff --git a/utils/generate_fake_data/resources/system.py b/utils/generate_fake_data/resources/system.py new file mode 100644 index 000000000..69a12d144 --- /dev/null +++ b/utils/generate_fake_data/resources/system.py @@ -0,0 +1,69 @@ +import random + +from .base import FakeDataGenerator +from system.models import * + + +class StatGenerator(FakeDataGenerator): + resource = 'stat' + + nodes = [ + { + 'node': 'guacamole-01', + 'ip': '192.168.1.1', + 'component': 'guacamole' + }, + { + 'node': 'koko-01', + 'ip': '192.168.1.2', + 'component': 'koko' + }, + { + 'node': 'omnidb-01', + 'ip': '192.168.1.3', + 'component': 'omnidb' + }, + { + 'node': 'core-01', + 'ip': '192.168.1.4', + 'component': 'core' + } + ] + items_value_range = { + 'cpu_load': (0, 3.0), + 'memory_used_percent': (20, 10.0), + 'disk_used_percent': (30, 80.0), + 'thread': (100, 100), + 'goroutine': (200, 500), + 'replay_upload_health': (0, [0, 1]), + 'command_upload_health': (0, [0, 1]), + 'session_active': (100, 50), + 'session_processed': (400, 400), + 'session_failed': (50, 100), + 'session_succeeded': (500, 300) + } + + def do_generate(self, batch, batch_size): + datetime = timezone.now() + for i in batch: + datetime = datetime - timezone.timedelta(minutes=1) + items = [] + for node in self.nodes: + for key, values in self.items_value_range.items(): + base, r = values + if isinstance(r, int): + value = int(random.random() * r) + elif isinstance(r, float): + value = round(random.random() * r, 2) + elif isinstance(r, list): + value = random.choice(r) + else: + continue + value += base + node.update({ + 'key': key, + 'value': value, + 'datetime': datetime + }) + items.append(Stat(**node)) + Stat.objects.bulk_create(items, ignore_conflicts=True) \ No newline at end of file diff --git a/utils/generate_fake_data/resources/users.py b/utils/generate_fake_data/resources/users.py index 05332e4e2..0d8cbab9a 100644 --- a/utils/generate_fake_data/resources/users.py +++ b/utils/generate_fake_data/resources/users.py @@ -47,7 +47,7 @@ class UserGenerator(FakeDataGenerator): def do_generate(self, batch, batch_size): users = [] for i in batch: - username = forgery_py.internet.user_name(True) + username = forgery_py.internet.user_name(True) + '-' + str(i) email = forgery_py.internet.email_address() u = User( username=username,