merge: with v3

This commit is contained in:
ibuler
2022-12-05 15:03:21 +08:00
700 changed files with 17940 additions and 28565 deletions

View File

@@ -1,9 +1,3 @@
from .command import *
from .session import *
from .status import *
from .storage import *
from .task import *
from .terminal import *
from .sharing import *
from .replay import *
from .endpoint import *
from .component import *
from .applet import *

View File

@@ -0,0 +1,2 @@
from .applet import *
from .host import *

View File

@@ -0,0 +1,64 @@
import yaml
import os.path
from django.conf import settings
from django.core.files.storage import default_storage
from django.db import models
from django.utils.translation import gettext_lazy as _
from common.db.models import JMSBaseModel
__all__ = ['Applet', 'AppletPublication']
class Applet(JMSBaseModel):
class Type(models.TextChoices):
general = 'general', _('General')
web = 'web', _('Web')
name = models.SlugField(max_length=128, verbose_name=_('Name'), unique=True)
display_name = models.CharField(max_length=128, verbose_name=_('Display name'))
version = models.CharField(max_length=16, verbose_name=_('Version'))
author = models.CharField(max_length=128, verbose_name=_('Author'))
type = models.CharField(max_length=16, verbose_name=_('Type'), default='general', choices=Type.choices)
is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
protocols = models.JSONField(default=list, verbose_name=_('Protocol'))
tags = models.JSONField(default=list, verbose_name=_('Tags'))
comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
hosts = models.ManyToManyField(
through_fields=('applet', 'host'), through='AppletPublication',
to='AppletHost', verbose_name=_('Hosts')
)
def __str__(self):
return self.name
@property
def path(self):
return default_storage.path('applets/{}'.format(self.name))
@property
def manifest(self):
path = os.path.join(self.path, 'manifest.yml')
if not os.path.exists(path):
return None
with open(path, 'r') as f:
return yaml.safe_load(f)
@property
def icon(self):
path = os.path.join(self.path, 'icon.png')
if not os.path.exists(path):
return None
return os.path.join(settings.MEDIA_URL, 'applets', self.name, 'icon.png')
class AppletPublication(JMSBaseModel):
applet = models.ForeignKey('Applet', on_delete=models.PROTECT, related_name='publications', verbose_name=_('Applet'))
host = models.ForeignKey('AppletHost', on_delete=models.PROTECT, related_name='publications', verbose_name=_('Host'))
status = models.CharField(max_length=16, default='ready', verbose_name=_('Status'))
comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
class Meta:
unique_together = ('applet', 'host')

View File

@@ -0,0 +1,130 @@
import os
from collections import defaultdict
from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from rest_framework.exceptions import ValidationError
from simple_history.utils import bulk_create_with_history
from assets.models import Host
from common.db.models import JMSBaseModel
from common.utils import random_string
__all__ = ['AppletHost', 'AppletHostDeployment']
class AppletHost(Host):
deploy_options = models.JSONField(default=dict, verbose_name=_('Deploy options'))
inited = models.BooleanField(default=False, verbose_name=_('Inited'))
date_inited = models.DateTimeField(null=True, blank=True, verbose_name=_('Date inited'))
date_synced = models.DateTimeField(null=True, blank=True, verbose_name=_('Date synced'))
terminal = models.OneToOneField(
'terminal.Terminal', on_delete=models.PROTECT, null=True, blank=True,
related_name='applet_host', verbose_name=_('Terminal')
)
applets = models.ManyToManyField(
'Applet', verbose_name=_('Applet'),
through='AppletPublication', through_fields=('host', 'applet'),
)
LOCKING_ORG = '00000000-0000-0000-0000-000000000004'
def __str__(self):
return self.name
@property
def load(self):
if not self.terminal:
return 'offline'
return self.terminal.load
def check_terminal_binding(self, request):
request_terminal = getattr(request.user, 'terminal', None)
if not request_terminal:
raise ValidationError('Request user has no terminal')
self.date_synced = timezone.now()
if self.terminal == request_terminal:
self.save(update_fields=['date_synced'])
else:
self.terminal = request_terminal
self.save(update_fields=['terminal', 'date_synced'])
def check_applets_state(self, applets_value_list):
applets = self.applets.all()
name_version_mapper = {
value['name']: value['version']
for value in applets_value_list
}
status_applets = defaultdict(list)
for applet in applets:
if applet.name not in name_version_mapper:
status_applets['unpublished'].append(applet)
elif applet.version != name_version_mapper[applet.name]:
status_applets['not_match'].append(applet)
else:
status_applets['published'].append(applet)
for status, applets in status_applets.items():
self.publications.filter(applet__in=applets) \
.exclude(status=status) \
.update(status=status)
@staticmethod
def random_username():
return 'jms_' + random_string(8)
@staticmethod
def random_password():
return random_string(16, special_char=True)
def generate_accounts(self):
amount = int(os.getenv('TERMINAL_ACCOUNTS_AMOUNT', 100))
now_count = self.accounts.filter(privileged=False).count()
need = amount - now_count
accounts = []
account_model = self.accounts.model
for i in range(need):
username = self.random_username()
password = self.random_password()
account = account_model(
username=username, secret=password, name=username,
asset_id=self.id, secret_type='password', version=1,
org_id=self.LOCKING_ORG, is_active=False,
)
accounts.append(account)
bulk_create_with_history(accounts, account_model, batch_size=20)
class AppletHostDeployment(JMSBaseModel):
host = models.ForeignKey('AppletHost', on_delete=models.CASCADE, verbose_name=_('Hosting'))
initial = models.BooleanField(default=False, verbose_name=_('Initial'))
status = models.CharField(max_length=16, default='', verbose_name=_('Status'))
date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True)
date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished"))
comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
task = models.UUIDField(null=True, verbose_name=_('Task'))
class Meta:
ordering = ('-date_start',)
def start(self, **kwargs):
from ...automations.deploy_applet_host import DeployAppletHostManager
manager = DeployAppletHostManager(self)
manager.run(**kwargs)
def install_applet(self, applet_id, **kwargs):
from ...automations.deploy_applet_host import DeployAppletHostManager
from .applet import Applet
if applet_id:
applet = Applet.objects.get(id=applet_id)
else:
applet = None
manager = DeployAppletHostManager(self, applet=applet)
manager.install_applet(**kwargs)
def save_task(self, task):
self.task = task
self.save(update_fields=['task'])

View File

@@ -0,0 +1,5 @@
from .terminal import *
from .task import *
from .endpoint import *
from .status import *
from .storage import *

View File

@@ -1,16 +1,16 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.validators import MinValueValidator, MaxValueValidator
from applications.models import Application
from ..utils import db_port_manager, DBPortManager
from common.db.models import JMSModel
from common.db.models import JMSBaseModel
from common.db.fields import PortField
from common.utils.ip import contains_ip
from ..utils import db_port_manager, DBPortManager
db_port_manager: DBPortManager
class Endpoint(JMSModel):
class Endpoint(JMSBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'), unique=True)
host = models.CharField(max_length=256, blank=True, verbose_name=_('Host'))
# value=0 表示 disabled
@@ -83,7 +83,7 @@ class Endpoint(JMSModel):
return endpoint
class EndpointRule(JMSModel):
class EndpointRule(JMSBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'), unique=True)
ip_group = models.JSONField(default=list, verbose_name=_('IP group'))
priority = models.IntegerField(

View File

@@ -0,0 +1,29 @@
import uuid
from django.db import models
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger
logger = get_logger(__name__)
class Status(models.Model):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
session_online = models.IntegerField(verbose_name=_("Session Online"), default=0)
cpu_load = models.FloatField(verbose_name=_("CPU Load"), default=0)
memory_used = models.FloatField(verbose_name=_("Memory Used"))
disk_used = models.FloatField(verbose_name=_("Disk Used"), default=0)
connections = models.IntegerField(verbose_name=_("Connections"), default=0)
threads = models.IntegerField(verbose_name=_("Threads"), default=0)
boot_time = models.FloatField(verbose_name=_("Boot Time"), default=0)
terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE)
date_created = models.DateTimeField(auto_now_add=True)
class Meta:
db_table = 'terminal_status'
get_latest_by = 'date_created'
verbose_name = _("Status")

View File

@@ -1,14 +1,13 @@
from __future__ import unicode_literals
import copy
import os
from importlib import import_module
import jms_storage
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from common.mixins import CommonModelMixin
from common.plugins.es import QuerySet as ESQuerySet
from common.utils import get_logger
@@ -16,8 +15,8 @@ from common.db.fields import EncryptJsonDictTextField
from common.utils.timezone import local_now_date_display
from terminal.backends import TYPE_ENGINE_MAPPING
from .terminal import Terminal
from .command import Command
from .. import const
from ..session.command import Command
from terminal import const
logger = get_logger(__file__)
@@ -54,21 +53,21 @@ class CommonStorageModelMixin(models.Model):
class CommandStorage(CommonStorageModelMixin, CommonModelMixin):
type = models.CharField(
max_length=16, choices=const.CommandStorageTypeChoices.choices,
default=const.CommandStorageTypeChoices.server.value, verbose_name=_('Type'),
max_length=16, choices=const.CommandStorageType.choices,
default=const.CommandStorageType.server.value, verbose_name=_('Type'),
)
@property
def type_null(self):
return self.type == const.CommandStorageTypeChoices.null.value
return self.type == const.CommandStorageType.null.value
@property
def type_server(self):
return self.type == const.CommandStorageTypeChoices.server.value
return self.type == const.CommandStorageType.server.value
@property
def type_es(self):
return self.type == const.CommandStorageTypeChoices.es.value
return self.type == const.CommandStorageType.es.value
@property
def type_null_or_server(self):
@@ -143,17 +142,17 @@ class CommandStorage(CommonStorageModelMixin, CommonModelMixin):
class ReplayStorage(CommonStorageModelMixin, CommonModelMixin):
type = models.CharField(
max_length=16, choices=const.ReplayStorageTypeChoices.choices,
default=const.ReplayStorageTypeChoices.server.value, verbose_name=_('Type')
max_length=16, choices=const.ReplayStorageType.choices,
default=const.ReplayStorageType.server.value, verbose_name=_('Type')
)
@property
def type_null(self):
return self.type == const.ReplayStorageTypeChoices.null.value
return self.type == const.ReplayStorageType.null.value
@property
def type_server(self):
return self.type == const.ReplayStorageTypeChoices.server.value
return self.type == const.ReplayStorageType.server.value
@property
def type_null_or_server(self):
@@ -161,11 +160,11 @@ class ReplayStorage(CommonStorageModelMixin, CommonModelMixin):
@property
def type_swift(self):
return self.type == const.ReplayStorageTypeChoices.swift.value
return self.type == const.ReplayStorageType.swift.value
@property
def type_ceph(self):
return self.type == const.ReplayStorageTypeChoices.ceph.value
return self.type == const.ReplayStorageType.ceph.value
@property
def config(self):
@@ -173,7 +172,7 @@ class ReplayStorage(CommonStorageModelMixin, CommonModelMixin):
# add type config
if self.type_ceph:
_type = const.ReplayStorageTypeChoices.s3.value
_type = const.ReplayStorageType.s3.value
else:
_type = self.type
_config.update({'TYPE': _type})

View File

@@ -1,60 +1,39 @@
import time
import uuid
from django.db import models
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger
from common.utils import get_logger, lazyproperty
from common.const.signals import SKIP_SIGNAL
from users.models import User
from orgs.utils import tmp_to_root_org
from .status import Status
from .. import const
from ..const import ComponentStatusChoices as StatusChoice
from .session import Session
from terminal.const import TerminalType as TypeChoices
from users.models import User
from ..session import Session
logger = get_logger(__file__)
class TerminalStatusMixin:
ALIVE_KEY = 'TERMINAL_ALIVE_{}'
id: str
ALIVE_KEY = 'TERMINAL_ALIVE_{}'
status_set: models.Manager
@property
def latest_status(self):
return Status.get_terminal_latest_status(self)
@lazyproperty
def last_stat(self):
return self.status_set.order_by('date_created').last()
@property
def latest_status_display(self):
return self.latest_status.label
@property
def latest_stat(self):
return Status.get_terminal_latest_stat(self)
@property
def is_normal(self):
return self.latest_status == StatusChoice.normal
@property
def is_high(self):
return self.latest_status == StatusChoice.high
@property
def is_critical(self):
return self.latest_status == StatusChoice.critical
@lazyproperty
def load(self):
from ...utils import ComputeLoadUtil
return ComputeLoadUtil.compute_load(self.last_stat)
@property
def is_alive(self):
key = self.ALIVE_KEY.format(self.id)
# return self.latest_status != StatusChoice.offline
return cache.get(key, False)
def set_alive(self, ttl=120):
key = self.ALIVE_KEY.format(self.id)
cache.set(key, True, ttl)
if not self.last_stat:
return False
return time.time() - self.last_stat.date_created.timestamp() < 150
class StorageMixin:
@@ -100,17 +79,14 @@ class Terminal(StorageMixin, TerminalStatusMixin, models.Model):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
name = models.CharField(max_length=128, verbose_name=_('Name'))
type = models.CharField(
choices=const.TerminalTypeChoices.choices, default=const.TerminalTypeChoices.koko.value,
choices=TypeChoices.choices, default=TypeChoices.koko,
max_length=64, verbose_name=_('type')
)
remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address'))
ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222)
http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000)
command_storage = models.CharField(max_length=128, verbose_name=_("Command storage"), default='default')
replay_storage = models.CharField(max_length=128, verbose_name=_("Replay storage"), default='default')
user = models.OneToOneField(User, related_name='terminal', verbose_name=_('Application User'), null=True, on_delete=models.CASCADE)
is_accepted = models.BooleanField(default=False, verbose_name=_('Is Accepted'))
is_deleted = models.BooleanField(default=False)
date_created = models.DateTimeField(auto_now_add=True)
comment = models.TextField(blank=True, verbose_name=_('Comment'))
@@ -169,9 +145,7 @@ class Terminal(StorageMixin, TerminalStatusMixin, models.Model):
def __str__(self):
status = "Active"
if not self.is_accepted:
status = "NotAccept"
elif self.is_deleted:
if self.is_deleted:
status = "Deleted"
elif not self.is_active:
status = "Disable"
@@ -180,10 +154,8 @@ class Terminal(StorageMixin, TerminalStatusMixin, models.Model):
return '%s: %s' % (self.name, status)
class Meta:
ordering = ('is_accepted',)
db_table = "terminal"
verbose_name = _("Terminal")
permissions = (
('view_terminalconfig', _('Can view terminal config')),
)

View File

@@ -0,0 +1,4 @@
from .command import *
from .session import *
from .replay import *
from .sharing import *

View File

@@ -4,7 +4,7 @@ from django.db import models
from django.db.models.signals import post_save
from django.utils.translation import ugettext_lazy as _
from ..backends.command.models import AbstractSessionCommand
from terminal.backends.command.models import AbstractSessionCommand
class CommandManager(models.Manager):
@@ -33,7 +33,7 @@ class Command(AbstractSessionCommand):
cls(**{
'user': random_string(6),
'asset': random_string(10),
'system_user': random_string(6),
'account': random_string(6),
'session': str(uuid.uuid4()),
'input': random_string(16),
'output': random_string(64),

View File

@@ -11,12 +11,12 @@ from django.core.files.storage import default_storage
from django.core.cache import cache
from assets.models import Asset
from applications.models import Application
from assets.const import Protocol
from users.models import User
from orgs.mixins.models import OrgModelMixin
from django.db.models import TextChoices
from common.utils import get_object_or_none, lazyproperty
from ..backends import get_multi_command_storage
from terminal.backends import get_multi_command_storage
class Session(OrgModelMixin):
@@ -26,28 +26,13 @@ class Session(OrgModelMixin):
WT = 'WT', 'Web Terminal'
DT = 'DT', 'DB Terminal'
class PROTOCOL(TextChoices):
SSH = 'ssh', 'ssh'
RDP = 'rdp', 'rdp'
VNC = 'vnc', 'vnc'
TELNET = 'telnet', 'telnet'
MYSQL = 'mysql', 'mysql'
ORACLE = 'oracle', 'oracle'
MARIADB = 'mariadb', 'mariadb'
SQLSERVER = 'sqlserver', 'sqlserver'
POSTGRESQL = 'postgresql', 'postgresql'
REDIS = 'redis', 'redis'
MONGODB = 'mongodb', 'MongoDB'
CLICKHOUSE = 'clickhouse', 'ClickHouse'
K8S = 'k8s', 'kubernetes'
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
user = models.CharField(max_length=128, verbose_name=_("User"), db_index=True)
user_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
asset = models.CharField(max_length=128, verbose_name=_("Asset"), db_index=True)
asset_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
system_user = models.CharField(max_length=128, verbose_name=_("System user"), db_index=True)
system_user_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
account = models.CharField(max_length=128, verbose_name=_("Account"), db_index=True)
protocol = models.CharField(default='ssh', max_length=16, db_index=True)
login_from = models.CharField(max_length=2, choices=LOGIN_FROM.choices, default="ST", verbose_name=_("Login from"))
remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
is_success = models.BooleanField(default=True, db_index=True)
@@ -55,7 +40,6 @@ class Session(OrgModelMixin):
has_replay = models.BooleanField(default=False, verbose_name=_("Replay"))
has_command = models.BooleanField(default=False, verbose_name=_("Command"))
terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.DO_NOTHING, db_constraint=False)
protocol = models.CharField(choices=PROTOCOL.choices, default='ssh', max_length=16, db_index=True)
date_start = models.DateTimeField(verbose_name=_("Date start"), db_index=True, default=timezone.now)
date_end = models.DateTimeField(verbose_name=_("Date end"), null=True)
@@ -134,29 +118,20 @@ class Session(OrgModelMixin):
@property
def can_join(self):
_PROTOCOL = self.PROTOCOL
if self.is_finished:
return False
if self.login_from == self.LOGIN_FROM.RT:
return False
if self.protocol in [
_PROTOCOL.SSH, _PROTOCOL.VNC, _PROTOCOL.RDP,
_PROTOCOL.TELNET, _PROTOCOL.K8S
Protocol.ssh, Protocol.vnc, Protocol.rdp,
Protocol.telnet, Protocol.k8s
]:
return True
else:
return False
@property
def db_protocols(self):
_PROTOCOL = self.PROTOCOL
return [_PROTOCOL.MYSQL, _PROTOCOL.MARIADB, _PROTOCOL.ORACLE,
_PROTOCOL.POSTGRESQL, _PROTOCOL.SQLSERVER, _PROTOCOL.CLICKHOUSE,
_PROTOCOL.REDIS, _PROTOCOL.MONGODB]
@property
def can_terminate(self):
_PROTOCOL = self.PROTOCOL
if self.is_finished:
return False
else:

View File

@@ -1,77 +0,0 @@
from __future__ import unicode_literals
import uuid
from django.db import models
from django.forms.models import model_to_dict
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger
logger = get_logger(__name__)
class Status(models.Model):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
session_online = models.IntegerField(verbose_name=_("Session Online"), default=0)
cpu_load = models.FloatField(verbose_name=_("CPU Load"), default=0)
memory_used = models.FloatField(verbose_name=_("Memory Used"))
disk_used = models.FloatField(verbose_name=_("Disk Used"), default=0)
connections = models.IntegerField(verbose_name=_("Connections"), default=0)
threads = models.IntegerField(verbose_name=_("Threads"), default=0)
boot_time = models.FloatField(verbose_name=_("Boot Time"), default=0)
terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE)
date_created = models.DateTimeField(auto_now_add=True)
CACHE_KEY = 'TERMINAL_STATUS_{}'
class Meta:
db_table = 'terminal_status'
get_latest_by = 'date_created'
verbose_name = _("Status")
def save_to_cache(self):
if not self.terminal:
return
key = self.CACHE_KEY.format(self.terminal.id)
data = model_to_dict(self)
cache.set(key, data, 60*3)
return data
@classmethod
def get_terminal_latest_status(cls, terminal):
from ..utils import ComputeStatUtil
stat = cls.get_terminal_latest_stat(terminal)
return ComputeStatUtil.compute_component_status(stat)
@classmethod
def get_terminal_latest_stat(cls, terminal):
key = cls.CACHE_KEY.format(terminal.id)
data = cache.get(key)
if not data:
return None
data.pop('terminal', None)
stat = cls(**data)
stat.terminal = terminal
stat.is_alive = terminal.is_alive
stat.keep_one_decimal_place()
return stat
def keep_one_decimal_place(self):
keys = ['cpu_load', 'memory_used', 'disk_used']
for key in keys:
value = getattr(self, key, 0)
if not isinstance(value, (int, float)):
continue
value = '%.1f' % value
setattr(self, key, float(value))
def save(self, force_insert=False, force_update=False, using=None,
update_fields=None):
self.terminal.set_alive(ttl=120)
return self.save_to_cache()
# return super().save()