perf: refactor virutal app same like applet

This commit is contained in:
ibuler
2026-08-06 21:51:36 +08:00
parent f12722d7f6
commit 3fa72aead2
17 changed files with 887 additions and 20 deletions

View File

@@ -243,7 +243,23 @@ class ConnectionToken(JMSOrgBaseModel):
virtual_app = VirtualApp.objects.filter(name=method.get('value')).first()
if not virtual_app:
return None
return virtual_app
provider = virtual_app.select_provider(self.user)
data = {
'name': virtual_app.name,
'image_name': virtual_app.image_name,
'image_port': virtual_app.image_port,
'image_protocol': virtual_app.image_protocol,
'provider': provider,
}
# Keep the legacy globally configured Panda path working until Lion can
# consume the selected provider. Once at least one publication exists,
# however, publication state becomes authoritative.
if provider is None and virtual_app.publications.exists():
raise JMSException({
'error': 'No provider available, please check the virtual app publication and provider status'
})
return data
def get_applet_option(self):
method = self.connect_method_object

View File

@@ -224,3 +224,32 @@ class ConnectTokenVirtualAppOptionSerializer(serializers.Serializer):
image_name = serializers.CharField(label=_('Image name'))
image_port = serializers.IntegerField(label=_('Image port'))
image_protocol = serializers.CharField(label=_('Image protocol'))
provider = serializers.SerializerMethodField(label=_('App Provider'))
@staticmethod
def get_provider(instance):
provider = instance.get('provider')
if provider is None:
return None
data = {
'id': str(provider.id),
'name': provider.name,
'hostname': provider.hostname,
'address': provider.address,
'host_id': str(provider.host_id) if provider.host_id else None,
'runtime_type': provider.runtime_type,
'connection_mode': provider.connection_mode,
'service_url': provider.service_url,
'load': provider.load,
}
if provider.connection_mode == provider.ConnectionMode.ssh:
data.update({
'host': _ConnectionTokenAssetSerializer(provider.host).data,
'account': _ConnectionTokenAccountSerializer(
provider.select_account()
).data,
'gateway': _ConnectionTokenGatewaySerializer(
provider.select_gateway()
).data if provider.select_gateway() else None,
})
return data

View File

@@ -1,17 +1,20 @@
from django.core.cache import cache
from django.db import transaction
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import ValidationError
from rest_framework.response import Response
from common.api import JMSBulkModelViewSet
from common.permissions import IsServiceAccount
from orgs.utils import tmp_to_builtin_org
from terminal.models import AppProvider
from terminal.models import AppProvider, AppProviderDeployment
from terminal.serializers import (
AppProviderSerializer, AppProviderContainerSerializer
AppProviderSerializer, AppProviderContainerSerializer,
AppProviderDeploymentSerializer,
)
from terminal.tasks import run_app_provider_deployment
__all__ = ['AppProviderViewSet', ]
__all__ = ['AppProviderViewSet', 'AppProviderDeploymentViewSet']
class AppProviderViewSet(JMSBulkModelViewSet):
@@ -24,26 +27,42 @@ class AppProviderViewSet(JMSBulkModelViewSet):
'status': 'terminal.view_appprovider',
}
cache_status_key_prefix = 'virtual_host_{}_status'
cache_status_key_prefix = AppProvider.cache_status_key_prefix
def dispatch(self, request, *args, **kwargs):
with tmp_to_builtin_org(system=1):
return super().dispatch(request, *args, **kwargs)
def get_permissions(self):
if self.action == 'create':
if self.action in ('create', 'startup') and getattr(
self.request.user, 'is_service_account', False
):
return [IsServiceAccount()]
return super().get_permissions()
def perform_create(self, serializer):
request_terminal = getattr(self.request.user, 'terminal', None)
if not request_terminal:
raise ValidationError('Request user has no terminal')
serializer.save(terminal=None)
return
data = dict()
data['terminal'] = request_terminal
data['id'] = self.request.user.id
# Registration is performed by the provider service account. Runtime
# routing and host binding remain administrator-controlled settings.
data['host'] = None
data['runtime_type'] = AppProvider.RuntimeType.docker
data['connection_mode'] = AppProvider.ConnectionMode.direct
data['service_url'] = ''
data['deploy_options'] = {}
serializer.save(**data)
@action(detail=True, methods=['post'])
def startup(self, request, *args, **kwargs):
instance = self.get_object()
instance.check_terminal_binding(request)
return Response({'msg': 'ok'})
@action(detail=True, methods=['get'], serializer_class=AppProviderContainerSerializer)
def containers(self, request, *args, **kwargs):
instance = self.get_object()
@@ -62,3 +81,23 @@ class AppProviderViewSet(JMSBulkModelViewSet):
key = self.cache_status_key_prefix.format(instance.id)
cache.set(key, validated_data, 60 * 3)
return Response({'msg': 'ok'})
class AppProviderDeploymentViewSet(viewsets.ModelViewSet):
serializer_class = AppProviderDeploymentSerializer
queryset = AppProviderDeployment.objects.all()
filterset_fields = ['provider', 'status']
@staticmethod
def start_deploy(instance):
run_app_provider_deployment.apply_async(
(instance.id,), task_id=str(instance.id)
)
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
instance = serializer.save()
instance.save_task(instance.id)
transaction.on_commit(lambda: self.start_deploy(instance))
return Response({'task': str(instance.id)}, status=201)

View File

@@ -3,6 +3,7 @@ import shutil
from typing import Callable
from django.core.files.storage import default_storage
from django.db import transaction
from django.utils.translation import gettext as _
from rest_framework import viewsets
from rest_framework.decorators import action
@@ -13,7 +14,8 @@ from rest_framework.serializers import ValidationError
from common.api import JMSBulkModelViewSet
from common.serializers import FileSerializer
from terminal import serializers
from terminal.models import VirtualAppPublication, VirtualApp
from terminal.models import VirtualAppPublication, VirtualApp, AppProviderDeployment
from terminal.tasks import run_app_provider_deployment
from common.utils.zip import safe_extract_zip
__all__ = ['VirtualAppViewSet', 'VirtualAppPublicationViewSet']
@@ -81,5 +83,36 @@ class VirtualAppViewSet(UploadMixin, JMSBulkModelViewSet):
class VirtualAppPublicationViewSet(viewsets.ModelViewSet):
queryset = VirtualAppPublication.objects.all()
serializer_class = serializers.VirtualAppPublicationSerializer
filterset_fields = ['app__name', 'provider__name', 'status']
filterset_fields = ['app', 'app__name', 'provider', 'provider__name', 'status']
search_fields = ['app__name', 'provider__name', ]
@staticmethod
def start_publish(publication):
deployment = AppProviderDeployment.objects.create(
provider=publication.provider,
publication=publication,
)
deployment.save_task(deployment.id)
transaction.on_commit(
lambda: run_app_provider_deployment.apply_async(
(deployment.id,), task_id=str(deployment.id)
)
)
return deployment
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
publication = serializer.save(status='pending')
deployment = self.start_publish(publication)
data = serializer.data
data['task'] = str(deployment.id)
return Response(data, status=201)
@action(detail=True, methods=['post'])
def publish(self, request, *args, **kwargs):
publication = self.get_object()
publication.status = 'pending'
publication.save(update_fields=['status', 'date_updated'])
deployment = self.start_publish(publication)
return Response({'task': str(deployment.id)}, status=201)

View File

@@ -0,0 +1,101 @@
import datetime
import os
import shutil
import uuid
import yaml
from django.conf import settings
from django.utils import timezone
from common.db.utils import safe_db_connection
from common.utils import get_logger
from ops.ansible import JMSInventory, SuperPlaybookRunner
logger = get_logger(__name__)
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
class DeployAppProviderManager:
def __init__(self, deployment):
self.deployment = deployment
self.provider = deployment.provider
self.run_dir = self.get_run_dir()
@staticmethod
def get_run_dir():
base = os.path.join(settings.ANSIBLE_DIR, 'app_provider_deploy')
now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
return os.path.join(base, f'{now}_{uuid.uuid4().hex[:8]}')
def generate_inventory(self):
inventory = JMSInventory(
[self.provider.host], account_policy='privileged_only'
)
path = os.path.join(self.run_dir, 'inventory', 'hosts.yml')
inventory.write_to_file(path)
return path
def generate_playbook(self):
template = 'publish.yml' if self.deployment.publication_id else 'playbook.yml'
with open(os.path.join(CURRENT_DIR, template)) as f:
plays = yaml.safe_load(f)
options = self.provider.deploy_options
core_host = options.get('CORE_HOST') or settings.SITE_URL or 'http://localhost:8080'
service_url = self.provider.service_url or f'http://{self.provider.host.address}:9001'
variables = {
**options,
'CORE_HOST': core_host.rstrip('/'),
'BOOTSTRAP_TOKEN': settings.BOOTSTRAP_TOKEN,
'PROVIDER_ID': str(self.provider.id),
'PROVIDER_NAME': self.provider.name,
'PANDA_HOST_IP': self.provider.host.address,
'PANDA_IMAGE': options.get('PANDA_IMAGE', 'jumpserver/panda:latest'),
'PANDA_RANGE_PORTS': options.get('PANDA_RANGE_PORTS', '6900-7900'),
'IGNORE_VERIFY_CERTS': options.get('IGNORE_VERIFY_CERTS', True),
}
if self.deployment.publication_id:
variables['APP_IMAGE'] = self.deployment.publication.app.image_name
for play in plays:
play['vars'].update(variables)
if self.provider.service_url != service_url:
self.provider.service_url = service_url
self.provider.save(update_fields=['service_url'])
path = os.path.join(self.run_dir, 'playbook', 'main.yml')
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w') as f:
yaml.safe_dump(plays, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
return path
def run(self):
try:
self.deployment.date_start = timezone.now()
runner = SuperPlaybookRunner(
inventory=self.generate_inventory(),
playbook=self.generate_playbook(),
project_dir=self.run_dir,
safety_mode='playbook_unsafe',
inventory_safety='json_escape',
)
result = runner.run()
self.deployment.status = result.status
if self.deployment.publication_id:
publication_status = (
'pending' if result.status in ('success', 'successful') else 'failed'
)
self.deployment.publication.status = publication_status
self.deployment.publication.save(update_fields=['status', 'date_updated'])
except Exception as exc:
logger.exception('Deploy app provider failed: %s', exc)
self.deployment.status = 'error'
if self.deployment.publication_id:
self.deployment.publication.status = 'failed'
self.deployment.publication.save(update_fields=['status', 'date_updated'])
finally:
self.deployment.date_finished = timezone.now()
with safe_db_connection():
self.deployment.save()
if not settings.DEBUG_DEV:
shutil.rmtree(self.run_dir, ignore_errors=True)

View File

@@ -0,0 +1,74 @@
---
- hosts: all
become: true
vars:
CORE_HOST: https://demo.example.com
BOOTSTRAP_TOKEN: PleaseChangeMe
PROVIDER_ID: 00000000-0000-0000-0000-000000000000
PROVIDER_NAME: panda
PANDA_HOST_IP: 127.0.0.1
PANDA_IMAGE: jumpserver/panda:latest
PANDA_RANGE_PORTS: 6900-7900
IGNORE_VERIFY_CERTS: true
tasks:
- name: Check Docker runtime
ansible.builtin.command: docker version
register: docker_version
changed_when: false
failed_when: false
- name: Download Docker installer
ansible.builtin.get_url:
url: https://get.docker.com
dest: /tmp/get-docker.sh
mode: '0700'
when: docker_version.rc != 0
- name: Install Docker runtime
ansible.builtin.command: sh /tmp/get-docker.sh
when: docker_version.rc != 0
- name: Ensure Docker service is running
ansible.builtin.service:
name: docker
state: started
enabled: true
- name: Ensure Panda data directory exists
ansible.builtin.file:
path: /opt/jumpserver/panda/data
state: directory
mode: '0700'
- name: Pull Panda image
ansible.builtin.command: docker pull {{ PANDA_IMAGE }}
- name: Remove previous Panda container
ansible.builtin.command: docker rm -f jms_panda
register: remove_panda
changed_when: remove_panda.rc == 0
failed_when: false
- name: Start Panda provider
ansible.builtin.shell: >-
docker run -d --name jms_panda --restart always --network host
-v /var/run/docker.sock:/var/run/docker.sock
-v /opt/jumpserver/panda/data:/opt/panda/data
-e CORE_HOST={{ CORE_HOST | quote }}
-e BOOTSTRAP_TOKEN={{ BOOTSTRAP_TOKEN | quote }}
-e IGNORE_VERIFY_CERTS={{ IGNORE_VERIFY_CERTS | string | lower | quote }}
-e PANDA_PROVIDER_ID={{ PROVIDER_ID | quote }}
-e PANDA_HOST_IP={{ PANDA_HOST_IP | quote }}
-e PANDA_RANGE_PORTS={{ PANDA_RANGE_PORTS | quote }}
{{ PANDA_IMAGE | quote }}
no_log: true
- name: Wait for Panda health endpoint
ansible.builtin.uri:
url: http://127.0.0.1:9001/panda/health/
status_code: 200
register: panda_health
retries: 30
delay: 2
until: panda_health.status == 200

View File

@@ -0,0 +1,9 @@
---
- hosts: all
become: true
vars:
APP_IMAGE: example/application:latest
tasks:
- name: Pull virtual application image
ansible.builtin.command: docker pull {{ APP_IMAGE }}

View File

@@ -0,0 +1,40 @@
# Generated by Django 5.2.13 on 2026-08-06 13:16
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0023_platformpackage_platform_package'),
('terminal', '0011_endpoint_magnus_port'),
]
operations = [
migrations.AddField(
model_name='appprovider',
name='connection_mode',
field=models.CharField(choices=[('direct', 'Direct'), ('ssh', 'SSH')], default='direct', max_length=16, verbose_name='Connection mode'),
),
migrations.AddField(
model_name='appprovider',
name='deploy_options',
field=models.JSONField(blank=True, default=dict, verbose_name='Deploy options'),
),
migrations.AddField(
model_name='appprovider',
name='host',
field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='app_provider', to='assets.host', verbose_name='Host'),
),
migrations.AddField(
model_name='appprovider',
name='runtime_type',
field=models.CharField(choices=[('docker', 'Docker'), ('podman', 'Podman')], default='docker', max_length=16, verbose_name='Runtime type'),
),
migrations.AddField(
model_name='appprovider',
name='service_url',
field=models.URLField(blank=True, default='', max_length=1024, verbose_name='Service URL'),
),
]

View File

@@ -0,0 +1,25 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('terminal', '0012_app_provider_runtime_connection'),
]
operations = [
migrations.AddField(
model_name='virtualapppublication',
name='app_version',
field=models.CharField(blank=True, default='', max_length=16, verbose_name='Published version'),
),
migrations.AddField(
model_name='virtualapppublication',
name='image_digest',
field=models.CharField(blank=True, default='', max_length=255, verbose_name='Image digest'),
),
migrations.AddField(
model_name='virtualapppublication',
name='date_synced',
field=models.DateTimeField(blank=True, null=True, verbose_name='Date synced'),
),
]

View File

@@ -0,0 +1,36 @@
# Generated by Django 5.2.13 on 2026-08-06 14:09
import django.db.models.deletion
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('terminal', '0013_virtual_app_publication_sync_state'),
]
operations = [
migrations.CreateModel(
name='AppProviderDeployment',
fields=[
('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')),
('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')),
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('status', models.CharField(default='pending', max_length=16, verbose_name='Status')),
('date_start', models.DateTimeField(blank=True, null=True, verbose_name='Date start')),
('date_finished', models.DateTimeField(blank=True, null=True, verbose_name='Date finished')),
('task', models.UUIDField(blank=True, null=True, verbose_name='Task')),
('provider', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='deployments', to='terminal.appprovider', verbose_name='App Provider')),
('publication', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='deployments', to='terminal.virtualapppublication', verbose_name='Virtual app publication')),
],
options={
'verbose_name': 'App Provider deployment',
'ordering': ('-date_created',),
},
),
]

View File

@@ -1,14 +1,41 @@
from django.db import models
from django.core.cache import cache
from django.utils.translation import gettext_lazy as _
from rest_framework.exceptions import ValidationError
from common.db.models import JMSBaseModel
__all__ = ['AppProvider', ]
__all__ = ['AppProvider', 'AppProviderDeployment']
class AppProvider(JMSBaseModel):
class RuntimeType(models.TextChoices):
docker = 'docker', 'Docker'
podman = 'podman', 'Podman'
class ConnectionMode(models.TextChoices):
direct = 'direct', _('Direct')
ssh = 'ssh', 'SSH'
cache_status_key_prefix = 'virtual_host_{}_status'
name = models.CharField(max_length=128, verbose_name=_('Name'), unique=True)
hostname = models.CharField(max_length=128, verbose_name=_('Hostname'))
host = models.OneToOneField(
'assets.Host', on_delete=models.SET_NULL, null=True, blank=True,
related_name='app_provider', verbose_name=_('Host'),
)
runtime_type = models.CharField(
max_length=16, choices=RuntimeType.choices, default=RuntimeType.docker,
verbose_name=_('Runtime type'),
)
connection_mode = models.CharField(
max_length=16, choices=ConnectionMode.choices, default=ConnectionMode.direct,
verbose_name=_('Connection mode'),
)
service_url = models.URLField(
max_length=1024, blank=True, default='', verbose_name=_('Service URL'),
)
deploy_options = models.JSONField(default=dict, blank=True, verbose_name=_('Deploy options'))
terminal = models.OneToOneField(
'terminal.Terminal', on_delete=models.CASCADE, null=True, blank=True,
related_name='app_provider', verbose_name=_('Terminal')
@@ -27,3 +54,67 @@ class AppProvider(JMSBaseModel):
if not self.terminal:
return 'offline'
return self.terminal.load
@property
def container_count(self):
containers = cache.get(self.cache_status_key_prefix.format(self.id), [])
return len(containers)
@property
def address(self):
return self.host.address if self.host else self.hostname
def select_gateway(self):
if not self.host or not self.host.zone:
return None
return self.host.zone.select_gateway()
def check_terminal_binding(self, request):
request_terminal = getattr(request.user, 'terminal', None)
if not request_terminal:
raise ValidationError('Request user has no terminal')
self.terminal = request_terminal
self.save(update_fields=['terminal', 'date_updated'])
def select_account(self):
if not self.host:
return None
return self.host.accounts.active().order_by(
'-privileged', '-date_updated'
).first()
@property
def connection_ready(self):
if self.connection_mode == self.ConnectionMode.direct:
return True
if not self.host:
return False
has_ssh = self.host.protocols.filter(name='ssh').exists()
return has_ssh and self.select_account() is not None
class AppProviderDeployment(JMSBaseModel):
provider = models.ForeignKey(
'AppProvider', on_delete=models.CASCADE, related_name='deployments',
verbose_name=_('App Provider'),
)
publication = models.ForeignKey(
'VirtualAppPublication', on_delete=models.CASCADE, null=True, blank=True,
related_name='deployments', verbose_name=_('Virtual app publication'),
)
status = models.CharField(max_length=16, default='pending', verbose_name=_('Status'))
date_start = models.DateTimeField(null=True, blank=True, verbose_name=_('Date start'))
date_finished = models.DateTimeField(null=True, blank=True, verbose_name=_('Date finished'))
task = models.UUIDField(null=True, blank=True, verbose_name=_('Task'))
class Meta:
ordering = ('-date_created',)
verbose_name = _('App Provider deployment')
def start(self):
from terminal.automations.deploy_app_provider import DeployAppProviderManager
DeployAppProviderManager(self).run()
def save_task(self, task):
self.task = task
self.save(update_fields=['task'])

View File

@@ -2,6 +2,7 @@ import os
import shutil
from django.conf import settings
from django.core.cache import cache
from django.core.files.storage import default_storage
from django.db import models
from django.utils._os import safe_join
@@ -11,10 +12,14 @@ from rest_framework.serializers import ValidationError
from assets.utils.platform_package import locate_package_root
from common.db.models import JMSBaseModel
from common.utils import lazyproperty
from common.utils import get_logger
from common.utils.yml import yaml_load_with_i18n
from terminal.const import ComponentLoad, PublishStatus
__all__ = ['VirtualApp', 'VirtualAppPublication']
logger = get_logger(__name__)
class VirtualApp(JMSBaseModel):
name = models.SlugField(max_length=128, verbose_name=_('Name'), unique=True)
@@ -33,6 +38,8 @@ class VirtualApp(JMSBaseModel):
to='AppProvider', verbose_name=_('Providers')
)
provider_prefer_key_tpl = 'virtual_app_provider_prefer_{}_{}'
class Meta:
verbose_name = _('Virtual app')
@@ -93,6 +100,65 @@ class VirtualApp(JMSBaseModel):
shutil.copytree(path, pkg_path)
return instance, serializer
def filter_available_providers(self):
"""Return providers where this app has been published successfully.
A provider without a bound terminal, or whose terminal reports offline,
must not receive a new virtual application instance.
"""
publications = self.publications.filter(
status=PublishStatus.success
).select_related('provider__terminal')
providers = [
publication.provider for publication in publications
if publication.provider.load != ComponentLoad.offline
and publication.provider.connection_ready
]
if not providers:
logger.info('No available provider for virtual app: %s', self.name)
return providers
@classmethod
def clear_provider_prefer(cls):
cache.delete_pattern(cls.provider_prefer_key_tpl.format('*', '*'))
@classmethod
def _select_provider_by_load(cls, providers):
load_priorities = {
ComponentLoad.normal: 0,
ComponentLoad.high: 1,
ComponentLoad.critical: 2,
}
return min(
providers,
key=lambda provider: (
load_priorities.get(provider.load, 3),
provider.container_count,
str(provider.id),
),
default=None,
)
def select_provider(self, user):
providers = self.filter_available_providers()
if not providers:
return None
prefer_key = self.provider_prefer_key_tpl.format(self.id, user.id)
preferred_provider_id = cache.get(prefer_key)
preferred_provider = next(
(item for item in providers if str(item.id) == str(preferred_provider_id)),
None,
)
provider = self._select_provider_by_load(providers)
# Affinity may choose a busier provider, but must never override a
# healthier load class.
if preferred_provider and preferred_provider.load == provider.load:
provider = preferred_provider
elif provider:
cache.set(prefer_key, str(provider.id), timeout=None)
return provider
class VirtualAppPublication(JMSBaseModel):
provider = models.ForeignKey(
@@ -102,6 +168,15 @@ class VirtualAppPublication(JMSBaseModel):
'VirtualApp', on_delete=models.CASCADE, related_name='publications', verbose_name=_('Virtual app')
)
status = models.CharField(max_length=16, default='pending', verbose_name=_('Status'))
app_version = models.CharField(
max_length=16, blank=True, default='', verbose_name=_('Published version')
)
image_digest = models.CharField(
max_length=255, blank=True, default='', verbose_name=_('Image digest')
)
date_synced = models.DateTimeField(
null=True, blank=True, verbose_name=_('Date synced')
)
class Meta:
verbose_name = _('Virtual app publication')

View File

@@ -1,4 +1,5 @@
from django.utils.translation import gettext_lazy as _
from django.utils import timezone
from rest_framework import serializers
from common.const.choices import Status
@@ -28,9 +29,19 @@ class VirtualAppSerializer(ManifestI18nMixin, serializers.ModelSerializer):
'protocols', 'tags', 'comment',
] + read_only_fields
def update(self, instance, validated_data):
image_changed = any(
field in validated_data and validated_data[field] != getattr(instance, field)
for field in ('version', 'image_name')
)
instance = super().update(instance, validated_data)
if image_changed:
instance.publications.update(status=PublishStatus.mismatch)
return instance
class VirtualAppPublicationSerializer(serializers.ModelSerializer):
app = ObjectRelatedField(attrs=('id', 'name', 'image_name',), label=_("Virtual app"),
app = ObjectRelatedField(attrs=('id', 'name', 'image_name', 'version'), label=_("Virtual app"),
queryset=VirtualApp.objects.all())
provider = ObjectRelatedField(queryset=AppProvider.objects.all(), label=_("App Provider"))
status = LabeledChoiceField(choices=PublishStatus.choices, label=_("Status"), default=Status.pending)
@@ -38,5 +49,12 @@ class VirtualAppPublicationSerializer(serializers.ModelSerializer):
class Meta:
model = VirtualAppPublication
fields_mini = ['id', 'provider', 'app']
read_only_fields = ['date_created', 'date_updated']
fields = fields_mini + ['status', 'comment'] + read_only_fields
read_only_fields = ['date_created', 'date_updated', 'date_synced']
fields = fields_mini + [
'status', 'app_version', 'image_digest', 'date_synced', 'comment'
] + ['date_created', 'date_updated']
def update(self, instance, validated_data):
if {'status', 'app_version', 'image_digest'} & validated_data.keys():
validated_data['date_synced'] = timezone.now()
return super().update(instance, validated_data)

View File

@@ -1,25 +1,57 @@
from django.conf import settings
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers.fields import LabeledChoiceField
from common.serializers.fields import ObjectRelatedField
from common.const.choices import Status
from assets.models import Host
from terminal import const
from ..models import AppProvider
from ..models import AppProvider, AppProviderDeployment
__all__ = ['AppProviderSerializer', 'AppProviderContainerSerializer', ]
__all__ = [
'AppProviderSerializer', 'AppProviderContainerSerializer',
'AppProviderDeploymentSerializer',
]
class AppProviderDeployOptionsSerializer(serializers.Serializer):
CORE_HOST = serializers.CharField(
default=settings.SITE_URL, max_length=1024, label=_('Core API')
)
IGNORE_VERIFY_CERTS = serializers.BooleanField(
default=True, label=_('Ignore Certificate Verification')
)
PANDA_IMAGE = serializers.CharField(
default='jumpserver/panda:latest', max_length=255, label=_('Panda image')
)
PANDA_RANGE_PORTS = serializers.CharField(
default='6900-7900', max_length=64, label=_('Container port range')
)
class AppProviderSerializer(serializers.ModelSerializer):
host = ObjectRelatedField(
queryset=Host.objects.all(), required=False, allow_null=True,
attrs=('id', 'name', 'address'), label=_('Host'),
)
load = LabeledChoiceField(
read_only=True, label=_('Load status'), choices=const.ComponentLoad.choices,
)
deploy_options = AppProviderDeployOptionsSerializer(
required=False, label=_('Deploy options')
)
class Meta:
model = AppProvider
field_mini = ['id', 'name', 'hostname']
read_only_fields = [
'date_created', 'date_updated',
'terminal', 'date_created', 'date_updated',
]
fields = field_mini + ['load', 'terminal'] + read_only_fields
fields = field_mini + [
'host', 'runtime_type', 'connection_mode', 'service_url',
'deploy_options', 'load', 'terminal',
] + read_only_fields
class AppProviderContainerSerializer(serializers.Serializer):
@@ -29,3 +61,36 @@ class AppProviderContainerSerializer(serializers.Serializer):
container_status = serializers.CharField(label=_('Container Status'))
container_ports = serializers.ListField(child=serializers.CharField(), label=_('Container Ports'))
class AppProviderDeploymentSerializer(serializers.ModelSerializer):
status = LabeledChoiceField(
choices=Status.choices, read_only=True, label=_('Status')
)
class Meta:
model = AppProviderDeployment
fields = [
'id', 'provider', 'publication', 'status', 'task', 'comment',
'date_start', 'date_finished', 'date_created', 'date_updated',
]
read_only_fields = [
'status', 'task', 'date_start', 'date_finished',
'date_created', 'date_updated',
]
def validate_provider(self, provider):
if not provider.host:
raise serializers.ValidationError(_('Provider host is required before deployment'))
if provider.runtime_type != AppProvider.RuntimeType.docker:
raise serializers.ValidationError(_('Only Docker runtime deployment is currently supported'))
return provider
def validate(self, attrs):
attrs = super().validate(attrs)
provider = attrs.get('provider')
publication = attrs.get('publication')
if publication and publication.provider_id != provider.id:
raise serializers.ValidationError(
{'publication': _('Publication does not belong to this provider')}
)
return attrs

View File

@@ -19,7 +19,7 @@ from .backends import server_replay_storage
from .const import ReplayStorageType, CommandStorageType
from .models import (
Status, Session, Task, AppletHostDeployment,
AppletHost, ReplayStorage, CommandStorage
AppletHost, AppProviderDeployment, ReplayStorage, CommandStorage
)
from .notifications import StorageConnectivityMessage
@@ -133,6 +133,17 @@ def run_applet_host_deployment(did, install_applets):
deployment.start(install_applets=install_applets)
@shared_task(
verbose_name=_('Run app provider deployment'),
activity_callback=lambda self, did, *args, **kwargs: ([did],),
description=_('Deploy the Panda runtime on an application provider host'),
)
def run_app_provider_deployment(did):
with tmp_to_builtin_org(system=1):
deployment = AppProviderDeployment.objects.get(id=did)
deployment.start()
@shared_task(
verbose_name=_('Install applet'),
activity_callback=lambda self, ids, applet_id, *args, **kwargs: (ids,),

View File

@@ -1,10 +1,15 @@
import os
import tempfile
from unittest import mock
from django.test import SimpleTestCase
from django.test.utils import override_settings
import yaml
from assets.utils.platform_package import locate_package_root
from terminal.models import Applet, VirtualApp
from terminal.models import Applet, AppProvider, VirtualApp
from terminal.const import ComponentLoad
from terminal.automations.deploy_app_provider import DeployAppProviderManager
class PackageRootLocateTests(SimpleTestCase):
@@ -49,3 +54,202 @@ class PackageRootLocateTests(SimpleTestCase):
root = VirtualApp.locate_pkg_root(extract_to, 'demo.zip')
self.assertEqual(root, extract_to)
class VirtualAppProviderSelectionTests(SimpleTestCase):
def setUp(self):
self.app = VirtualApp(id='00000000-0000-0000-0000-000000000001', name='demo')
self.user = mock.Mock(id='00000000-0000-0000-0000-000000000002')
@mock.patch('terminal.models.virtualapp.virtualapp.cache')
def test_select_provider_prefers_previous_available_provider(self, mocked_cache):
provider1 = mock.Mock(
id='00000000-0000-0000-0000-000000000003',
load=ComponentLoad.normal,
container_count=0,
)
provider2 = mock.Mock(
id='00000000-0000-0000-0000-000000000004',
load=ComponentLoad.normal,
container_count=1,
)
mocked_cache.get.return_value = str(provider2.id)
self.app.filter_available_providers = mock.Mock(return_value=[provider1, provider2])
selected = self.app.select_provider(self.user)
self.assertIs(selected, provider2)
@mock.patch('terminal.models.virtualapp.virtualapp.cache')
def test_select_provider_does_not_prefer_degraded_provider(self, mocked_cache):
normal = mock.Mock(
id='00000000-0000-0000-0000-000000000003',
load=ComponentLoad.normal,
container_count=1,
)
preferred_high = mock.Mock(
id='00000000-0000-0000-0000-000000000004',
load=ComponentLoad.high,
container_count=0,
)
mocked_cache.get.return_value = str(preferred_high.id)
self.app.filter_available_providers = mock.Mock(
return_value=[preferred_high, normal]
)
selected = self.app.select_provider(self.user)
self.assertIs(selected, normal)
@mock.patch('terminal.models.virtualapp.virtualapp.cache')
def test_select_provider_uses_lower_load_then_container_count(self, mocked_cache):
high = mock.Mock(
id='00000000-0000-0000-0000-000000000003',
load=ComponentLoad.high,
container_count=0,
)
normal_busy = mock.Mock(
id='00000000-0000-0000-0000-000000000004',
load=ComponentLoad.normal,
container_count=2,
)
normal_idle = mock.Mock(
id='00000000-0000-0000-0000-000000000005',
load=ComponentLoad.normal,
container_count=0,
)
mocked_cache.get.return_value = None
self.app.filter_available_providers = mock.Mock(
return_value=[high, normal_busy, normal_idle]
)
selected = self.app.select_provider(self.user)
self.assertIs(selected, normal_idle)
def test_filter_available_providers_requires_success_and_online(self):
online = mock.Mock(load=ComponentLoad.normal, connection_ready=True)
offline = mock.Mock(load=ComponentLoad.offline, connection_ready=True)
publications = mock.Mock()
publications.select_related.return_value = [
mock.Mock(provider=online),
mock.Mock(provider=offline),
]
publication_manager = mock.Mock()
publication_manager.filter.return_value = publications
with mock.patch.object(
VirtualApp, 'publications', new=mock.PropertyMock(return_value=publication_manager)
):
providers = self.app.filter_available_providers()
self.assertEqual(providers, [online])
def test_filter_available_providers_excludes_unready_ssh_provider(self):
ready = mock.Mock(load=ComponentLoad.normal, connection_ready=True)
unready = mock.Mock(load=ComponentLoad.normal, connection_ready=False)
publications = mock.Mock()
publications.select_related.return_value = [
mock.Mock(provider=ready), mock.Mock(provider=unready),
]
publication_manager = mock.Mock()
publication_manager.filter.return_value = publications
with mock.patch.object(
VirtualApp, 'publications', new=mock.PropertyMock(return_value=publication_manager)
):
providers = self.app.filter_available_providers()
self.assertEqual(providers, [ready])
class AppProviderRuntimeTests(SimpleTestCase):
def test_address_falls_back_to_legacy_hostname(self):
provider = AppProvider(hostname='192.0.2.10')
self.assertEqual(provider.address, '192.0.2.10')
def test_address_uses_bound_host(self):
provider = AppProvider(hostname='legacy-address')
provider.__dict__['host_id'] = '00000000-0000-0000-0000-000000000001'
host = mock.Mock(address='198.51.100.10')
with mock.patch.object(
AppProvider, 'host', new=mock.PropertyMock(return_value=host)
):
self.assertEqual(provider.address, '198.51.100.10')
@mock.patch('terminal.models.virtualapp.provider.cache')
def test_container_count_uses_reported_provider_status(self, mocked_cache):
provider = AppProvider(id='00000000-0000-0000-0000-000000000001')
mocked_cache.get.return_value = [{'id': 'one'}, {'id': 'two'}]
self.assertEqual(provider.container_count, 2)
def test_ssh_provider_requires_host_ssh_protocol_and_account(self):
provider = AppProvider(connection_mode=AppProvider.ConnectionMode.ssh)
provider.__dict__['host_id'] = '00000000-0000-0000-0000-000000000001'
host = mock.Mock()
host.protocols.filter.return_value.exists.return_value = True
host.accounts.active.return_value.order_by.return_value.first.return_value = mock.Mock()
with mock.patch.object(
AppProvider, 'host', new=mock.PropertyMock(return_value=host)
):
self.assertTrue(provider.connection_ready)
host.accounts.active.return_value.order_by.return_value.first.return_value = None
with mock.patch.object(
AppProvider, 'host', new=mock.PropertyMock(return_value=host)
):
self.assertFalse(provider.connection_ready)
class AppProviderDeploymentTests(SimpleTestCase):
@override_settings(
SITE_URL='https://core.example.com', BOOTSTRAP_TOKEN='bootstrap-test',
DEBUG_DEV=True,
)
def test_generated_playbook_binds_panda_to_provider(self):
provider = mock.Mock(
id='00000000-0000-0000-0000-000000000010',
host=mock.Mock(address='192.0.2.10'),
service_url='',
deploy_options={
'PANDA_IMAGE': 'jumpserver/panda:test',
'PANDA_RANGE_PORTS': '7000-7100',
},
)
provider.name = 'provider-one'
deployment = mock.Mock(provider=provider, publication_id=None)
with tempfile.TemporaryDirectory() as ansible_dir, override_settings(
ANSIBLE_DIR=ansible_dir
):
manager = DeployAppProviderManager(deployment)
path = manager.generate_playbook()
with open(path) as stream:
play = yaml.safe_load(stream)[0]
variables = play['vars']
self.assertEqual(variables['PROVIDER_ID'], str(provider.id))
self.assertEqual(variables['PANDA_HOST_IP'], '192.0.2.10')
self.assertEqual(variables['PANDA_IMAGE'], 'jumpserver/panda:test')
self.assertEqual(provider.service_url, 'http://192.0.2.10:9001')
@override_settings(SITE_URL='https://core.example.com', BOOTSTRAP_TOKEN='token')
def test_publish_playbook_pulls_virtual_app_image(self):
provider = mock.Mock(
id='00000000-0000-0000-0000-000000000010',
host=mock.Mock(address='192.0.2.10'), service_url='http://192.0.2.10:9001',
deploy_options={},
)
provider.name = 'provider-one'
publication = mock.Mock(app=mock.Mock(image_name='example/app:v1'))
deployment = mock.Mock(
provider=provider, publication_id='publication-id', publication=publication,
)
with tempfile.TemporaryDirectory() as ansible_dir, override_settings(
ANSIBLE_DIR=ansible_dir
):
path = DeployAppProviderManager(deployment).generate_playbook()
with open(path) as stream:
play = yaml.safe_load(stream)[0]
self.assertEqual(play['vars']['APP_IMAGE'], 'example/app:v1')
self.assertEqual(len(play['tasks']), 1)

View File

@@ -30,6 +30,7 @@ router.register(r'applet-publications', api.AppletPublicationViewSet, 'applet-pu
router.register(r'applet-host-deployments', api.AppletHostDeploymentViewSet, 'applet-host-deployment')
router.register(r'virtual-apps', api.VirtualAppViewSet, 'virtual-app')
router.register(r'app-providers', api.AppProviderViewSet, 'app-provider')
router.register(r'app-provider-deployments', api.AppProviderDeploymentViewSet, 'app-provider-deployment')
router.register(r'app-providers/((?P<provider>[^/.]+)/)?apps', api.AppProviderAppViewSet, 'app-provider-app')
router.register(r'virtual-app-publications', api.VirtualAppPublicationViewSet, 'virtual-app-publication')