merge: with v3

This commit is contained in:
ibuler
2022-12-05 15:03:21 +08:00
700 changed files with 17940 additions and 28565 deletions

View File

@@ -1,11 +1,6 @@
# -*- coding: utf-8 -*-
#
from .terminal import *
from .session import *
from .command import *
from .task import *
from .storage import *
from .status import *
from .sharing import *
from .endpoint import *
from .component import *
from .applet import *
from .db_listen_port import *

View File

@@ -0,0 +1,3 @@
from .applet import *
from .host import *
from .relation import *

View File

@@ -0,0 +1,120 @@
import shutil
import zipfile
import yaml
import os.path
from typing import Callable
from django.http import HttpResponse
from django.core.files.storage import default_storage
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import ValidationError
from common.utils import is_uuid
from common.drf.serializers import FileSerializer
from terminal import serializers
from terminal.models import AppletPublication, Applet
__all__ = ['AppletViewSet', 'AppletPublicationViewSet']
class DownloadUploadMixin:
get_serializer: Callable
request: Request
get_object: Callable
def extract_and_check_file(self, request):
serializer = self.get_serializer(data=self.request.data)
serializer.is_valid(raise_exception=True)
file = serializer.validated_data['file']
save_to = 'applets/{}'.format(file.name + '.tmp.zip')
if default_storage.exists(save_to):
default_storage.delete(save_to)
rel_path = default_storage.save(save_to, file)
path = default_storage.path(rel_path)
extract_to = default_storage.path('applets/{}.tmp'.format(file.name))
if os.path.exists(extract_to):
shutil.rmtree(extract_to)
with zipfile.ZipFile(path) as zp:
if zp.testzip() is not None:
return Response({'msg': 'Invalid Zip file'}, status=400)
zp.extractall(extract_to)
tmp_dir = os.path.join(extract_to, file.name.replace('.zip', ''))
files = ['manifest.yml', 'icon.png', 'i18n.yml', 'setup.yml']
for name in files:
path = os.path.join(tmp_dir, name)
if not os.path.exists(path):
raise ValidationError({'error': 'Missing file {}'.format(name)})
with open(os.path.join(tmp_dir, 'manifest.yml')) as f:
manifest = yaml.safe_load(f)
if not manifest.get('name', ''):
raise ValidationError({'error': 'Missing name in manifest.yml'})
return manifest, tmp_dir
@action(detail=False, methods=['post'], serializer_class=FileSerializer)
def upload(self, request, *args, **kwargs):
manifest, tmp_dir = self.extract_and_check_file(request)
name = manifest['name']
update = request.query_params.get('update')
instance = Applet.objects.filter(name=name).first()
if instance and not update:
return Response({'error': 'Applet already exists: {}'.format(name)}, status=400)
serializer = serializers.AppletSerializer(data=manifest, instance=instance)
serializer.is_valid(raise_exception=True)
save_to = default_storage.path('applets/{}'.format(name))
if os.path.exists(save_to):
shutil.rmtree(save_to)
shutil.move(tmp_dir, save_to)
serializer.save()
return Response(serializer.data, status=201)
@action(detail=True, methods=['get'])
def download(self, request, *args, **kwargs):
instance = self.get_object()
path = default_storage.path('applets/{}'.format(instance.name))
zip_path = shutil.make_archive(path, 'zip', path)
with open(zip_path, 'rb') as f:
response = HttpResponse(f.read(), status=200, content_type='application/octet-stream')
response['Content-Disposition'] = 'attachment; filename*=UTF-8\'\'{}.zip'.format(instance.name)
return response
class AppletViewSet(DownloadUploadMixin, viewsets.ModelViewSet):
queryset = Applet.objects.all()
serializer_class = serializers.AppletSerializer
rbac_perms = {
'upload': 'terminal.add_applet',
'download': 'terminal.view_applet',
}
def get_object(self):
pk = self.kwargs.get('pk')
if not is_uuid(pk):
return self.queryset.get(name=pk)
else:
return self.queryset.get(pk=pk)
def perform_destroy(self, instance):
if not instance.name:
raise ValidationError('Applet is not null')
path = default_storage.path('applets/{}'.format(instance.name))
if os.path.exists(path):
shutil.rmtree(path)
instance.delete()
class AppletPublicationViewSet(viewsets.ModelViewSet):
queryset = AppletPublication.objects.all()
serializer_class = serializers.AppletPublicationSerializer
filterset_fields = ['host', 'applet', 'status']
search_fields = ['applet__name', 'applet__display_name', 'host__name']

View File

@@ -0,0 +1,63 @@
from rest_framework import viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from common.drf.api import JMSModelViewSet
from common.permissions import IsServiceAccount
from orgs.utils import tmp_to_builtin_org
from terminal.models import AppletHost, AppletHostDeployment
from terminal.serializers import (
AppletHostSerializer, AppletHostDeploymentSerializer,
AppletHostStartupSerializer, AppletHostDeployAppletSerializer
)
from terminal.tasks import run_applet_host_deployment, run_applet_host_deployment_install_applet
__all__ = ['AppletHostViewSet', 'AppletHostDeploymentViewSet']
class AppletHostViewSet(JMSModelViewSet):
serializer_class = AppletHostSerializer
queryset = AppletHost.objects.all()
def dispatch(self, request, *args, **kwargs):
with tmp_to_builtin_org(system=1):
return super().dispatch(request, *args, **kwargs)
def get_permissions(self):
if self.action == 'startup':
return [IsServiceAccount()]
return super().get_permissions()
@action(methods=['post'], detail=True, serializer_class=AppletHostStartupSerializer)
def startup(self, request, *args, **kwargs):
instance = self.get_object()
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
instance.check_terminal_binding(request)
return Response({'msg': 'ok'})
class AppletHostDeploymentViewSet(viewsets.ModelViewSet):
serializer_class = AppletHostDeploymentSerializer
queryset = AppletHostDeployment.objects.all()
rbac_perms = (
('applets', 'terminal.view_AppletHostDeployment'),
)
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
instance = serializer.save()
task = run_applet_host_deployment.delay(instance.id)
instance.save_task(task.id)
return Response({'task': str(task.id)}, status=201)
@action(methods=['post'], detail=False, serializer_class=AppletHostDeployAppletSerializer)
def applets(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
applet_id = serializer.validated_data.get('applet_id')
instance = serializer.save()
task = run_applet_host_deployment_install_applet.delay(instance.id, applet_id)
instance.save_task(task.id)
return Response({'task': str(task.id)}, status=201)

View File

@@ -0,0 +1,86 @@
from typing import Callable
from django.shortcuts import get_object_or_404
from django.conf import settings
from rest_framework.request import Request
from rest_framework.decorators import action
from rest_framework.response import Response
from common.drf.api import JMSModelViewSet
from common.permissions import IsServiceAccount
from common.utils import is_uuid
from orgs.utils import tmp_to_builtin_org
from rbac.permissions import RBACPermission
from terminal.models import AppletHost
from terminal.serializers import (
AppletHostAccountSerializer,
AppletPublicationSerializer,
AppletHostAppletReportSerializer,
)
class HostMixin:
request: Request
permission_denied: Callable
kwargs: dict
rbac_perms = (
('list', 'terminal.view_applethost'),
('retrieve', 'terminal.view_applethost'),
)
def get_permissions(self):
if self.kwargs.get('host') and settings.DEBUG:
return [RBACPermission()]
else:
return [IsServiceAccount()]
def self_host(self):
try:
return self.request.user.terminal.applet_host
except AttributeError:
raise self.permission_denied(self.request, 'User has no applet host')
def pk_host(self):
return get_object_or_404(AppletHost, id=self.kwargs.get('host'))
@property
def host(self):
if self.kwargs.get('host'):
return self.pk_host()
else:
return self.self_host()
class AppletHostAccountsViewSet(HostMixin, JMSModelViewSet):
serializer_class = AppletHostAccountSerializer
def get_queryset(self):
with tmp_to_builtin_org(system=1):
queryset = self.host.accounts.all()
return queryset
class AppletHostAppletViewSet(HostMixin, JMSModelViewSet):
host: AppletHost
serializer_class = AppletPublicationSerializer
def get_object(self):
pk = self.kwargs.get('pk')
if not is_uuid(pk):
return self.host.publications.get(applet__name=pk)
else:
return self.host.publications.get(pk=pk)
def get_queryset(self):
queryset = self.host.publications.all()
return queryset
@action(methods=['post'], detail=False)
def reports(self, request, *args, **kwargs):
serializer = AppletHostAppletReportSerializer(data=request.data, many=True)
serializer.is_valid(raise_exception=True)
data = serializer.validated_data
self.host.check_applets_state(data)
publications = self.host.publications.all()
serializer = AppletPublicationSerializer(publications, many=True)
return Response(serializer.data)

View File

@@ -0,0 +1,4 @@
from .terminal import *
from .storage import *
from .status import *
from .endpoint import *

View File

@@ -1,18 +1,16 @@
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework import status
from rest_framework.request import Request
from common.drf.api import JMSBulkModelViewSet
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import get_object_or_404
from assets.models import Asset
from orgs.utils import tmp_to_root_org
from applications.models import Application
from terminal.models import Session
from ..models import Endpoint, EndpointRule
from .. import serializers
from common.permissions import IsValidUserOrConnectionToken
from django.utils.translation import ugettext_lazy as _
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.request import Request
from rest_framework.response import Response
from assets.models import Asset
from common.drf.api import JMSBulkModelViewSet
from common.permissions import IsValidUserOrConnectionToken
from orgs.utils import tmp_to_root_org
from terminal import serializers
from terminal.models import Session, Endpoint, EndpointRule
__all__ = ['EndpointViewSet', 'EndpointRuleViewSet']
@@ -25,8 +23,7 @@ class SmartEndpointViewMixin:
target_instance: None
target_protocol: None
@action(methods=['get'], detail=False, permission_classes=[IsValidUserOrConnectionToken],
url_path='smart')
@action(methods=['get'], detail=False, permission_classes=[IsValidUserOrConnectionToken])
def smart(self, request, *args, **kwargs):
self.target_instance = self.get_target_instance()
self.target_protocol = self.get_target_protocol()
@@ -58,21 +55,16 @@ class SmartEndpointViewMixin:
def get_target_instance(self):
request = self.request
asset_id = request.GET.get('asset_id')
app_id = request.GET.get('app_id')
session_id = request.GET.get('session_id')
token_id = request.GET.get('token')
if token_id:
from authentication.models import ConnectionToken
token = ConnectionToken.objects.filter(id=token_id).first()
if token:
if token.asset:
asset_id = token.asset.id
elif token.application:
app_id = token.application.id
if token and token.asset:
asset_id = token.asset.id
if asset_id:
pk, model = asset_id, Asset
elif app_id:
pk, model = app_id, Application
elif session_id:
pk, model = session_id, Session
else:
@@ -84,7 +76,10 @@ class SmartEndpointViewMixin:
return instance
def get_target_protocol(self):
return self.request.GET.get('protocol')
protocol = None
if not protocol:
protocol = self.request.GET.get('protocol')
return protocol
class EndpointViewSet(SmartEndpointViewMixin, JMSBulkModelViewSet):

View File

@@ -9,9 +9,9 @@ from rest_framework import viewsets, generics
from rest_framework.views import Response
from rest_framework import status
from ..models import Terminal, Status, Session
from .. import serializers
from ..utils import TypedComponentsStatusMetricsUtil
from terminal.models import Terminal, Status, Session
from terminal import serializers
from terminal.utils import TypedComponentsStatusMetricsUtil
logger = logging.getLogger(__file__)
@@ -21,7 +21,7 @@ __all__ = ['StatusViewSet', 'ComponentsMetricsAPIView']
class StatusViewSet(viewsets.ModelViewSet):
queryset = Status.objects.all()
serializer_class = serializers.StatusSerializer
serializer_class = serializers.StatSerializer
session_serializer_class = serializers.SessionSerializer
task_serializer_class = serializers.TaskSerializer

View File

@@ -11,8 +11,8 @@ from django_filters import utils
from terminal import const
from common.const.http import GET
from terminal.filters import CommandStorageFilter, CommandFilter, CommandFilterForStorageTree
from ..models import CommandStorage, ReplayStorage
from ..serializers import CommandStorageSerializer, ReplayStorageSerializer
from terminal.models import CommandStorage, ReplayStorage
from terminal.serializers import CommandStorageSerializer, ReplayStorageSerializer
__all__ = [
'CommandStorageViewSet', 'CommandStorageTestConnectiveApi',
@@ -61,7 +61,7 @@ class CommandStorageViewSet(BaseStorageViewSetMixin, viewsets.ModelViewSet):
if not filterset.is_valid():
raise utils.translate_validation(filterset.errors)
command_qs = filterset.qs
if storage.type == const.CommandStorageTypeChoices.es:
if storage.type == const.CommandStorageType.es:
command_count = command_qs.count(limit_to_max_result_window=False)
else:
command_count = command_qs.count()

View File

@@ -1,26 +1,25 @@
# -*- coding: utf-8 -*-
#
import logging
import uuid
from django.core.cache import cache
from rest_framework import generics
from rest_framework.views import APIView, Response
from rest_framework import status
from django.conf import settings
from django.utils.translation import gettext_lazy as _
from rest_framework import generics
from rest_framework import status
from rest_framework.views import APIView, Response
from common.exceptions import JMSException
from common.drf.api import JMSBulkModelViewSet
from common.utils import get_object_or_none, get_request_ip
from common.exceptions import JMSException
from common.permissions import IsValidUser
from common.permissions import WithBootstrapToken
from ..models import Terminal
from .. import serializers
from .. import exceptions
from common.utils import get_request_os
from terminal import serializers
from terminal.const import TerminalType
from terminal.models import Terminal
__all__ = [
'TerminalViewSet', 'TerminalConfig',
'TerminalRegistrationApi',
'TerminalViewSet', 'TerminalConfig',
'TerminalRegistrationApi', 'ConnectMethodListApi'
]
logger = logging.getLogger(__file__)
@@ -42,43 +41,12 @@ class TerminalViewSet(JMSBulkModelViewSet):
self.perform_destroy(instance)
return Response(status=status.HTTP_204_NO_CONTENT)
def create(self, request, *args, **kwargs):
if isinstance(request.data, list):
raise exceptions.BulkCreateNotSupport()
name = request.data.get('name')
remote_ip = request.META.get('REMOTE_ADDR')
x_real_ip = request.META.get('X-Real-IP')
remote_addr = x_real_ip or remote_ip
terminal = get_object_or_none(Terminal, name=name, is_deleted=False)
if terminal:
msg = 'Terminal name %s already used' % name
return Response({'msg': msg}, status=409)
serializer = self.serializer_class(data={
'name': name, 'remote_addr': remote_addr
})
if serializer.is_valid():
terminal = serializer.save()
# App should use id, token get access key, if accepted
token = uuid.uuid4().hex
cache.set(token, str(terminal.id), 3600)
data = {"id": str(terminal.id), "token": token, "msg": "Need accept"}
return Response(data, status=201)
else:
data = serializer.errors
logger.error("Register terminal error: {}".format(data))
return Response(data, status=400)
def filter_queryset(self, queryset):
queryset = super().filter_queryset(queryset)
s = self.request.query_params.get('status')
if not s:
return queryset
filtered_queryset_id = [str(q.id) for q in queryset if q.latest_status == s]
filtered_queryset_id = [str(q.id) for q in queryset if q.load == s]
queryset = queryset.filter(id__in=filtered_queryset_id)
return queryset
@@ -103,3 +71,16 @@ class TerminalRegistrationApi(generics.CreateAPIView):
data = {"error": "service account registration disabled"}
return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
return super().create(request, *args, **kwargs)
class ConnectMethodListApi(generics.ListAPIView):
serializer_class = serializers.ConnectMethodSerializer
permission_classes = [IsValidUser]
def get_queryset(self):
os = get_request_os(self.request)
return TerminalType.get_protocols_connect_methods(os)
def list(self, request, *args, **kwargs):
queryset = self.get_queryset()
return Response(queryset)

View File

@@ -0,0 +1,4 @@
from .session import *
from .sharing import *
from .command import *
from .task import *

View File

@@ -13,11 +13,11 @@ from common.drf.api import JMSBulkModelViewSet
from common.utils import get_logger
from terminal.backends.command.serializers import InsecureCommandAlertSerializer
from terminal.exceptions import StorageInvalid
from ..backends import (
from terminal.backends import (
get_command_storage, get_multi_command_storage,
SessionCommandSerializer,
)
from ..notifications import CommandAlertMessage
from terminal.notifications import CommandAlertMessage
logger = get_logger(__name__)
__all__ = ['CommandViewSet', 'InsecureCommandAlertAPI']
@@ -26,7 +26,7 @@ __all__ = ['CommandViewSet', 'InsecureCommandAlertAPI']
class CommandQueryMixin:
command_store = get_command_storage()
filterset_fields = [
"asset", "system_user", "user", "session",
"asset", "account", "user", "session",
"risk_level", "input"
]
default_days_ago = 5
@@ -56,7 +56,7 @@ class CommandQueryMixin:
multi_command_storage = get_multi_command_storage()
queryset = multi_command_storage.filter(
date_from=date_from, date_to=date_to,
user=q.get("user"), asset=q.get("asset"), system_user=q.get("system_user"),
user=q.get("user"), asset=q.get("asset"), account=q.get("account"),
input=q.get("input"), session=q.get("session_id", q.get('session')),
risk_level=self.get_query_risk_level(), org_id=self.get_org_id(),
)
@@ -91,7 +91,7 @@ class CommandViewSet(JMSBulkModelViewSet):
{
"user": "admin",
"asset": "localhost",
"system_user": "web",
"account": "web",
"session": "xxxxxx",
"input": "whoami",
"output": "d2hvbWFp", # base64.b64encode(s)

View File

@@ -3,43 +3,44 @@
import os
import tarfile
from django.db.models import F
from django.shortcuts import get_object_or_404, reverse
from django.utils.translation import ugettext as _
from django.utils.encoding import escape_uri_path
from django.http import FileResponse
from django.core.files.storage import default_storage
from django.db.models import F
from django.http import FileResponse
from django.shortcuts import get_object_or_404, reverse
from django.utils.encoding import escape_uri_path
from django.utils.translation import ugettext as _
from rest_framework import generics
from rest_framework import viewsets, views
from rest_framework.response import Response
from rest_framework.decorators import action
from rest_framework.permissions import IsAuthenticated
from rest_framework import generics
from rest_framework.response import Response
from common.utils import data_to_json
from common.const.http import GET
from common.utils import get_logger, get_object_or_none
from common.mixins.api import AsyncApiMixin
from common.drf.filters import DatetimeRangeFilter
from common.drf.renders import PassthroughRenderer
from common.mixins.api import AsyncApiMixin
from common.utils import data_to_json
from common.utils import get_logger, get_object_or_none
from orgs.mixins.api import OrgBulkModelViewSet
from orgs.utils import tmp_to_root_org, tmp_to_org
from terminal import serializers
from terminal.models import Session
from terminal.utils import (
find_session_replay_local, download_session_replay,
is_session_approver, get_session_replay_url
)
from users.models import User
from .. import utils
from ..utils import find_session_replay_local, download_session_replay
from ..models import Session
from .. import serializers
from terminal.utils import is_session_approver
__all__ = [
'SessionViewSet', 'SessionReplayViewSet', 'SessionJoinValidateAPI',
'MySessionAPIView',
'SessionViewSet', 'SessionReplayViewSet',
'SessionJoinValidateAPI', 'MySessionAPIView',
]
logger = get_logger(__name__)
class MySessionAPIView(generics.ListAPIView):
permission_classes = (IsAuthenticated, )
permission_classes = (IsAuthenticated,)
serializer_class = serializers.SessionSerializer
def get_queryset(self):
@@ -55,7 +56,7 @@ class SessionViewSet(OrgBulkModelViewSet):
'display': serializers.SessionDisplaySerializer,
}
search_fields = [
"user", "asset", "system_user", "remote_addr",
"user", "asset", "account", "remote_addr",
"protocol", "is_finished", 'login_from',
]
filterset_fields = search_fields + ['terminal']
@@ -93,7 +94,7 @@ class SessionViewSet(OrgBulkModelViewSet):
url_name='replay-download')
def download(self, request, *args, **kwargs):
session = self.get_object()
local_path, url = utils.get_session_replay_url(session)
local_path, url = get_session_replay_url(session)
if local_path is None:
return Response({"error": url}, status=404)
file = self.prepare_offline_file(session, local_path)
@@ -108,7 +109,7 @@ class SessionViewSet(OrgBulkModelViewSet):
return response
def get_queryset(self):
queryset = super().get_queryset().prefetch_related('terminal')\
queryset = super().get_queryset().prefetch_related('terminal') \
.annotate(terminal_display=F('terminal__name'))
return queryset
@@ -168,7 +169,7 @@ class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet):
data = {
'type': tp, 'src': url,
'user': session.user, 'asset': session.asset,
'system_user': session.system_user,
'system_user': session.account,
'date_start': session.date_start,
'date_end': session.date_end,
'download_url': download_url,

View File

@@ -5,9 +5,8 @@ from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from common.const.http import PATCH
from common.permissions import IsValidUser
from orgs.mixins.api import OrgModelViewSet
from .. import serializers, models
from terminal import serializers, models
__all__ = ['SessionSharingViewSet', 'SessionJoinRecordsViewSet']

View File

@@ -7,10 +7,10 @@ from rest_framework import status
from rest_framework.permissions import IsAuthenticated
from common.utils import get_object_or_none
from ..models import Session, Task
from .. import serializers
from terminal.utils import is_session_approver
from orgs.utils import tmp_to_root_org
from terminal.models import Session, Task
from terminal import serializers
from terminal.utils import is_session_approver
__all__ = ['TaskViewSet', 'KillSessionAPI', 'KillSessionForTicketAPI']
logger = logging.getLogger(__file__)