diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 11529202f..68c23301d 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -273,6 +273,7 @@ class Config(dict): 'SESSION_COOKIE_SECURE': False, 'CSRF_COOKIE_SECURE': False, 'REFERER_CHECK_ENABLED': False, + 'SERVER_REPLAY_STORAGE': {} } def compatible_auth_openid_of_key(self): diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index cdea6c52d..97479115f 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -12,6 +12,17 @@ DEFAULT_TERMINAL_COMMAND_STORAGE = { }, } TERMINAL_COMMAND_STORAGE = DYNAMIC.TERMINAL_COMMAND_STORAGE or {} + +# Server 类型的录像存储 +SERVER_REPLAY_STORAGE = CONFIG.SERVER_REPLAY_STORAGE +# SERVER_REPLAY_STORAGE = { +# 'TYPE': 's3', +# 'BUCKET': '', +# 'ACCESS_KEY': '', +# 'SECRET_KEY': '', +# 'ENDPOINT': '' +# } + DEFAULT_TERMINAL_REPLAY_STORAGE = { "default": { "TYPE": "server", diff --git a/apps/terminal/api/session.py b/apps/terminal/api/session.py index bf0a69789..16033b390 100644 --- a/apps/terminal/api/session.py +++ b/apps/terminal/api/session.py @@ -125,7 +125,7 @@ class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet): if serializer.is_valid(): file = serializer.validated_data['file'] - name, err = session.save_to_storage(file) + name, err = session.save_replay_to_storage(file) if not name: msg = "Failed save replay `{}`: {}".format(session_id, err) logger.error(msg) @@ -155,6 +155,7 @@ class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet): return data def is_need_async(self): + return False if self.action != 'retrieve': return False return True diff --git a/apps/terminal/backends/__init__.py b/apps/terminal/backends/__init__.py index b2a9c557c..52a33b359 100644 --- a/apps/terminal/backends/__init__.py +++ b/apps/terminal/backends/__init__.py @@ -1,5 +1,7 @@ from importlib import import_module from django.conf import settings +from django.utils.functional import LazyObject + from .command.serializers import SessionCommandSerializer from ..const import COMMAND_STORAGE_TYPE_SERVER @@ -17,6 +19,13 @@ def get_command_storage(): return storage +def get_server_replay_storage(): + from jms_storage import get_object_storage + config = settings.SERVER_REPLAY_STORAGE + storage = get_object_storage(config) + return storage + + def get_terminal_command_storages(): from ..models import CommandStorage storage_list = {} @@ -40,3 +49,15 @@ def get_multi_command_storage(): return storage +class ServerReplayStorage(LazyObject): + def _setup(self): + self._wrapped = get_server_replay_storage() + + +class ServerCommandStorage(LazyObject): + def _setup(self): + self._wrapped = get_command_storage() + + +server_command_storage = ServerCommandStorage() +server_replay_storage = ServerReplayStorage() diff --git a/apps/terminal/backends/replay/__init__.py b/apps/terminal/backends/replay/__init__.py new file mode 100644 index 000000000..b801d311e --- /dev/null +++ b/apps/terminal/backends/replay/__init__.py @@ -0,0 +1,3 @@ +""" +Replay 的 backend 已移动到 jms_storage 模块中 +""" \ No newline at end of file diff --git a/apps/terminal/models.py b/apps/terminal/models.py index 6fdf8e349..f4b43291f 100644 --- a/apps/terminal/models.py +++ b/apps/terminal/models.py @@ -253,14 +253,18 @@ class Session(OrgModelMixin): return False return True - def save_to_storage(self, f): + def save_replay_to_storage(self, f): local_path = self.get_local_path() try: name = default_storage.save(local_path, f) - return name, None except OSError as e: return None, e + if settings.SERVER_REPLAY_STORAGE: + from .tasks import upload_session_replay_to_external_storage + upload_session_replay_to_external_storage.delay(str(self.id)) + return name, None + @classmethod def set_sessions_active(cls, sessions_id): data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in sessions_id} @@ -456,4 +460,3 @@ class ReplayStorage(CommonModelMixin): def can_delete(self): return not self.in_defaults() - diff --git a/apps/terminal/tasks.py b/apps/terminal/tasks.py index ffe3347c2..b1342543d 100644 --- a/apps/terminal/tasks.py +++ b/apps/terminal/tasks.py @@ -9,11 +9,12 @@ from django.utils import timezone from django.conf import settings from django.core.files.storage import default_storage - from ops.celery.decorator import ( register_as_period_task, after_app_ready_start, after_app_shutdown_clean_periodic ) from .models import Status, Session, Command +from .backends import server_replay_storage +from .utils import find_session_replay_local CACHE_REFRESH_INTERVAL = 10 @@ -68,3 +69,26 @@ def clean_expired_session_period(): # 删除session记录 session.delete() + +@shared_task +def upload_session_replay_to_external_storage(session_id): + logger.info(f'Start upload session to external storage: {session_id}') + session = Session.objects.filter(id=session_id).first() + if not session: + logger.error(f'Session db item not found: {session_id}') + return + local_path, foobar = find_session_replay_local(session) + if not local_path: + logger.error(f'Session replay not found, may be upload error: {local_path}') + return + abs_path = default_storage.path(local_path) + remote_path = session.get_rel_replay_path() + ok, err = server_replay_storage.upload(abs_path, remote_path) + if not ok: + logger.error(f'Session replay upload to external error: {err}') + return + try: + default_storage.delete(local_path) + except: + pass + return diff --git a/apps/terminal/utils.py b/apps/terminal/utils.py index 2ea2768bc..f48823f16 100644 --- a/apps/terminal/utils.py +++ b/apps/terminal/utils.py @@ -2,43 +2,18 @@ # import os -from django.core.cache import cache +from django.conf import settings from django.core.files.storage import default_storage import jms_storage -from assets.models import Asset, SystemUser -from users.models import User from common.utils import get_logger -from .const import USERS_CACHE_KEY, ASSETS_CACHE_KEY, SYSTEM_USER_CACHE_KEY + +from .backends import server_replay_storage from .models import ReplayStorage logger = get_logger(__name__) -def get_session_asset_list(): - return Asset.objects.values_list('hostname', flat=True) - - -def get_session_user_list(): - return User.objects.exclude(role=User.ROLE.APP).values_list('username', flat=True) - - -def get_session_system_user_list(): - return SystemUser.objects.values_list('username', flat=True) - - -def get_user_list_from_cache(): - return cache.get(USERS_CACHE_KEY) - - -def get_asset_list_from_cache(): - return cache.get(ASSETS_CACHE_KEY) - - -def get_system_user_list_from_cache(): - return cache.get(SYSTEM_USER_CACHE_KEY) - - def find_session_replay_local(session): # 新版本和老版本的文件后缀不同 session_path = session.get_rel_replay_path() # 存在外部存储上的路径 @@ -62,6 +37,8 @@ def download_session_replay(session): for storage in replay_storages if not storage.in_defaults() } + if settings.SERVER_REPLAY_STORAGE: + configs['SERVER_REPLAY_STORAGE'] = settings.SERVER_REPLAY_STORAGE if not configs: msg = "Not found replay file, and not remote storage set" return None, msg