Compare commits

..

1 Commits

Author SHA1 Message Date
fit2bot
7d3eb8c75d feat: Update v3.0.0 2023-02-23 21:14:21 +08:00
28 changed files with 65 additions and 172 deletions

View File

@@ -24,7 +24,6 @@ ARG DEPENDENCIES=" \
libjpeg-dev \
libldap2-dev \
libsasl2-dev \
libssl-dev \
libxml2-dev \
libxmlsec1-dev \
libxmlsec1-openssl \

2
GITSHA
View File

@@ -1 +1 @@
86dab4fc6ea6b683efbe384a0694af4edb9f6716
19276e6bd4bf7bcc805351c89c90e5af8eafcca9

View File

@@ -6,7 +6,7 @@ from rest_framework.response import Response
from accounts import serializers
from accounts.filters import AccountFilterSet
from accounts.models import Account
from assets.models import Asset, Node
from assets.models import Asset
from common.permissions import UserConfirmation, ConfirmType
from common.views.mixins import RecordViewLogMixin
from orgs.mixins.api import OrgBulkModelViewSet
@@ -28,7 +28,6 @@ class AccountViewSet(OrgBulkModelViewSet):
rbac_perms = {
'partial_update': ['accounts.change_account'],
'su_from_accounts': 'accounts.view_account',
'username_suggestions': 'accounts.view_account',
}
@action(methods=['get'], detail=False, url_path='su-from-accounts')
@@ -48,29 +47,6 @@ class AccountViewSet(OrgBulkModelViewSet):
serializer = serializers.AccountSerializer(accounts, many=True)
return Response(data=serializer.data)
@action(methods=['get'], detail=False, url_path='username-suggestions')
def username_suggestions(self, request, *args, **kwargs):
asset_ids = request.query_params.get('assets')
node_keys = request.query_params.get('keys')
username = request.query_params.get('username')
assets = Asset.objects.all()
if asset_ids:
assets = assets.filter(id__in=asset_ids.split(','))
if node_keys:
patten = Node.get_node_all_children_key_pattern(node_keys.split(','))
assets = assets.filter(nodes__key__regex=patten)
accounts = Account.objects.filter(asset__in=assets)
if username:
accounts = accounts.filter(username__icontains=username)
usernames = list(accounts.values_list('username', flat=True).distinct()[:10])
usernames.sort()
common = [i for i in usernames if i in usernames if i.lower() in ['root', 'admin', 'administrator']]
others = [i for i in usernames if i not in common]
usernames = common + others
return Response(data=usernames)
class AccountSecretsViewSet(RecordViewLogMixin, AccountViewSet):
"""

View File

@@ -206,7 +206,7 @@ class ChangeSecretManager(AccountBasePlaybookManager):
serializer = serializer_cls(recorders, many=True)
header = [str(v.label) for v in serializer.child.fields.values()]
rows = [[str(i) for i in row.values()] for row in serializer.data]
rows = [list(row.values()) for row in serializer.data]
if not rows:
return False

View File

@@ -8,7 +8,7 @@ from orgs.utils import tmp_to_org, tmp_to_root_org
logger = get_logger(__file__)
def task_activity_callback(self, pid, trigger, tp, *args, **kwargs):
def task_activity_callback(self, pid, trigger, tp):
model = AutomationTypes.get_type_model(tp)
with tmp_to_root_org():
instance = get_object_or_none(model, pk=pid)

View File

@@ -9,7 +9,7 @@ from orgs.utils import tmp_to_org, tmp_to_root_org
logger = get_logger(__file__)
def task_activity_callback(self, pid, trigger, *args, **kwargs):
def task_activity_callback(self, pid, trigger):
from accounts.models import AccountBackupAutomation
with tmp_to_root_org():
plan = get_object_or_none(AccountBackupAutomation, pk=pid)

View File

@@ -27,7 +27,7 @@ def gather_asset_accounts_util(nodes, task_name):
@shared_task(
queue="ansible", verbose_name=_('Gather asset accounts'),
activity_callback=lambda self, node_ids, task_name=None, *args, **kwargs: (node_ids, None)
activity_callback=lambda self, node_ids, task_name=None: (node_ids, None)
)
def gather_asset_accounts_task(node_ids, task_name=None):
if task_name is None:

View File

@@ -13,7 +13,7 @@ __all__ = [
@shared_task(
queue="ansible", verbose_name=_('Push accounts to assets'),
activity_callback=lambda self, account_ids, *args, **kwargs: (account_ids, None)
activity_callback=lambda self, account_ids, asset_ids: (account_ids, None)
)
def push_accounts_to_assets_task(account_ids):
from accounts.models import PushAccountAutomation

View File

@@ -99,14 +99,13 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet):
("platform", serializers.PlatformSerializer),
("suggestion", serializers.MiniAssetSerializer),
("gateways", serializers.GatewaySerializer),
("spec_info", serializers.SpecSerializer),
("spec_info", serializers.SpecSerializer)
)
rbac_perms = (
("match", "assets.match_asset"),
("platform", "assets.view_platform"),
("gateways", "assets.view_gateway"),
("spec_info", "assets.view_asset"),
("info", "assets.view_asset"),
)
extra_filter_backends = [LabelFilterBackend, IpInFilterBackend, NodeFilterBackend]

View File

@@ -21,10 +21,4 @@ class HostViewSet(AssetViewSet):
@action(methods=["GET"], detail=True, url_path="info")
def info(self, *args, **kwargs):
asset = super().get_object()
serializer = self.get_serializer(asset.info)
data = serializer.data
data['asset'] = {
'id': asset.id, 'name': asset.name,
'address': asset.address
}
return Response(data)
return Response(asset.info)

View File

@@ -93,7 +93,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='asset',
name='address',
field=models.CharField(db_index=True, max_length=767, verbose_name='Address'),
field=models.CharField(db_index=True, max_length=1024, verbose_name='Address'),
),
migrations.AddField(
model_name='asset',

View File

@@ -1,15 +1,12 @@
# Generated by Django 3.2.12 on 2022-07-11 06:13
import time
from django.utils import timezone
from itertools import groupby
from django.db import migrations
def migrate_asset_accounts(apps, schema_editor):
auth_book_model = apps.get_model('assets', 'AuthBook')
account_model = apps.get_model('accounts', 'Account')
account_history_model = apps.get_model('accounts', 'HistoricalAccount')
count = 0
bulk_size = 1000
@@ -23,35 +20,34 @@ def migrate_asset_accounts(apps, schema_editor):
break
count += len(auth_books)
accounts = []
# auth book 和 account 相同的属性
same_attrs = [
'id', 'username', 'comment', 'date_created', 'date_updated',
'created_by', 'asset_id', 'org_id',
]
# 认证的属性,可能是 auth_book 的,可能是 system_user 的
# 认证的属性,可能是 authbook 的,可能是 systemuser 的
auth_attrs = ['password', 'private_key', 'token']
all_attrs = same_attrs + auth_attrs
accounts = []
for auth_book in auth_books:
account_values = {'version': 1}
values = {'version': 1}
system_user = auth_book.systemuser
if system_user:
# 更新一次系统用户的认证属性
account_values.update({attr: getattr(system_user, attr, '') for attr in all_attrs})
account_values['created_by'] = str(system_user.id)
account_values['privileged'] = system_user.type == 'admin' \
or system_user.username in ['root', 'Administrator']
values.update({attr: getattr(system_user, attr, '') for attr in all_attrs})
values['created_by'] = str(system_user.id)
values['privileged'] = system_user.type == 'admin'
auth_book_auth = {attr: getattr(auth_book, attr, '') for attr in all_attrs if getattr(auth_book, attr, '')}
# 最终优先使用 auth_book 的认证属性
account_values.update(auth_book_auth)
# 最终使用 authbook 的认证属性
values.update(auth_book_auth)
auth_infos = []
username = account_values['username']
username = values['username']
for attr in auth_attrs:
secret = account_values.pop(attr, None)
secret = values.pop(attr, None)
if not secret:
continue
@@ -70,48 +66,13 @@ def migrate_asset_accounts(apps, schema_editor):
auth_infos.append((username, 'password', ''))
for name, secret_type, secret in auth_infos:
if not name:
continue
account = account_model(**account_values, name=name, secret=secret, secret_type=secret_type)
account = account_model(**values, name=name, secret=secret, secret_type=secret_type)
accounts.append(account)
accounts.sort(key=lambda x: (x.name, x.asset_id, x.date_updated))
grouped_accounts = groupby(accounts, lambda x: (x.name, x.asset_id))
accounts_to_add = []
accounts_to_history = []
for key, _accounts in grouped_accounts:
_accounts = list(_accounts)
if not _accounts:
continue
_account = _accounts[-1]
accounts_to_add.append(_account)
_account_history = []
for ac in _accounts:
if not ac.secret:
continue
if ac.id != _account.id and ac.secret == _account.secret:
continue
history_data = {
'id': _account.id,
'secret': ac.secret,
'secret_type': ac.secret_type,
'history_date': ac.date_updated,
'history_type': '~',
'history_change_reason': 'from account {}'.format(_account.name),
}
_account_history.append(account_history_model(**history_data))
_account.version = len(_account_history)
accounts_to_history.extend(_account_history)
account_model.objects.bulk_create(accounts_to_add, ignore_conflicts=True)
account_history_model.objects.bulk_create(accounts_to_history, ignore_conflicts=True)
account_model.objects.bulk_create(accounts, ignore_conflicts=True)
print("\t - Create asset accounts: {}-{} using: {:.2f}s".format(
count - len(auth_books), count, time.time() - start
))
print("\t - accounts: {}".format(len(accounts_to_add)))
print("\t - histories: {}".format(len(accounts_to_history)))
def migrate_db_accounts(apps, schema_editor):
@@ -169,9 +130,6 @@ def migrate_db_accounts(apps, schema_editor):
values['secret_type'] = secret_type
values['secret'] = secret
if not name:
continue
for app in apps:
values['asset_id'] = str(app.id)
account = account_model(**values)

View File

@@ -100,7 +100,7 @@ class Asset(NodesRelationMixin, AbsConnectivity, JMSOrgBaseModel):
Type = const.AllTypes
name = models.CharField(max_length=128, verbose_name=_('Name'))
address = models.CharField(max_length=767, verbose_name=_('Address'), db_index=True)
address = models.CharField(max_length=1024, verbose_name=_('Address'), db_index=True)
platform = models.ForeignKey(Platform, on_delete=models.PROTECT, verbose_name=_("Platform"), related_name='assets')
domain = models.ForeignKey("assets.Domain", null=True, blank=True, related_name='assets',
verbose_name=_("Domain"), on_delete=models.SET_NULL)

View File

@@ -489,7 +489,7 @@ class SomeNodesMixin:
return cls.default_node()
if ori_org and ori_org.is_root():
return cls.default_node()
return None
org_roots = cls.org_root_nodes()
org_roots_length = len(org_roots)

View File

@@ -8,7 +8,7 @@ from orgs.utils import tmp_to_root_org, tmp_to_org
logger = get_logger(__file__)
def task_activity_callback(self, pid, trigger, tp, *args, **kwargs):
def task_activity_callback(self, pid, trigger, tp):
model = AutomationTypes.get_type_model(tp)
with tmp_to_root_org():
instance = get_object_or_none(model, pk=pid)

View File

@@ -225,7 +225,6 @@ class MFAMixin:
self.request.session['auth_mfa_time'] = time.time()
self.request.session['auth_mfa_required'] = 0
self.request.session['auth_mfa_type'] = mfa_type
MFABlockUtils(self.request.user.username, self.get_request_ip()).clean_failed_count()
def clean_mfa_mark(self):
keys = ['auth_mfa', 'auth_mfa_time', 'auth_mfa_required', 'auth_mfa_type']

View File

@@ -32,14 +32,11 @@ class UserLoginMFAView(mixins.AuthMixin, FormView):
return super().get(*args, **kwargs)
def form_valid(self, form):
from users.utils import MFABlockUtils
code = form.cleaned_data.get('code')
mfa_type = form.cleaned_data.get('mfa_type')
try:
self._do_check_user_mfa(code, mfa_type)
user, ip = self.get_user_from_session(), self.get_request_ip()
MFABlockUtils(user.username, ip).clean_failed_count()
return redirect_to_guard_view('mfa_ok')
except (errors.MFAFailedError, errors.BlockMFAError) as e:
form.add_error('code', e.msg)

View File

@@ -10,7 +10,7 @@ from .utils import get_logger
logger = get_logger(__file__)
def task_activity_callback(self, subject, message, recipient_list, *args, **kwargs):
def task_activity_callback(self, subject, message, recipient_list, **kwargs):
from users.models import User
email_list = recipient_list
resource_ids = list(User.objects.filter(email__in=email_list).values_list('id', flat=True))

View File

@@ -3,10 +3,8 @@
from celery import shared_task
from celery.exceptions import SoftTimeLimitExceeded
from django.utils.translation import ugettext_lazy as _
from django_celery_beat.models import PeriodicTask
from common.utils import get_logger, get_object_or_none
from ops.celery import app
from orgs.utils import tmp_to_org, tmp_to_root_org
from .celery.decorator import (
register_as_period_task, after_app_ready_start
@@ -21,7 +19,7 @@ from .notifications import ServerPerformanceCheckUtil
logger = get_logger(__file__)
def job_task_activity_callback(self, job_id, *args, **kwargs):
def job_task_activity_callback(self, job_id, trigger):
job = get_object_or_none(Job, id=job_id)
if not job:
return
@@ -50,7 +48,7 @@ def run_ops_job(job_id):
logger.error("Start adhoc execution error: {}".format(e))
def job_execution_task_activity_callback(self, execution_id, *args, **kwargs):
def job_execution_task_activity_callback(self, execution_id, trigger):
execution = get_object_or_none(JobExecution, id=execution_id)
if not execution:
return
@@ -80,14 +78,16 @@ def run_ops_job_execution(execution_id, **kwargs):
@after_app_ready_start
def clean_celery_periodic_tasks():
"""清除celery定时任务"""
logger.info('Start clean celery periodic tasks.')
register_tasks = PeriodicTask.objects.all()
for task in register_tasks:
if task.task in app.tasks:
continue
task_name = task.name
need_cleaned_tasks = [
'handle_be_interrupted_change_auth_task_periodic',
]
logger.info('Start clean celery periodic tasks: {}'.format(need_cleaned_tasks))
for task_name in need_cleaned_tasks:
logger.info('Start clean task: {}'.format(task_name))
task = get_celery_periodic_task(task_name)
if task is None:
logger.info('Task does not exist: {}'.format(task_name))
continue
disable_celery_periodic_task(task_name)
delete_celery_periodic_task(task_name)
task = get_celery_periodic_task(task_name)

View File

@@ -114,7 +114,9 @@ class OrgResourceStatisticsCache(OrgRelatedCache):
@staticmethod
def compute_total_count_today_active_assets():
t = local_zero_hour()
return Session.objects.filter(date_start__gte=t).values('asset_id').distinct().count()
return Session.objects.filter(
date_start__gte=t, is_success=False
).values('asset_id').distinct().count()
@staticmethod
def compute_total_count_today_failed_sessions():

View File

@@ -102,10 +102,7 @@ def on_post_delete_refresh_org_resource_statistics_cache(sender, instance, **kwa
def _refresh_session_org_resource_statistics_cache(instance: Session):
cache_field_name = [
'total_count_online_users', 'total_count_online_sessions',
'total_count_today_active_assets','total_count_today_failed_sessions'
]
cache_field_name = ['total_count_online_users', 'total_count_online_sessions', 'total_count_today_failed_sessions']
org_cache = OrgResourceStatisticsCache(instance.org)
org_cache.expire(*cache_field_name)

View File

@@ -30,12 +30,6 @@ class BaseUserPermedAssetsApi(SelfOrPKUserMixin, ListAPIView):
filterset_class = AssetFilterSet
serializer_class = serializers.AssetPermedSerializer
def get_serializer_class(self):
serializer_class = super().get_serializer_class()
if self.request.query_params.get('id'):
serializer_class = serializers.AssetPermedDetailSerializer
return serializer_class
def get_queryset(self):
if getattr(self, 'swagger_fake_view', False):
return Asset.objects.none()

View File

@@ -15,7 +15,7 @@ from perms.serializers.permission import ActionChoicesField
__all__ = [
'NodePermedSerializer', 'AssetPermedSerializer',
'AssetPermedDetailSerializer', 'AccountsPermedSerializer'
'AccountsPermedSerializer'
]
@@ -46,12 +46,6 @@ class AssetPermedSerializer(OrgResourceModelSerializerMixin):
return queryset
class AssetPermedDetailSerializer(AssetPermedSerializer):
class Meta(AssetPermedSerializer.Meta):
fields = AssetPermedSerializer.Meta.fields + ['spec_info']
read_only_fields = fields
class NodePermedSerializer(serializers.ModelSerializer):
class Meta:
model = Node

View File

@@ -1,6 +1,5 @@
from collections import defaultdict
from orgs.utils import tmp_to_org
from accounts.models import Account
from accounts.const import AliasAccount
from .permission import AssetPermissionUtil
@@ -17,11 +16,10 @@ class PermAccountUtil(AssetPermissionUtil):
:param asset: Asset
:param account_name: 可能是 @USER @INPUT 字符串
"""
with tmp_to_org(asset.org):
permed_accounts = self.get_permed_accounts_for_user(user, asset)
accounts_mapper = {account.alias: account for account in permed_accounts}
account = accounts_mapper.get(account_name)
return account
permed_accounts = self.get_permed_accounts_for_user(user, asset)
accounts_mapper = {account.alias: account for account in permed_accounts}
account = accounts_mapper.get(account_name)
return account
def get_permed_accounts_for_user(self, user, asset):
""" 获取授权给用户某个资产的账号 """

View File

@@ -18,18 +18,14 @@ user_perms = (
('assets', 'asset', 'match', 'asset'),
('assets', 'systemuser', 'match', 'systemuser'),
('assets', 'node', 'match', 'node'),
("ops", "adhoc", "*", "*"),
("ops", "playbook", "*", "*"),
("ops", "job", "*", "*"),
("ops", "jobexecution", "*", "*"),
)
system_user_perms = (
('authentication', 'connectiontoken', 'add,change,view', 'connectiontoken'),
('authentication', 'temptoken', 'add,change,view', 'temptoken'),
('authentication', 'accesskey', '*', '*'),
('tickets', 'ticket', 'view', 'ticket'),
) + user_perms + _view_all_joined_org_perms
('authentication', 'connectiontoken', 'add,change,view', 'connectiontoken'),
('authentication', 'temptoken', 'add,change,view', 'temptoken'),
('authentication', 'accesskey', '*', '*'),
('tickets', 'ticket', 'view', 'ticket'),
) + user_perms + _view_all_joined_org_perms
_auditor_perms = (
('rbac', 'menupermission', 'view', 'audit'),
@@ -45,6 +41,7 @@ auditor_perms = user_perms + _auditor_perms
system_auditor_perms = system_user_perms + _auditor_perms + _view_root_perms
app_exclude_perms = [
('users', 'user', 'add,delete', 'user'),
('orgs', 'org', 'add,delete,change', 'org'),

View File

@@ -97,13 +97,13 @@ class RBACPermission(permissions.DjangoModelPermissions):
else:
model_cls = queryset.model
except AssertionError as e:
# logger.error(f'Error get model cls: {e}')
logger.error(f'Error get model cls: {e}')
model_cls = None
except AttributeError as e:
# logger.error(f'Error get model cls: {e}')
logger.error(f'Error get model cls: {e}')
model_cls = None
except Exception as e:
# logger.error('Error get model class: {} of {}'.format(e, view))
logger.error('Error get model class: {} of {}'.format(e, view))
raise e
return model_cls

View File

@@ -17,17 +17,17 @@ class WebMethod(TextChoices):
@classmethod
def get_methods(cls):
methods = {
return {
Protocol.ssh: [cls.web_cli, cls.web_sftp],
Protocol.telnet: [cls.web_cli],
Protocol.rdp: [cls.web_gui],
Protocol.vnc: [cls.web_gui],
Protocol.mysql: [cls.web_cli],
Protocol.mariadb: [cls.web_cli],
Protocol.oracle: [cls.web_cli],
Protocol.postgresql: [cls.web_cli],
Protocol.sqlserver: [cls.web_cli],
Protocol.mysql: [cls.web_cli, cls.web_gui],
Protocol.mariadb: [cls.web_cli, cls.web_gui],
Protocol.oracle: [cls.web_cli, cls.web_gui],
Protocol.postgresql: [cls.web_cli, cls.web_gui],
Protocol.sqlserver: [cls.web_cli, cls.web_gui],
Protocol.redis: [cls.web_cli],
Protocol.mongodb: [cls.web_cli],
Protocol.clickhouse: [cls.web_cli],
@@ -35,13 +35,6 @@ class WebMethod(TextChoices):
Protocol.k8s: [cls.web_cli],
Protocol.http: []
}
if not settings.XPACK_ENABLED:
return methods
web_gui_dbs = [Protocol.mysql, Protocol.mariadb, Protocol.oracle, Protocol.postgresql]
for db in web_gui_dbs:
methods[db].append(cls.web_gui)
return methods
class NativeClient(TextChoices):
@@ -137,6 +130,8 @@ class AppletMethod:
from .models import Applet, AppletHost
methods = defaultdict(list)
if not settings.XPACK_ENABLED:
return methods
has_applet_hosts = AppletHost.objects.all().exists()
applets = Applet.objects.filter(is_active=True)

View File

@@ -80,20 +80,14 @@ def upload_session_replay_to_external_storage(session_id):
return
@shared_task(
verbose_name=_('Run applet host deployment'),
activity_callback=lambda self, did, *args, **kwargs: ([did], )
)
@shared_task(verbose_name=_('Run applet host deployment'), activity_callback=lambda did: ([did], ))
def run_applet_host_deployment(did):
with tmp_to_builtin_org(system=1):
deployment = AppletHostDeployment.objects.get(id=did)
deployment.start()
@shared_task(
verbose_name=_('Install applet'),
activity_callback=lambda self, did, applet_id, *args, **kwargs: ([did],)
)
@shared_task(verbose_name=_('Install applet'), activity_callback=lambda did, applet_id: ([did],))
def run_applet_host_deployment_install_applet(did, applet_id):
with tmp_to_builtin_org(system=1):
deployment = AppletHostDeployment.objects.get(id=did)