diff --git a/apps/accounts/api/account/account.py b/apps/accounts/api/account/account.py index 94ee9086b..573f731cc 100644 --- a/apps/accounts/api/account/account.py +++ b/apps/accounts/api/account/account.py @@ -26,6 +26,7 @@ class AccountViewSet(OrgBulkModelViewSet): filterset_class = AccountFilterSet serializer_classes = { 'default': serializers.AccountSerializer, + 'retrieve': serializers.AccountDetailSerializer, } rbac_perms = { 'partial_update': ['accounts.change_account'], @@ -133,11 +134,13 @@ class AccountHistoriesSecretAPI(ExtraFilterFieldsMixin, RecordViewLogMixin, List def get_queryset(self): account = self.get_object() histories = account.history.all() - last_history = account.history.first() - if not last_history: + latest_history = account.history.first() + if not latest_history: return histories - - if account.secret == last_history.secret \ - and account.secret_type == last_history.secret_type: - histories = histories.exclude(history_id=last_history.history_id) + if account.secret != latest_history.secret: + return histories + if account.secret_type != latest_history.secret_type: + return histories + histories = histories.exclude(history_id=latest_history.history_id) return histories + diff --git a/apps/accounts/backends/__init__.py b/apps/accounts/backends/__init__.py new file mode 100644 index 000000000..9bfd63dfb --- /dev/null +++ b/apps/accounts/backends/__init__.py @@ -0,0 +1,41 @@ +import os +from django.utils.functional import LazyObject +from importlib import import_module + +from common.utils import get_logger +from ..const import VaultTypeChoices + +__all__ = ['vault_client', 'get_vault_client'] + + +logger = get_logger(__file__) + + +def get_vault_client(raise_exception=False, **kwargs): + try: + tp = kwargs.get('VAULT_TYPE') + module_path = f'apps.accounts.backends.{tp}.main' + client = import_module(module_path).Vault(**kwargs) + except Exception as e: + logger.error(f'Init vault client failed: {e}') + if raise_exception: + raise + tp = VaultTypeChoices.local + module_path = f'apps.accounts.backends.{tp}.main' + kwargs['VAULT_TYPE'] = tp + client = import_module(module_path).Vault(**kwargs) + return client + + +class VaultClient(LazyObject): + + def _setup(self): + from jumpserver import settings as js_settings + from django.conf import settings + vault_config_names = [k for k in js_settings.__dict__.keys() if k.startswith('VAULT_')] + vault_configs = {name: getattr(settings, name, None) for name in vault_config_names} + self._wrapped = get_vault_client(**vault_configs) + + +""" 为了安全, 页面修改配置, 重启服务后才会重新初始化 vault_client """ +vault_client = VaultClient() diff --git a/apps/accounts/backends/base.py b/apps/accounts/backends/base.py new file mode 100644 index 000000000..12547a7cb --- /dev/null +++ b/apps/accounts/backends/base.py @@ -0,0 +1,77 @@ +from abc import ABC, abstractmethod + +from django.forms.models import model_to_dict + +__all__ = ['BaseVault'] + + +class BaseVault(ABC): + + def __init__(self, *args, **kwargs): + self.type = kwargs.get('VAULT_TYPE') + + def is_type(self, tp): + return self.type == tp + + def get(self, instance): + """ 返回 secret 值 """ + return self._get(instance) + + def create(self, instance): + if not instance.secret_has_save_to_vault: + self._create(instance) + self._clean_db_secret(instance) + self.save_metadata(instance) + + if instance.is_sync_metadata: + self.save_metadata(instance) + + def update(self, instance): + if not instance.secret_has_save_to_vault: + self._update(instance) + self._clean_db_secret(instance) + self.save_metadata(instance) + + if instance.is_sync_metadata: + self.save_metadata(instance) + + def delete(self, instance): + self._delete(instance) + + def save_metadata(self, instance): + metadata = model_to_dict(instance, fields=[ + 'name', 'username', 'secret_type', + 'connectivity', 'su_from', 'privileged' + ]) + metadata = {field: str(value) for field, value in metadata.items()} + return self._save_metadata(instance, metadata) + + # -------- abstractmethod -------- # + + @abstractmethod + def _get(self, instance): + raise NotImplementedError + + @abstractmethod + def _create(self, instance): + raise NotImplementedError + + @abstractmethod + def _update(self, instance): + raise NotImplementedError + + @abstractmethod + def _delete(self, instance): + raise NotImplementedError + + @abstractmethod + def _clean_db_secret(self, instance): + raise NotImplementedError + + @abstractmethod + def _save_metadata(self, instance, metadata): + raise NotImplementedError + + @abstractmethod + def is_active(self, *args, **kwargs) -> (bool, str): + raise NotImplementedError diff --git a/apps/accounts/backends/hcp/__init__.py b/apps/accounts/backends/hcp/__init__.py new file mode 100644 index 000000000..15b6a64ba --- /dev/null +++ b/apps/accounts/backends/hcp/__init__.py @@ -0,0 +1 @@ +from .main import * diff --git a/apps/accounts/backends/hcp/entries.py b/apps/accounts/backends/hcp/entries.py new file mode 100644 index 000000000..ff73ef98d --- /dev/null +++ b/apps/accounts/backends/hcp/entries.py @@ -0,0 +1,84 @@ +import sys +from abc import ABC + +from common.db.utils import Encryptor +from common.utils import lazyproperty + +current_module = sys.modules[__name__] + +__all__ = ['build_entry'] + + +class BaseEntry(ABC): + + def __init__(self, instance): + self.instance = instance + + @lazyproperty + def full_path(self): + path_base = self.path_base + path_spec = self.path_spec + path = f'{path_base}/{path_spec}' + return path + + @property + def path_base(self): + path = f'orgs/{self.instance.org_id}' + return path + + @property + def path_spec(self): + raise NotImplementedError + + def to_internal_data(self): + secret = getattr(self.instance, '_secret', None) + if secret is not None: + secret = Encryptor(secret).encrypt() + data = {'secret': secret} + return data + + @staticmethod + def to_external_data(data): + secret = data.pop('secret', None) + if secret is not None: + secret = Encryptor(secret).decrypt() + return secret + + +class AccountEntry(BaseEntry): + + @property + def path_spec(self): + path = f'assets/{self.instance.asset_id}/accounts/{self.instance.id}' + return path + + +class AccountTemplateEntry(BaseEntry): + + @property + def path_spec(self): + path = f'account-templates/{self.instance.id}' + return path + + +class HistoricalAccountEntry(BaseEntry): + + @property + def path_base(self): + account = self.instance.instance + path = f'accounts/{account.id}/' + return path + + @property + def path_spec(self): + path = f'histories/{self.instance.history_id}' + return path + + +def build_entry(instance) -> BaseEntry: + class_name = instance.__class__.__name__ + entry_class_name = f'{class_name}Entry' + entry_class = getattr(current_module, entry_class_name, None) + if not entry_class: + raise Exception(f'Entry class {entry_class_name} is not found') + return entry_class(instance) diff --git a/apps/accounts/backends/hcp/main.py b/apps/accounts/backends/hcp/main.py new file mode 100644 index 000000000..9fb752c1b --- /dev/null +++ b/apps/accounts/backends/hcp/main.py @@ -0,0 +1,47 @@ +from .entries import build_entry +from .service import VaultKVClient +from ..base import BaseVault + +__all__ = ['Vault'] + + +class Vault(BaseVault): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.client = VaultKVClient( + url=kwargs.get('VAULT_HCP_HOST'), + token=kwargs.get('VAULT_HCP_TOKEN'), + mount_point=kwargs.get('VAULT_HCP_MOUNT_POINT') + ) + + def is_active(self): + return self.client.is_active() + + def _get(self, instance): + entry = build_entry(instance) + # TODO: get data 是不是层数太多了 + data = self.client.get(path=entry.full_path).get('data', {}) + data = entry.to_external_data(data) + return data + + def _create(self, instance): + entry = build_entry(instance) + data = entry.to_internal_data() + self.client.create(path=entry.full_path, data=data) + + def _update(self, instance): + entry = build_entry(instance) + data = entry.to_internal_data() + self.client.patch(path=entry.full_path, data=data) + + def _delete(self, instance): + entry = build_entry(instance) + self.client.delete(path=entry.full_path) + + def _clean_db_secret(self, instance): + instance.is_sync_metadata = False + instance.mark_secret_save_to_vault() + + def _save_metadata(self, instance, metadata): + entry = build_entry(instance) + self.client.update_metadata(path=entry.full_path, metadata=metadata) diff --git a/apps/accounts/backends/hcp/service.py b/apps/accounts/backends/hcp/service.py new file mode 100644 index 000000000..179f98edb --- /dev/null +++ b/apps/accounts/backends/hcp/service.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# +import hvac +from hvac import exceptions +from requests.exceptions import ConnectionError + +from common.utils import get_logger + +logger = get_logger(__name__) + +__all__ = ['VaultKVClient'] + + +class VaultKVClient(object): + max_versions = 20 + + def __init__(self, url, token, mount_point): + assert isinstance(self.max_versions, int) and self.max_versions >= 3, ( + 'max_versions must to be an integer that is greater than or equal to 3' + ) + self.client = hvac.Client(url=url, token=token) + self.mount_point = mount_point + self.enable_secrets_engine_if_need() + + def is_active(self): + try: + if not self.client.sys.is_initialized(): + return False, 'Vault is not initialized' + if self.client.sys.is_sealed(): + return False, 'Vault is sealed' + if not self.client.is_authenticated(): + return False, 'Vault is not authenticated' + except ConnectionError as e: + logger.error(str(e)) + return False, f'Vault is not reachable: {e}' + else: + return True, '' + + def enable_secrets_engine_if_need(self): + secrets_engines = self.client.sys.list_mounted_secrets_engines() + mount_points = secrets_engines.keys() + if f'{self.mount_point}/' in mount_points: + return + self.client.sys.enable_secrets_engine( + backend_type='kv', + path=self.mount_point, + options={'version': 2} # TODO: version 是否从配置中读取? + ) + self.client.secrets.kv.v2.configure( + max_versions=self.max_versions, + mount_point=self.mount_point + ) + + def get(self, path, version=None): + try: + response = self.client.secrets.kv.v2.read_secret_version( + path=path, + version=version, + mount_point=self.mount_point + ) + except exceptions.InvalidPath as e: + logger.error('Get secret error: {}'.format(e)) + return {} + data = response.get('data', {}) + return data + + def create(self, path, data: dict): + self._update_or_create(path=path, data=data) + + def update(self, path, data: dict): + """ 未更新的数据会被删除 """ + self._update_or_create(path=path, data=data) + + def patch(self, path, data: dict): + """ 未更新的数据不会被删除 """ + self.client.secrets.kv.v2.patch( + path=path, + secret=data, + mount_point=self.mount_point + ) + + def delete(self, path): + self.client.secrets.kv.v2.delete_metadata_and_all_versions( + path=path, + mount_point=self.mount_point, + ) + + def _update_or_create(self, path, data: dict): + self.client.secrets.kv.v2.create_or_update_secret( + path=path, + secret=data, + mount_point=self.mount_point + ) + + def update_metadata(self, path, metadata: dict): + try: + self.client.secrets.kv.v2.update_metadata( + path=path, + mount_point=self.mount_point, + custom_metadata=metadata + ) + except exceptions.InvalidPath as e: + logger.error('Update metadata error: {}'.format(e)) diff --git a/apps/accounts/backends/local/__init__.py b/apps/accounts/backends/local/__init__.py new file mode 100644 index 000000000..15b6a64ba --- /dev/null +++ b/apps/accounts/backends/local/__init__.py @@ -0,0 +1 @@ +from .main import * diff --git a/apps/accounts/backends/local/main.py b/apps/accounts/backends/local/main.py new file mode 100644 index 000000000..a3e608bbc --- /dev/null +++ b/apps/accounts/backends/local/main.py @@ -0,0 +1,36 @@ +from common.utils import get_logger +from ..base import BaseVault + +logger = get_logger(__name__) + +__all__ = ['Vault'] + + +class Vault(BaseVault): + + def is_active(self): + return True, '' + + def _get(self, instance): + secret = getattr(instance, '_secret', None) + return secret + + def _create(self, instance): + """ Ignore """ + pass + + def _update(self, instance): + """ Ignore """ + pass + + def _delete(self, instance): + """ Ignore """ + pass + + def _save_metadata(self, instance, metadata): + """ Ignore """ + pass + + def _clean_db_secret(self, instance): + """ Ignore *重要* 不能删除本地 secret """ + pass diff --git a/apps/accounts/const/__init__.py b/apps/accounts/const/__init__.py index 6db502556..0e1997f00 100644 --- a/apps/accounts/const/__init__.py +++ b/apps/accounts/const/__init__.py @@ -1,2 +1,3 @@ from .account import * from .automation import * +from .vault import * diff --git a/apps/accounts/const/vault.py b/apps/accounts/const/vault.py new file mode 100644 index 000000000..8b30b8afa --- /dev/null +++ b/apps/accounts/const/vault.py @@ -0,0 +1,9 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +__all__ = ['VaultTypeChoices'] + + +class VaultTypeChoices(models.TextChoices): + local = 'local', _('Local Vault') + hcp = 'hcp', _('HCP Vault') diff --git a/apps/accounts/migrations/0012_auto_20230621_1456.py b/apps/accounts/migrations/0012_auto_20230621_1456.py new file mode 100644 index 000000000..389e2b63d --- /dev/null +++ b/apps/accounts/migrations/0012_auto_20230621_1456.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.19 on 2023-06-21 06:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0011_auto_20230506_1443'), + ] + + operations = [ + migrations.RenameField( + model_name='account', + old_name='secret', + new_name='_secret', + ), + migrations.RenameField( + model_name='accounttemplate', + old_name='secret', + new_name='_secret', + ), + migrations.RenameField( + model_name='historicalaccount', + old_name='secret', + new_name='_secret', + ), + ] diff --git a/apps/accounts/models/account.py b/apps/accounts/models/account.py index d84978b17..c6da60d48 100644 --- a/apps/accounts/models/account.py +++ b/apps/accounts/models/account.py @@ -7,9 +7,10 @@ from simple_history.models import HistoricalRecords from assets.models.base import AbsConnectivity from common.utils import lazyproperty from .base import BaseAccount +from .mixins import VaultModelMixin from ..const import AliasAccount, Source -__all__ = ['Account', 'AccountTemplate'] +__all__ = ['Account', 'AccountTemplate', 'AccountHistoricalRecords'] class AccountHistoricalRecords(HistoricalRecords): @@ -32,7 +33,7 @@ class AccountHistoricalRecords(HistoricalRecords): diff = attrs - history_attrs if not diff: return - super().post_save(instance, created, using=using, **kwargs) + return super().post_save(instance, created, using=using, **kwargs) def create_history_model(self, model, inherited): if self.included_fields and not self.excluded_fields: @@ -53,7 +54,7 @@ class Account(AbsConnectivity, BaseAccount): on_delete=models.SET_NULL, verbose_name=_("Su from") ) version = models.IntegerField(default=0, verbose_name=_('Version')) - history = AccountHistoricalRecords(included_fields=['id', 'secret', 'secret_type', 'version']) + history = AccountHistoricalRecords(included_fields=['id', '_secret', 'secret_type', 'version']) source = models.CharField(max_length=30, default=Source.LOCAL, verbose_name=_('Source')) source_id = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Source ID')) @@ -198,3 +199,21 @@ class AccountTemplate(BaseAccount): return self.bulk_update_accounts(accounts, {'secret': self.secret}) self.bulk_create_history_accounts(accounts, user_id) + + +def replace_history_model_with_mixin(): + """ + 替换历史模型中的父类为指定的Mixin类。 + + Parameters: + model (class): 历史模型类,例如 Account.history.model + mixin_class (class): 要替换为的Mixin类 + + Returns: + None + """ + model = Account.history.model + model.__bases__ = (VaultModelMixin,) + model.__bases__ + + +replace_history_model_with_mixin() diff --git a/apps/accounts/models/base.py b/apps/accounts/models/base.py index 3f9f70883..b06012c41 100644 --- a/apps/accounts/models/base.py +++ b/apps/accounts/models/base.py @@ -9,33 +9,32 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from accounts.const import SecretType -from common.db import fields from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, random_string, lazyproperty, parse_ssh_public_key_str, is_openssh_format_key ) +from accounts.models.mixins import VaultModelMixin, VaultManagerMixin, VaultQuerySetMixin from orgs.mixins.models import JMSOrgBaseModel, OrgManager logger = get_logger(__file__) -class BaseAccountQuerySet(models.QuerySet): +class BaseAccountQuerySet(VaultQuerySetMixin, models.QuerySet): def active(self): return self.filter(is_active=True) -class BaseAccountManager(OrgManager): +class BaseAccountManager(VaultManagerMixin, OrgManager): def active(self): return self.get_queryset().active() -class BaseAccount(JMSOrgBaseModel): +class BaseAccount(VaultModelMixin, JMSOrgBaseModel): name = models.CharField(max_length=128, verbose_name=_("Name")) username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True) secret_type = models.CharField( max_length=16, choices=SecretType.choices, default=SecretType.PASSWORD, verbose_name=_('Secret type') ) - secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret')) privileged = models.BooleanField(verbose_name=_("Privileged"), default=False) is_active = models.BooleanField(default=True, verbose_name=_("Is active")) diff --git a/apps/accounts/models/mixins/__init__.py b/apps/accounts/models/mixins/__init__.py new file mode 100644 index 000000000..203d7b3cb --- /dev/null +++ b/apps/accounts/models/mixins/__init__.py @@ -0,0 +1 @@ +from .vault import * diff --git a/apps/accounts/models/mixins/vault.py b/apps/accounts/models/mixins/vault.py new file mode 100644 index 000000000..95c85265d --- /dev/null +++ b/apps/accounts/models/mixins/vault.py @@ -0,0 +1,88 @@ +from django.db import models +from django.db.models.signals import post_save +from django.utils.translation import ugettext_lazy as _ + +from common.db import fields + +__all__ = ['VaultQuerySetMixin', 'VaultManagerMixin', 'VaultModelMixin'] + + +class VaultQuerySetMixin(models.QuerySet): + + def update(self, **kwargs): + """ + 1. 替换 secret 为 _secret + 2. 触发 post_save 信号 + """ + if 'secret' in kwargs: + kwargs.update({ + '_secret': kwargs.pop('secret') + }) + rows = super().update(**kwargs) + + # 为了获取更新后的对象所以单独查询一次 + ids = self.values_list('id', flat=True) + objs = self.model.objects.filter(id__in=ids) + for obj in objs: + post_save.send(obj.__class__, instance=obj, created=False) + return rows + + +class VaultManagerMixin(models.Manager): + """ 触发 bulk_create 和 bulk_update 操作下的 post_save 信号 """ + + def bulk_create(self, objs, batch_size=None, ignore_conflicts=False): + objs = super().bulk_create(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts) + for obj in objs: + post_save.send(obj.__class__, instance=obj, created=True) + return objs + + def bulk_update(self, objs, batch_size=None, ignore_conflicts=False): + objs = super().bulk_update(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts) + for obj in objs: + post_save.send(obj.__class__, instance=obj, created=False) + return objs + + +class VaultModelMixin(models.Model): + _secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret')) + is_sync_metadata = True + + class Meta: + abstract = True + + # 缓存 secret 值, lazy-property 不能用 + __secret = False + + @property + def secret(self): + if self.__secret is False: + from accounts.backends import vault_client + secret = vault_client.get(self) + if not secret and not self.secret_has_save_to_vault: + # vault_client 获取不到, 并且 secret 没有保存到 vault, 就从 self._secret 获取 + secret = self._secret + self.__secret = secret + return self.__secret + + @secret.setter + def secret(self, value): + """ + 保存的时候通过 post_save 信号监听进行处理, + 先保存到 db, 再保存到 vault 同时删除本地 db _secret 值 + """ + self._secret = value + + _secret_save_to_vault_mark = '# Secret-has-been-saved-to-vault #' + + def mark_secret_save_to_vault(self): + self._secret = self._secret_save_to_vault_mark + self.save() + + @property + def secret_has_save_to_vault(self): + return self._secret == self._secret_save_to_vault_mark + + def save(self, *args, **kwargs): + """ 通过 post_save signal 处理 _secret 数据 """ + return super().save(*args, **kwargs) diff --git a/apps/accounts/serializers/account/account.py b/apps/accounts/serializers/account/account.py index 54035e4be..9a4f1536b 100644 --- a/apps/accounts/serializers/account/account.py +++ b/apps/accounts/serializers/account/account.py @@ -198,7 +198,6 @@ class AccountAssetSerializer(serializers.ModelSerializer): class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerializer): asset = AccountAssetSerializer(label=_('Asset')) - has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True) source = LabeledChoiceField( choices=Source.choices, label=_("Source"), required=False, allow_null=True, default=Source.LOCAL @@ -233,6 +232,15 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize return queryset +class AccountDetailSerializer(AccountSerializer): + has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True) + + class Meta(AccountSerializer.Meta): + model = Account + fields = AccountSerializer.Meta.fields + ['has_secret'] + read_only_fields = AccountSerializer.Meta.read_only_fields + ['has_secret'] + + class AssetAccountBulkSerializerResultSerializer(serializers.Serializer): asset = serializers.CharField(read_only=True, label=_('Asset')) state = serializers.CharField(read_only=True, label=_('State')) diff --git a/apps/accounts/serializers/account/base.py b/apps/accounts/serializers/account/base.py index e2a837b03..5289ea25b 100644 --- a/apps/accounts/serializers/account/base.py +++ b/apps/accounts/serializers/account/base.py @@ -61,20 +61,18 @@ class AuthValidateMixin(serializers.Serializer): class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer): - has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True) class Meta: model = BaseAccount fields_mini = ['id', 'name', 'username'] fields_small = fields_mini + [ - 'secret_type', 'secret', 'has_secret', 'passphrase', + 'secret_type', 'secret', 'passphrase', 'privileged', 'is_active', 'spec_info', ] fields_other = ['created_by', 'date_created', 'date_updated', 'comment'] fields = fields_small + fields_other read_only_fields = [ - 'has_secret', 'spec_info', - 'date_verified', 'created_by', 'date_created', + 'spec_info', 'date_verified', 'created_by', 'date_created', ] extra_kwargs = { 'spec_info': {'label': _('Spec info')}, diff --git a/apps/accounts/signal_handlers.py b/apps/accounts/signal_handlers.py index cf09842cc..868e85614 100644 --- a/apps/accounts/signal_handlers.py +++ b/apps/accounts/signal_handlers.py @@ -1,8 +1,9 @@ -from django.db.models.signals import pre_save +from django.db.models.signals import pre_save, post_save, post_delete from django.dispatch import receiver +from accounts.backends import vault_client from common.utils import get_logger -from .models import Account +from .models import Account, AccountTemplate logger = get_logger(__name__) @@ -13,3 +14,23 @@ def on_account_pre_save(sender, instance, **kwargs): instance.version = 1 else: instance.version = instance.history.count() + + +class VaultSignalHandler(object): + """ 处理 Vault 相关的信号 """ + + @staticmethod + def save_to_vault(sender, instance, created, **kwargs): + if created: + vault_client.create(instance) + else: + vault_client.update(instance) + + @staticmethod + def delete_to_vault(sender, instance, **kwargs): + vault_client.delete(instance) + + +for model in (Account, AccountTemplate, Account.history.model): + post_save.connect(VaultSignalHandler.save_to_vault, sender=model) + post_delete.connect(VaultSignalHandler.delete_to_vault, sender=model) diff --git a/apps/accounts/tasks/vault.py b/apps/accounts/tasks/vault.py new file mode 100644 index 000000000..a6c9ed6db --- /dev/null +++ b/apps/accounts/tasks/vault.py @@ -0,0 +1,56 @@ +import datetime + +from celery import shared_task +from django.utils.translation import gettext_lazy as _ + +from accounts.backends import vault_client +from accounts.models import Account, AccountTemplate +from common.utils import get_logger +from orgs.utils import tmp_to_root_org +from ..const import VaultTypeChoices + +logger = get_logger(__name__) + + +@shared_task(verbose_name=_('Sync secret to vault')) +def sync_secret_to_vault(): + if vault_client.is_type(VaultTypeChoices.local): + # 这里不能判断 settings.VAULT_TYPE, 必须判断当前 vault_client 的类型 + print('\033[35m>>> 当前 Vault 类型为本地数据库, 不需要同步') + return + + print('\033[33m>>> 开始同步密钥数据到 Vault ({})'.format(datetime.datetime.now())) + with tmp_to_root_org(): + to_sync_models = [Account, AccountTemplate, Account.history.model] + for model in to_sync_models: + print(f'\033[33m>>> 开始同步: {model.__module__}') + succeeded = [] + failed = [] + skipped = [] + instances = model.objects.all() + for instance in instances: + instance_desc = f'[{instance}]' + if instance.secret_has_save_to_vault: + print(f'\033[32m- 跳过同步: {instance_desc}, 原因: [已同步]') + skipped.append(instance) + continue + try: + vault_client.create(instance) + except Exception as e: + failed.append(instance) + print(f'\033[31m- 同步失败: {instance_desc}, 原因: [{e}]') + else: + succeeded.append(instance) + print(f'\033[32m- 同步成功: {instance_desc}') + + total = len(succeeded) + len(failed) + len(skipped) + print( + f'\033[33m>>> 同步完成: {model.__module__}, ' + f'共计: {total}, ' + f'成功: {len(succeeded)}, ' + f'失败: {len(failed)}, ' + f'跳过: {len(skipped)}' + ) + + print('\033[33m>>> 全部同步完成 ({})'.format(datetime.datetime.now())) + print('\033[0m') diff --git a/apps/accounts/utils.py b/apps/accounts/utils.py index 1a24008fc..c0e3fc1cd 100644 --- a/apps/accounts/utils.py +++ b/apps/accounts/utils.py @@ -1,9 +1,7 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers -from accounts.const import ( - SecretType, DEFAULT_PASSWORD_RULES -) +from accounts.const import SecretType, DEFAULT_PASSWORD_RULES from common.utils import ssh_key_gen, random_string from common.utils import validate_ssh_private_key, parse_ssh_private_key_str diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index f9f2d9f43..ad969d52e 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -11,12 +11,12 @@ from django.core.exceptions import ValidationError from django.core.validators import MinValueValidator, MaxValueValidator from django.db import models from django.db.models import Q, Manager, QuerySet -from django.utils.encoding import force_str from django.utils.translation import gettext_lazy as _ from rest_framework.utils.encoders import JSONEncoder from common.local import add_encrypted_field_set -from common.utils import signer, crypto, contains_ip +from common.utils import contains_ip +from .utils import Encryptor from .validators import PortRangeValidator __all__ = [ @@ -139,20 +139,11 @@ class EncryptMixin: EncryptMixin要放在最前面 """ - def decrypt_from_signer(self, value): - return signer.unsign(value) or "" - def from_db_value(self, value, expression, connection, context=None): if value is None: return value - value = force_str(value) - plain_value = crypto.decrypt(value) - - # 如果没有解开,使用原来的signer解密 - if not plain_value: - plain_value = self.decrypt_from_signer(value) - + plain_value = Encryptor(value).decrypt() # 可能和Json mix,所以要先解密,再json sp = super() if hasattr(sp, "from_db_value"): @@ -167,9 +158,9 @@ class EncryptMixin: sp = super() if hasattr(sp, "get_prep_value"): value = sp.get_prep_value(value) - value = force_str(value) + # 替换新的加密方式 - return crypto.encrypt(value) + return Encryptor(value).encrypt() class EncryptTextField(EncryptMixin, models.TextField): diff --git a/apps/common/db/utils.py b/apps/common/db/utils.py index 23384f2c8..23197a4f9 100644 --- a/apps/common/db/utils.py +++ b/apps/common/db/utils.py @@ -1,8 +1,9 @@ from contextlib import contextmanager from django.db import connections +from django.utils.encoding import force_text -from common.utils import get_logger +from common.utils import get_logger, signer, crypto logger = get_logger(__file__) @@ -55,3 +56,19 @@ def safe_db_connection(): close_old_connections() yield close_old_connections() + + +class Encryptor: + def __init__(self, value): + self.value = force_text(value) + + def decrypt(self): + plain_value = crypto.decrypt(self.value) + + # 如果没有解开,使用原来的signer解密 + if not plain_value: + plain_value = signer.unsign(self.value) or "" + return plain_value + + def encrypt(self): + return crypto.encrypt(self.value) diff --git a/apps/common/decorators.py b/apps/common/decorators.py index 4881c1517..353407ab1 100644 --- a/apps/common/decorators.py +++ b/apps/common/decorators.py @@ -67,12 +67,18 @@ class EventLoopThread(threading.Thread): _loop_thread = EventLoopThread() _loop_thread.setDaemon(True) _loop_thread.start() -executor = ThreadPoolExecutor(max_workers=10, - thread_name_prefix='debouncer') +executor = ThreadPoolExecutor( + max_workers=10, + thread_name_prefix='debouncer' +) _loop_debouncer_func_task_cache = {} _loop_debouncer_func_args_cache = {} +def get_loop(): + return _loop_thread.get_loop() + + def cancel_or_remove_debouncer_task(cache_key): task = _loop_debouncer_func_task_cache.get(cache_key, None) if not task: diff --git a/apps/common/sdk/im/dingtalk/__init__.py b/apps/common/sdk/im/dingtalk/__init__.py index 4f54e1ea2..65d466008 100644 --- a/apps/common/sdk/im/dingtalk/__init__.py +++ b/apps/common/sdk/im/dingtalk/__init__.py @@ -1,10 +1,10 @@ -import time -import hmac import base64 +import hmac +import time -from common.utils import get_logger -from common.sdk.im.utils import digest, as_request from common.sdk.im.mixin import BaseRequest +from common.sdk.im.utils import digest, as_request +from common.utils import get_logger from users.utils import construct_user_email logger = get_logger(__file__) diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 067e3f7ee..98a6477d4 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -251,6 +251,12 @@ class Config(dict): # 临时密码 'AUTH_TEMP_TOKEN': False, + # Vault + 'VAULT_TYPE': 'local', + 'VAULT_HCP_HOST': '', + 'VAULT_HCP_TOKEN': '', + 'VAULT_HCP_MOUNT_POINT': 'jumpserver', + # Auth LDAP settings 'AUTH_LDAP': False, 'AUTH_LDAP_SERVER_URI': 'ldap://localhost:389', diff --git a/apps/jumpserver/settings/auth.py b/apps/jumpserver/settings/auth.py index fa3217f50..23c95df69 100644 --- a/apps/jumpserver/settings/auth.py +++ b/apps/jumpserver/settings/auth.py @@ -174,6 +174,12 @@ AUTH_OAUTH2_LOGOUT_URL_NAME = "authentication:oauth2:logout" # 临时 token AUTH_TEMP_TOKEN = CONFIG.AUTH_TEMP_TOKEN +# Vault +VAULT_TYPE = CONFIG.VAULT_TYPE +VAULT_HCP_HOST = CONFIG.VAULT_HCP_HOST +VAULT_HCP_TOKEN = CONFIG.VAULT_HCP_TOKEN +VAULT_HCP_MOUNT_POINT = CONFIG.VAULT_HCP_MOUNT_POINT + # Other setting # 这个是 User Login Private Token TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION diff --git a/apps/orgs/mixins/models.py b/apps/orgs/mixins/models.py index a9494ad1a..f41d4a67b 100644 --- a/apps/orgs/mixins/models.py +++ b/apps/orgs/mixins/models.py @@ -119,3 +119,4 @@ class OrgModelMixin(models.Model): class JMSOrgBaseModel(JMSBaseModel, OrgModelMixin): class Meta: abstract = True + diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index 6813e9f0b..ceed23907 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # import uuid -from inspect import signature -from functools import wraps -from werkzeug.local import LocalProxy from contextlib import contextmanager +from functools import wraps +from inspect import signature + +from werkzeug.local import LocalProxy from common.local import thread_local from .models import Organization @@ -133,6 +134,7 @@ def org_aware_func(org_arg_name): :param org_arg_name: 函数中包含org_id的对象是哪个参数 :return: """ + def decorate(func): @wraps(func) def wrapper(*args, **kwargs): @@ -149,7 +151,9 @@ def org_aware_func(org_arg_name): with tmp_to_org(org_aware_resource.org_id): # print("Current org id: {}".format(org_aware_resource.org_id)) return func(*args, **kwargs) + return wrapper + return decorate @@ -162,4 +166,5 @@ def ensure_in_real_or_default_org(func): if not current_org or current_org.is_root(): raise ValueError('You must in a real or default org!') return func(*args, **kwargs) + return wrapper diff --git a/apps/settings/api/__init__.py b/apps/settings/api/__init__.py index 176a6c2c6..13f4a2b18 100644 --- a/apps/settings/api/__init__.py +++ b/apps/settings/api/__init__.py @@ -1,8 +1,9 @@ -from .settings import * -from .ldap import * -from .wecom import * from .dingtalk import * -from .feishu import * -from .public import * from .email import * +from .feishu import * +from .ldap import * +from .public import * +from .settings import * from .sms import * +from .vault import * +from .wecom import * diff --git a/apps/settings/api/settings.py b/apps/settings/api/settings.py index 4bcef2889..ab7be6e9b 100644 --- a/apps/settings/api/settings.py +++ b/apps/settings/api/settings.py @@ -50,6 +50,7 @@ class SettingsApi(generics.RetrieveUpdateAPIView): 'huawei': serializers.HuaweiSMSSettingSerializer, 'cmpp2': serializers.CMPP2SMSSettingSerializer, 'custom': serializers.CustomSMSSettingSerializer, + 'vault': serializers.VaultSettingSerializer, } rbac_category_permissions = { @@ -75,6 +76,7 @@ class SettingsApi(generics.RetrieveUpdateAPIView): 'sms': 'settings.change_sms', 'alibaba': 'settings.change_sms', 'tencent': 'settings.change_sms', + 'vault': 'settings.change_vault', } def get_queryset(self): diff --git a/apps/settings/api/vault.py b/apps/settings/api/vault.py new file mode 100644 index 000000000..339b13ab3 --- /dev/null +++ b/apps/settings/api/vault.py @@ -0,0 +1,60 @@ +from django.utils.translation import gettext_lazy as _ +from django.conf import settings +from rest_framework import status +from rest_framework.generics import GenericAPIView +from rest_framework.views import Response, APIView + +from accounts.tasks.vault import sync_secret_to_vault +from accounts.backends import get_vault_client +from settings.models import Setting +from .. import serializers + + +class VaultTestingAPI(GenericAPIView): + serializer_class = serializers.VaultSettingSerializer + rbac_perms = { + 'POST': 'settings.change_vault' + } + + def get_config(self, request): + serializer = self.serializer_class(data=request.data) + serializer.is_valid(raise_exception=True) + data = serializer.validated_data + for k, v in data.items(): + if v: + continue + # 页面没有传递值, 从 settings 中获取 + data[k] = getattr(settings, k, None) + return data + + def post(self, request): + config = self.get_config(request) + try: + client = get_vault_client(raise_exception=True, **config) + ok, error = client.is_active() + except Exception as e: + ok, error = False, str(e) + + if ok: + _status, msg = status.HTTP_200_OK, _('Test success') + else: + _status, msg = status.HTTP_400_BAD_REQUEST, error + + return Response(status=_status, data={'msg': msg}) + + +class VaultSyncDataAPI(APIView): + perm_model = Setting + rbac_perms = { + 'POST': 'settings.change_vault' + } + + def post(self, request, *args, **kwargs): + task = self._run_task() + return Response({'task': task.id}, status=status.HTTP_201_CREATED) + + @staticmethod + def _run_task(): + task = sync_secret_to_vault.delay() + return task + diff --git a/apps/settings/migrations/0008_alter_setting_options.py b/apps/settings/migrations/0008_alter_setting_options.py new file mode 100644 index 000000000..9a8691d69 --- /dev/null +++ b/apps/settings/migrations/0008_alter_setting_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.19 on 2023-06-30 10:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('settings', '0007_migrate_ldap_sync_org_ids'), + ] + + operations = [ + migrations.AlterModelOptions( + name='setting', + options={'permissions': [('change_email', 'Can change email setting'), ('change_auth', 'Can change auth setting'), ('change_vault', 'Can change vault setting'), ('change_systemmsgsubscription', 'Can change system msg sub setting'), ('change_sms', 'Can change sms setting'), ('change_security', 'Can change security setting'), ('change_clean', 'Can change clean setting'), ('change_interface', 'Can change interface setting'), ('change_license', 'Can change license setting'), ('change_terminal', 'Can change terminal setting'), ('change_other', 'Can change other setting')], 'verbose_name': 'System setting'}, + ), + ] diff --git a/apps/settings/models.py b/apps/settings/models.py index ab570d07d..6aebf79b1 100644 --- a/apps/settings/models.py +++ b/apps/settings/models.py @@ -159,6 +159,7 @@ class Setting(models.Model): permissions = [ ('change_email', _('Can change email setting')), ('change_auth', _('Can change auth setting')), + ('change_vault', _('Can change vault setting')), ('change_systemmsgsubscription', _('Can change system msg sub setting')), ('change_sms', _('Can change sms setting')), ('change_security', _('Can change security setting')), diff --git a/apps/settings/serializers/__init__.py b/apps/settings/serializers/__init__.py index 0a55f645d..7abfb74e3 100644 --- a/apps/settings/serializers/__init__.py +++ b/apps/settings/serializers/__init__.py @@ -1,13 +1,14 @@ # coding: utf-8 # -from .basic import * from .auth import * -from .email import * -from .public import * -from .settings import * -from .security import * -from .terminal import * +from .basic import * from .cleaning import * +from .email import * from .other import * +from .public import * +from .security import * +from .settings import * +from .terminal import * +from .vault import * diff --git a/apps/settings/serializers/vault.py b/apps/settings/serializers/vault.py new file mode 100644 index 000000000..b3f8d4e78 --- /dev/null +++ b/apps/settings/serializers/vault.py @@ -0,0 +1,23 @@ +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from accounts.const import VaultTypeChoices +from common.serializers.fields import EncryptedField + +__all__ = ['VaultSettingSerializer'] + + +class VaultSettingSerializer(serializers.Serializer): + VAULT_TYPE = serializers.ChoiceField( + default=VaultTypeChoices.local, choices=VaultTypeChoices.choices, + required=False, label=_('Type') + ) + VAULT_HCP_HOST = serializers.CharField( + max_length=256, allow_blank=True, required=False, label=_('Host') + ) + VAULT_HCP_TOKEN = EncryptedField( + max_length=256, allow_blank=True, required=False, label=_('Token') + ) + VAULT_HCP_MOUNT_POINT = serializers.CharField( + max_length=256, allow_blank=True, required=False, label=_('Mount Point') + ) diff --git a/apps/settings/urls/api_urls.py b/apps/settings/urls/api_urls.py index ef94d02ba..ba074d4ee 100644 --- a/apps/settings/urls/api_urls.py +++ b/apps/settings/urls/api_urls.py @@ -18,6 +18,8 @@ urlpatterns = [ path('feishu/testing/', api.FeiShuTestingAPI.as_view(), name='feishu-testing'), path('sms//testing/', api.SMSTestingAPI.as_view(), name='sms-testing'), path('sms/backend/', api.SMSBackendAPI.as_view(), name='sms-backend'), + path('vault/testing/', api.VaultTestingAPI.as_view(), name='vault-testing'), + path('vault/sync/', api.VaultSyncDataAPI.as_view(), name='vault-sync'), path('setting/', api.SettingsApi.as_view(), name='settings-setting'), path('logo/', api.SettingsLogoApi.as_view(), name='settings-logo'), diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a94cd102f..303358615 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -133,3 +133,5 @@ ipython==8.14.0 ForgeryPy3==0.3.1 django-debug-toolbar==4.1.0 Pympler==1.0.1 +hvac==1.1.1 +pyhcl==0.4.4