feat: 账号密钥用vault储存 (#10830)

* feat: 账号密钥用vault储存

* perf: 优化 Vault

* perf: 重构 Vault Backend 设计架构 (未完成)

* perf: 重构 Vault Backend 设计架构 (未完成2)

* perf: 重构 Vault Backend 设计架构 (未完成3)

* perf: 重构 Vault Backend 设计架构 (未完成4)

* perf: 重构 Vault Backend 设计架构 (未完成5)

* perf: 重构 Vault Backend 设计架构 (已完成)

* perf: 重构 Vault Backend 设计架构 (已完成)

* perf: 重构 Vault Backend 设计架构 (已完成)

* perf: 小优化

* perf: 优化

---------

Co-authored-by: feng <1304903146@qq.com>
Co-authored-by: Bai <baijiangjie@gmail.com>
Co-authored-by: feng626 <57284900+feng626@users.noreply.github.com>
This commit is contained in:
fit2bot
2023-07-31 17:39:30 +08:00
committed by GitHub
parent 7776158279
commit 3b615719fe
38 changed files with 819 additions and 59 deletions

View File

@@ -26,6 +26,7 @@ class AccountViewSet(OrgBulkModelViewSet):
filterset_class = AccountFilterSet
serializer_classes = {
'default': serializers.AccountSerializer,
'retrieve': serializers.AccountDetailSerializer,
}
rbac_perms = {
'partial_update': ['accounts.change_account'],
@@ -133,11 +134,13 @@ class AccountHistoriesSecretAPI(ExtraFilterFieldsMixin, RecordViewLogMixin, List
def get_queryset(self):
account = self.get_object()
histories = account.history.all()
last_history = account.history.first()
if not last_history:
latest_history = account.history.first()
if not latest_history:
return histories
if account.secret == last_history.secret \
and account.secret_type == last_history.secret_type:
histories = histories.exclude(history_id=last_history.history_id)
if account.secret != latest_history.secret:
return histories
if account.secret_type != latest_history.secret_type:
return histories
histories = histories.exclude(history_id=latest_history.history_id)
return histories

View File

@@ -0,0 +1,41 @@
import os
from django.utils.functional import LazyObject
from importlib import import_module
from common.utils import get_logger
from ..const import VaultTypeChoices
__all__ = ['vault_client', 'get_vault_client']
logger = get_logger(__file__)
def get_vault_client(raise_exception=False, **kwargs):
try:
tp = kwargs.get('VAULT_TYPE')
module_path = f'apps.accounts.backends.{tp}.main'
client = import_module(module_path).Vault(**kwargs)
except Exception as e:
logger.error(f'Init vault client failed: {e}')
if raise_exception:
raise
tp = VaultTypeChoices.local
module_path = f'apps.accounts.backends.{tp}.main'
kwargs['VAULT_TYPE'] = tp
client = import_module(module_path).Vault(**kwargs)
return client
class VaultClient(LazyObject):
def _setup(self):
from jumpserver import settings as js_settings
from django.conf import settings
vault_config_names = [k for k in js_settings.__dict__.keys() if k.startswith('VAULT_')]
vault_configs = {name: getattr(settings, name, None) for name in vault_config_names}
self._wrapped = get_vault_client(**vault_configs)
""" 为了安全, 页面修改配置, 重启服务后才会重新初始化 vault_client """
vault_client = VaultClient()

View File

@@ -0,0 +1,77 @@
from abc import ABC, abstractmethod
from django.forms.models import model_to_dict
__all__ = ['BaseVault']
class BaseVault(ABC):
def __init__(self, *args, **kwargs):
self.type = kwargs.get('VAULT_TYPE')
def is_type(self, tp):
return self.type == tp
def get(self, instance):
""" 返回 secret 值 """
return self._get(instance)
def create(self, instance):
if not instance.secret_has_save_to_vault:
self._create(instance)
self._clean_db_secret(instance)
self.save_metadata(instance)
if instance.is_sync_metadata:
self.save_metadata(instance)
def update(self, instance):
if not instance.secret_has_save_to_vault:
self._update(instance)
self._clean_db_secret(instance)
self.save_metadata(instance)
if instance.is_sync_metadata:
self.save_metadata(instance)
def delete(self, instance):
self._delete(instance)
def save_metadata(self, instance):
metadata = model_to_dict(instance, fields=[
'name', 'username', 'secret_type',
'connectivity', 'su_from', 'privileged'
])
metadata = {field: str(value) for field, value in metadata.items()}
return self._save_metadata(instance, metadata)
# -------- abstractmethod -------- #
@abstractmethod
def _get(self, instance):
raise NotImplementedError
@abstractmethod
def _create(self, instance):
raise NotImplementedError
@abstractmethod
def _update(self, instance):
raise NotImplementedError
@abstractmethod
def _delete(self, instance):
raise NotImplementedError
@abstractmethod
def _clean_db_secret(self, instance):
raise NotImplementedError
@abstractmethod
def _save_metadata(self, instance, metadata):
raise NotImplementedError
@abstractmethod
def is_active(self, *args, **kwargs) -> (bool, str):
raise NotImplementedError

View File

@@ -0,0 +1 @@
from .main import *

View File

@@ -0,0 +1,84 @@
import sys
from abc import ABC
from common.db.utils import Encryptor
from common.utils import lazyproperty
current_module = sys.modules[__name__]
__all__ = ['build_entry']
class BaseEntry(ABC):
def __init__(self, instance):
self.instance = instance
@lazyproperty
def full_path(self):
path_base = self.path_base
path_spec = self.path_spec
path = f'{path_base}/{path_spec}'
return path
@property
def path_base(self):
path = f'orgs/{self.instance.org_id}'
return path
@property
def path_spec(self):
raise NotImplementedError
def to_internal_data(self):
secret = getattr(self.instance, '_secret', None)
if secret is not None:
secret = Encryptor(secret).encrypt()
data = {'secret': secret}
return data
@staticmethod
def to_external_data(data):
secret = data.pop('secret', None)
if secret is not None:
secret = Encryptor(secret).decrypt()
return secret
class AccountEntry(BaseEntry):
@property
def path_spec(self):
path = f'assets/{self.instance.asset_id}/accounts/{self.instance.id}'
return path
class AccountTemplateEntry(BaseEntry):
@property
def path_spec(self):
path = f'account-templates/{self.instance.id}'
return path
class HistoricalAccountEntry(BaseEntry):
@property
def path_base(self):
account = self.instance.instance
path = f'accounts/{account.id}/'
return path
@property
def path_spec(self):
path = f'histories/{self.instance.history_id}'
return path
def build_entry(instance) -> BaseEntry:
class_name = instance.__class__.__name__
entry_class_name = f'{class_name}Entry'
entry_class = getattr(current_module, entry_class_name, None)
if not entry_class:
raise Exception(f'Entry class {entry_class_name} is not found')
return entry_class(instance)

View File

@@ -0,0 +1,47 @@
from .entries import build_entry
from .service import VaultKVClient
from ..base import BaseVault
__all__ = ['Vault']
class Vault(BaseVault):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = VaultKVClient(
url=kwargs.get('VAULT_HCP_HOST'),
token=kwargs.get('VAULT_HCP_TOKEN'),
mount_point=kwargs.get('VAULT_HCP_MOUNT_POINT')
)
def is_active(self):
return self.client.is_active()
def _get(self, instance):
entry = build_entry(instance)
# TODO: get data 是不是层数太多了
data = self.client.get(path=entry.full_path).get('data', {})
data = entry.to_external_data(data)
return data
def _create(self, instance):
entry = build_entry(instance)
data = entry.to_internal_data()
self.client.create(path=entry.full_path, data=data)
def _update(self, instance):
entry = build_entry(instance)
data = entry.to_internal_data()
self.client.patch(path=entry.full_path, data=data)
def _delete(self, instance):
entry = build_entry(instance)
self.client.delete(path=entry.full_path)
def _clean_db_secret(self, instance):
instance.is_sync_metadata = False
instance.mark_secret_save_to_vault()
def _save_metadata(self, instance, metadata):
entry = build_entry(instance)
self.client.update_metadata(path=entry.full_path, metadata=metadata)

View File

@@ -0,0 +1,103 @@
# -*- coding: utf-8 -*-
#
import hvac
from hvac import exceptions
from requests.exceptions import ConnectionError
from common.utils import get_logger
logger = get_logger(__name__)
__all__ = ['VaultKVClient']
class VaultKVClient(object):
max_versions = 20
def __init__(self, url, token, mount_point):
assert isinstance(self.max_versions, int) and self.max_versions >= 3, (
'max_versions must to be an integer that is greater than or equal to 3'
)
self.client = hvac.Client(url=url, token=token)
self.mount_point = mount_point
self.enable_secrets_engine_if_need()
def is_active(self):
try:
if not self.client.sys.is_initialized():
return False, 'Vault is not initialized'
if self.client.sys.is_sealed():
return False, 'Vault is sealed'
if not self.client.is_authenticated():
return False, 'Vault is not authenticated'
except ConnectionError as e:
logger.error(str(e))
return False, f'Vault is not reachable: {e}'
else:
return True, ''
def enable_secrets_engine_if_need(self):
secrets_engines = self.client.sys.list_mounted_secrets_engines()
mount_points = secrets_engines.keys()
if f'{self.mount_point}/' in mount_points:
return
self.client.sys.enable_secrets_engine(
backend_type='kv',
path=self.mount_point,
options={'version': 2} # TODO: version 是否从配置中读取?
)
self.client.secrets.kv.v2.configure(
max_versions=self.max_versions,
mount_point=self.mount_point
)
def get(self, path, version=None):
try:
response = self.client.secrets.kv.v2.read_secret_version(
path=path,
version=version,
mount_point=self.mount_point
)
except exceptions.InvalidPath as e:
logger.error('Get secret error: {}'.format(e))
return {}
data = response.get('data', {})
return data
def create(self, path, data: dict):
self._update_or_create(path=path, data=data)
def update(self, path, data: dict):
""" 未更新的数据会被删除 """
self._update_or_create(path=path, data=data)
def patch(self, path, data: dict):
""" 未更新的数据不会被删除 """
self.client.secrets.kv.v2.patch(
path=path,
secret=data,
mount_point=self.mount_point
)
def delete(self, path):
self.client.secrets.kv.v2.delete_metadata_and_all_versions(
path=path,
mount_point=self.mount_point,
)
def _update_or_create(self, path, data: dict):
self.client.secrets.kv.v2.create_or_update_secret(
path=path,
secret=data,
mount_point=self.mount_point
)
def update_metadata(self, path, metadata: dict):
try:
self.client.secrets.kv.v2.update_metadata(
path=path,
mount_point=self.mount_point,
custom_metadata=metadata
)
except exceptions.InvalidPath as e:
logger.error('Update metadata error: {}'.format(e))

View File

@@ -0,0 +1 @@
from .main import *

View File

@@ -0,0 +1,36 @@
from common.utils import get_logger
from ..base import BaseVault
logger = get_logger(__name__)
__all__ = ['Vault']
class Vault(BaseVault):
def is_active(self):
return True, ''
def _get(self, instance):
secret = getattr(instance, '_secret', None)
return secret
def _create(self, instance):
""" Ignore """
pass
def _update(self, instance):
""" Ignore """
pass
def _delete(self, instance):
""" Ignore """
pass
def _save_metadata(self, instance, metadata):
""" Ignore """
pass
def _clean_db_secret(self, instance):
""" Ignore *重要* 不能删除本地 secret """
pass

View File

@@ -1,2 +1,3 @@
from .account import *
from .automation import *
from .vault import *

View File

@@ -0,0 +1,9 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
__all__ = ['VaultTypeChoices']
class VaultTypeChoices(models.TextChoices):
local = 'local', _('Local Vault')
hcp = 'hcp', _('HCP Vault')

View File

@@ -0,0 +1,28 @@
# Generated by Django 3.2.19 on 2023-06-21 06:56
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('accounts', '0011_auto_20230506_1443'),
]
operations = [
migrations.RenameField(
model_name='account',
old_name='secret',
new_name='_secret',
),
migrations.RenameField(
model_name='accounttemplate',
old_name='secret',
new_name='_secret',
),
migrations.RenameField(
model_name='historicalaccount',
old_name='secret',
new_name='_secret',
),
]

View File

@@ -7,9 +7,10 @@ from simple_history.models import HistoricalRecords
from assets.models.base import AbsConnectivity
from common.utils import lazyproperty
from .base import BaseAccount
from .mixins import VaultModelMixin
from ..const import AliasAccount, Source
__all__ = ['Account', 'AccountTemplate']
__all__ = ['Account', 'AccountTemplate', 'AccountHistoricalRecords']
class AccountHistoricalRecords(HistoricalRecords):
@@ -32,7 +33,7 @@ class AccountHistoricalRecords(HistoricalRecords):
diff = attrs - history_attrs
if not diff:
return
super().post_save(instance, created, using=using, **kwargs)
return super().post_save(instance, created, using=using, **kwargs)
def create_history_model(self, model, inherited):
if self.included_fields and not self.excluded_fields:
@@ -53,7 +54,7 @@ class Account(AbsConnectivity, BaseAccount):
on_delete=models.SET_NULL, verbose_name=_("Su from")
)
version = models.IntegerField(default=0, verbose_name=_('Version'))
history = AccountHistoricalRecords(included_fields=['id', 'secret', 'secret_type', 'version'])
history = AccountHistoricalRecords(included_fields=['id', '_secret', 'secret_type', 'version'])
source = models.CharField(max_length=30, default=Source.LOCAL, verbose_name=_('Source'))
source_id = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Source ID'))
@@ -198,3 +199,21 @@ class AccountTemplate(BaseAccount):
return
self.bulk_update_accounts(accounts, {'secret': self.secret})
self.bulk_create_history_accounts(accounts, user_id)
def replace_history_model_with_mixin():
"""
替换历史模型中的父类为指定的Mixin类。
Parameters:
model (class): 历史模型类,例如 Account.history.model
mixin_class (class): 要替换为的Mixin类
Returns:
None
"""
model = Account.history.model
model.__bases__ = (VaultModelMixin,) + model.__bases__
replace_history_model_with_mixin()

View File

@@ -9,33 +9,32 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from accounts.const import SecretType
from common.db import fields
from common.utils import (
ssh_key_string_to_obj, ssh_key_gen, get_logger,
random_string, lazyproperty, parse_ssh_public_key_str, is_openssh_format_key
)
from accounts.models.mixins import VaultModelMixin, VaultManagerMixin, VaultQuerySetMixin
from orgs.mixins.models import JMSOrgBaseModel, OrgManager
logger = get_logger(__file__)
class BaseAccountQuerySet(models.QuerySet):
class BaseAccountQuerySet(VaultQuerySetMixin, models.QuerySet):
def active(self):
return self.filter(is_active=True)
class BaseAccountManager(OrgManager):
class BaseAccountManager(VaultManagerMixin, OrgManager):
def active(self):
return self.get_queryset().active()
class BaseAccount(JMSOrgBaseModel):
class BaseAccount(VaultModelMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_("Name"))
username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True)
secret_type = models.CharField(
max_length=16, choices=SecretType.choices, default=SecretType.PASSWORD, verbose_name=_('Secret type')
)
secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
privileged = models.BooleanField(verbose_name=_("Privileged"), default=False)
is_active = models.BooleanField(default=True, verbose_name=_("Is active"))

View File

@@ -0,0 +1 @@
from .vault import *

View File

@@ -0,0 +1,88 @@
from django.db import models
from django.db.models.signals import post_save
from django.utils.translation import ugettext_lazy as _
from common.db import fields
__all__ = ['VaultQuerySetMixin', 'VaultManagerMixin', 'VaultModelMixin']
class VaultQuerySetMixin(models.QuerySet):
def update(self, **kwargs):
"""
1. 替换 secret 为 _secret
2. 触发 post_save 信号
"""
if 'secret' in kwargs:
kwargs.update({
'_secret': kwargs.pop('secret')
})
rows = super().update(**kwargs)
# 为了获取更新后的对象所以单独查询一次
ids = self.values_list('id', flat=True)
objs = self.model.objects.filter(id__in=ids)
for obj in objs:
post_save.send(obj.__class__, instance=obj, created=False)
return rows
class VaultManagerMixin(models.Manager):
""" 触发 bulk_create 和 bulk_update 操作下的 post_save 信号 """
def bulk_create(self, objs, batch_size=None, ignore_conflicts=False):
objs = super().bulk_create(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts)
for obj in objs:
post_save.send(obj.__class__, instance=obj, created=True)
return objs
def bulk_update(self, objs, batch_size=None, ignore_conflicts=False):
objs = super().bulk_update(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts)
for obj in objs:
post_save.send(obj.__class__, instance=obj, created=False)
return objs
class VaultModelMixin(models.Model):
_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
is_sync_metadata = True
class Meta:
abstract = True
# 缓存 secret 值, lazy-property 不能用
__secret = False
@property
def secret(self):
if self.__secret is False:
from accounts.backends import vault_client
secret = vault_client.get(self)
if not secret and not self.secret_has_save_to_vault:
# vault_client 获取不到, 并且 secret 没有保存到 vault, 就从 self._secret 获取
secret = self._secret
self.__secret = secret
return self.__secret
@secret.setter
def secret(self, value):
"""
保存的时候通过 post_save 信号监听进行处理,
先保存到 db, 再保存到 vault 同时删除本地 db _secret 值
"""
self._secret = value
_secret_save_to_vault_mark = '# Secret-has-been-saved-to-vault #'
def mark_secret_save_to_vault(self):
self._secret = self._secret_save_to_vault_mark
self.save()
@property
def secret_has_save_to_vault(self):
return self._secret == self._secret_save_to_vault_mark
def save(self, *args, **kwargs):
""" 通过 post_save signal 处理 _secret 数据 """
return super().save(*args, **kwargs)

View File

@@ -198,7 +198,6 @@ class AccountAssetSerializer(serializers.ModelSerializer):
class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerializer):
asset = AccountAssetSerializer(label=_('Asset'))
has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
source = LabeledChoiceField(
choices=Source.choices, label=_("Source"), required=False,
allow_null=True, default=Source.LOCAL
@@ -233,6 +232,15 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize
return queryset
class AccountDetailSerializer(AccountSerializer):
has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
class Meta(AccountSerializer.Meta):
model = Account
fields = AccountSerializer.Meta.fields + ['has_secret']
read_only_fields = AccountSerializer.Meta.read_only_fields + ['has_secret']
class AssetAccountBulkSerializerResultSerializer(serializers.Serializer):
asset = serializers.CharField(read_only=True, label=_('Asset'))
state = serializers.CharField(read_only=True, label=_('State'))

View File

@@ -61,20 +61,18 @@ class AuthValidateMixin(serializers.Serializer):
class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
class Meta:
model = BaseAccount
fields_mini = ['id', 'name', 'username']
fields_small = fields_mini + [
'secret_type', 'secret', 'has_secret', 'passphrase',
'secret_type', 'secret', 'passphrase',
'privileged', 'is_active', 'spec_info',
]
fields_other = ['created_by', 'date_created', 'date_updated', 'comment']
fields = fields_small + fields_other
read_only_fields = [
'has_secret', 'spec_info',
'date_verified', 'created_by', 'date_created',
'spec_info', 'date_verified', 'created_by', 'date_created',
]
extra_kwargs = {
'spec_info': {'label': _('Spec info')},

View File

@@ -1,8 +1,9 @@
from django.db.models.signals import pre_save
from django.db.models.signals import pre_save, post_save, post_delete
from django.dispatch import receiver
from accounts.backends import vault_client
from common.utils import get_logger
from .models import Account
from .models import Account, AccountTemplate
logger = get_logger(__name__)
@@ -13,3 +14,23 @@ def on_account_pre_save(sender, instance, **kwargs):
instance.version = 1
else:
instance.version = instance.history.count()
class VaultSignalHandler(object):
""" 处理 Vault 相关的信号 """
@staticmethod
def save_to_vault(sender, instance, created, **kwargs):
if created:
vault_client.create(instance)
else:
vault_client.update(instance)
@staticmethod
def delete_to_vault(sender, instance, **kwargs):
vault_client.delete(instance)
for model in (Account, AccountTemplate, Account.history.model):
post_save.connect(VaultSignalHandler.save_to_vault, sender=model)
post_delete.connect(VaultSignalHandler.delete_to_vault, sender=model)

View File

@@ -0,0 +1,56 @@
import datetime
from celery import shared_task
from django.utils.translation import gettext_lazy as _
from accounts.backends import vault_client
from accounts.models import Account, AccountTemplate
from common.utils import get_logger
from orgs.utils import tmp_to_root_org
from ..const import VaultTypeChoices
logger = get_logger(__name__)
@shared_task(verbose_name=_('Sync secret to vault'))
def sync_secret_to_vault():
if vault_client.is_type(VaultTypeChoices.local):
# 这里不能判断 settings.VAULT_TYPE, 必须判断当前 vault_client 的类型
print('\033[35m>>> 当前 Vault 类型为本地数据库, 不需要同步')
return
print('\033[33m>>> 开始同步密钥数据到 Vault ({})'.format(datetime.datetime.now()))
with tmp_to_root_org():
to_sync_models = [Account, AccountTemplate, Account.history.model]
for model in to_sync_models:
print(f'\033[33m>>> 开始同步: {model.__module__}')
succeeded = []
failed = []
skipped = []
instances = model.objects.all()
for instance in instances:
instance_desc = f'[{instance}]'
if instance.secret_has_save_to_vault:
print(f'\033[32m- 跳过同步: {instance_desc}, 原因: [已同步]')
skipped.append(instance)
continue
try:
vault_client.create(instance)
except Exception as e:
failed.append(instance)
print(f'\033[31m- 同步失败: {instance_desc}, 原因: [{e}]')
else:
succeeded.append(instance)
print(f'\033[32m- 同步成功: {instance_desc}')
total = len(succeeded) + len(failed) + len(skipped)
print(
f'\033[33m>>> 同步完成: {model.__module__}, '
f'共计: {total}, '
f'成功: {len(succeeded)}, '
f'失败: {len(failed)}, '
f'跳过: {len(skipped)}'
)
print('\033[33m>>> 全部同步完成 ({})'.format(datetime.datetime.now()))
print('\033[0m')

View File

@@ -1,9 +1,7 @@
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from accounts.const import (
SecretType, DEFAULT_PASSWORD_RULES
)
from accounts.const import SecretType, DEFAULT_PASSWORD_RULES
from common.utils import ssh_key_gen, random_string
from common.utils import validate_ssh_private_key, parse_ssh_private_key_str