mirror of
https://github.com/jumpserver/jumpserver.git
synced 2025-09-18 00:08:31 +00:00
* Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) (#5548) * tree: v0.1 * tree: v0.2 * tree: v0.3 * tree: v0.4 * tree: 添加并发锁未请求到时的debug日志 * 以空间换时间的方式优化资产树 * Reactor tree togther v2 (#5576) * Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) (#5548) * tree: v0.1 * tree: v0.2 * tree: v0.3 * tree: v0.4 * tree: 添加并发锁未请求到时的debug日志 * 以空间换时间的方式优化资产树 * 修改授权适配新方案 * 添加树处理工具 * 完成新的用户授权树计算以及修改一些信号 * 重构了获取资产的一些 api * 重构了一些节点的api * 整理了一些代码 * 完成了api 的重构 * 重构检查节点数量功能 * 完成重构授权树工具类 * api 添加强制刷新参数 * 整理一些信号 * 处理一些信号的问题 * 完成了信号的处理 * 重构了资产树相关的锁机制 * RebuildUserTreeTask 还得添加回来 * 优化下不能在root组织的检查函数 * 优化资产树变化时锁的使用 * 修改一些算法的小工具 * 资产树锁不再校验是否在具体组织里 * 整理了一些信号的位置 * 修复资产与节点关系维护的bug * 去掉一些调试代码 * 修复资产授权过期检查刷新授权树的 bug * 添加了可重入锁 * 添加一些计时，优化一些sql * 增加 union 查询的支持 * 尝试用 sql 解决节点资产数量问题 * 开始优化计算授权树节点资产数量不用冗余表 * 新代码能跑起来了，修复一下bug * 去掉 UserGrantedMappingNode 换成 UserAssetGrantedTreeNodeRelation * 修了些bug，做了些优化 * 优化QuerySetStage 执行逻辑 * 与小白的内存结合了 * 删掉老的表，迁移新的 assets_amount 字段 * 优化用户授权页面资产列表 count 慢 * 修复批量命令数量不对 * 修改获取非直接授权节点的 children 的逻辑 * 获取整棵树的节点 * 回退锁 * 整理迁移脚本 * 改变更新树策略 * perf: 修改一波缩进 * fix: 修改handler名称 * 修复授权树获取资产sql 泛滥 * 修复授权规则有效bug * 修复一些bug * 修复一些bug * 又修了一些小bug * 去掉了老的 get_nodes_all_assets * 修改一些写法 * Reactor tree togther b2 (#5570) * fix: 修改handler名称 * perf: 优化生成树 * perf: 去掉注释 * 优化了一些 * 重新生成迁移脚本 * 去掉周期检查节点资产数量的任务 * Pr@reactor tree togther guang@perf mapping (#5573) * fix: 修改handler名称 * perf: mapping 拆分出来 * 修改名称 * perf: 修改锁名 * perf: 去掉检查节点任务 * perf: 修改一下名称 * perf: 优化一波 Co-authored-by: Jiangjie.Bai <32935519+BaiJiangJie@users.noreply.github.com> Co-authored-by: Bai <bugatti_it@163.com> Co-authored-by: xinwen <coderWen@126.com> Co-authored-by: xinwen <coderWen@126.com> Co-authored-by: 老广 <ibuler@qq.com>
This commit is contained in:
Jiangjie.Bai
GitHub
@@ -254,3 +254,22 @@ def get_disk_usage():
|
||||
mount_points = [p.mountpoint for p in partitions]
|
||||
usages = {p: psutil.disk_usage(p) for p in mount_points}
|
||||
return usages
|
||||
|
||||
|
||||
class Time:
|
||||
def __init__(self):
|
||||
self._timestamps = []
|
||||
self._msgs = []
|
||||
|
||||
def begin(self):
|
||||
self._timestamps.append(time.time())
|
||||
|
||||
def time(self, msg):
|
||||
self._timestamps.append(time.time())
|
||||
self._msgs.append(msg)
|
||||
|
||||
def print(self):
|
||||
last, *timestamps = self._timestamps
|
||||
for timestamp, msg in zip(timestamps, self._msgs):
|
||||
logger.debug(f'TIME_IT: {msg} {timestamp-last}')
|
||||
last = timestamp
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from functools import wraps
|
||||
import threading
|
||||
|
||||
from redis_lock import Lock as RedisLock
|
||||
from redis_lock import Lock as RedisLock, NotAcquired
|
||||
from redis import Redis
|
||||
from django.db import transaction
|
||||
|
||||
from common.utils import get_logger
|
||||
from common.utils.inspect import copy_function_args
|
||||
@@ -16,7 +17,8 @@ class AcquireFailed(RuntimeError):
|
||||
|
||||
|
||||
class DistributedLock(RedisLock):
|
||||
def __init__(self, name, blocking=True, expire=60*2, auto_renewal=True):
|
||||
def __init__(self, name, blocking=True, expire=None, release_lock_on_transaction_commit=False,
|
||||
release_raise_exc=False, auto_renewal_seconds=60*2):
|
||||
"""
|
||||
使用 redis 构造的分布式锁
|
||||
|
||||
@@ -25,31 +27,46 @@ class DistributedLock(RedisLock):
|
||||
:param blocking:
|
||||
该参数只在锁作为装饰器或者 `with` 时有效。
|
||||
:param expire:
|
||||
锁的过期时间，注意不一定是锁到这个时间就释放了，分两种情况
|
||||
当 `auto_renewal=False` 时，锁会释放
|
||||
当 `auto_renewal=True` 时，如果过期之前程序还没释放锁，我们会延长锁的存活时间。
|
||||
这里的作用是防止程序意外终止没有释放锁，导致死锁。
|
||||
锁的过期时间
|
||||
:param release_lock_on_transaction_commit:
|
||||
是否在当前事务结束后再释放锁
|
||||
:param release_raise_exc:
|
||||
释放锁时，如果没有持有锁是否抛异常或静默
|
||||
:param auto_renewal_seconds:
|
||||
当持有一个无限期锁的时候，刷新锁的时间，具体参考 `redis_lock.Lock#auto_renewal`
|
||||
"""
|
||||
self.kwargs_copy = copy_function_args(self.__init__, locals())
|
||||
redis = Redis(host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, password=CONFIG.REDIS_PASSWORD)
|
||||
|
||||
if expire is None:
|
||||
expire = auto_renewal_seconds
|
||||
auto_renewal = True
|
||||
else:
|
||||
auto_renewal = False
|
||||
|
||||
super().__init__(redis_client=redis, name=name, expire=expire, auto_renewal=auto_renewal)
|
||||
self._blocking = blocking
|
||||
self._release_lock_on_transaction_commit = release_lock_on_transaction_commit
|
||||
self._release_raise_exc = release_raise_exc
|
||||
|
||||
def __enter__(self):
|
||||
thread_id = threading.current_thread().ident
|
||||
logger.debug(f'DISTRIBUTED_LOCK: <thread_id:{thread_id}> attempt to acquire <lock:{self._name}> ...')
|
||||
logger.debug(f'Attempt to acquire global lock: thread {thread_id} lock {self._name}')
|
||||
acquired = self.acquire(blocking=self._blocking)
|
||||
if self._blocking and not acquired:
|
||||
logger.debug(f'DISTRIBUTED_LOCK: <thread_id:{thread_id}> was not acquired <lock:{self._name}>, but blocking=True')
|
||||
logger.debug(f'Not acquired lock, but blocking=True, thread {thread_id} lock {self._name}')
|
||||
raise EnvironmentError("Lock wasn't acquired, but blocking=True")
|
||||
if not acquired:
|
||||
logger.debug(f'DISTRIBUTED_LOCK: <thread_id:{thread_id}> acquire <lock:{self._name}> failed')
|
||||
logger.debug(f'Not acquired the lock, thread {thread_id} lock {self._name}')
|
||||
raise AcquireFailed
|
||||
logger.debug(f'DISTRIBUTED_LOCK: <thread_id:{thread_id}> acquire <lock:{self._name}> ok')
|
||||
logger.debug(f'Acquire lock success, thread {thread_id} lock {self._name}')
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
|
||||
self.release()
|
||||
if self._release_lock_on_transaction_commit:
|
||||
transaction.on_commit(self.release)
|
||||
else:
|
||||
self.release()
|
||||
|
||||
def __call__(self, func):
|
||||
@wraps(func)
|
||||
@@ -57,5 +74,17 @@ class DistributedLock(RedisLock):
|
||||
# 要创建一个新的锁对象
|
||||
with self.__class__(**self.kwargs_copy):
|
||||
return func(*args, **kwds)
|
||||
|
||||
return inner
|
||||
|
||||
def locked_by_me(self):
|
||||
if self.locked():
|
||||
if self.get_owner_id() == self.id:
|
||||
return True
|
||||
return False
|
||||
|
||||
def release(self):
|
||||
try:
|
||||
super().release()
|
||||
except AcquireFailed as e:
|
||||
if self._release_raise_exc:
|
||||
raise e
|
||||
|
||||
Reference in New Issue
Block a user