merge: with v3

apps/common/utils/common.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 #
 import re
-import socket
 from django.templatetags.static import static
 from collections import OrderedDict
 from itertools import chain
@@ -14,6 +13,7 @@ import ipaddress
 import psutil
 import platform
 import os
+import socket
 
 from django.conf import settings
 
@@ -344,7 +344,7 @@ def get_file_by_arch(dir, filename):
     return file_path
 
 
-def pretty_string(data: str, max_length=128, ellipsis_str='...'):
+def pretty_string(data, max_length=128, ellipsis_str='...'):
     """
     params:
        data: abcdefgh
@@ -353,6 +353,7 @@ def pretty_string(data: str, max_length=128, ellipsis_str='...'):
    return:
        ab...gh
     """
+    data = str(data)
     if len(data) < max_length:
         return data
     remain_length = max_length - len(ellipsis_str)

apps/common/utils/django.py

@@ -2,11 +2,11 @@
 #
 import re
 
-from django.shortcuts import reverse as dj_reverse
 from django.conf import settings
-from django.utils import timezone
 from django.db import models
 from django.db.models.signals import post_save, pre_save
+from django.shortcuts import reverse as dj_reverse
+from django.utils import timezone
 
 UUID_PATTERN = re.compile(r'[0-9a-zA-Z\-]{36}')
 
@@ -80,3 +80,18 @@ def bulk_create_with_signal(cls: models.Model, items, **kwargs):
     for i in items:
         post_save.send(sender=cls, instance=i, created=True)
     return result
+
+
+def get_request_os(request):
+    """获取请求的操作系统"""
+    agent = request.META.get('HTTP_USER_AGENT', '').lower()
+   
+    if agent is None:
+        return 'unknown'
+    if 'windows' in agent.lower():
+        return 'windows'
+    if 'mac' in agent.lower():
+        return 'mac'
+    if 'linux' in agent.lower():
+        return 'linux'
+    return 'unknown'

apps/common/utils/encode.py

@@ -1,24 +1,23 @@
 # -*- coding: utf-8 -*-
 #
-import re
-import json
-from six import string_types
 import base64
-import os
-import time
 import hashlib
+import json
+import os
+import re
+import time
 from io import StringIO
-from itertools import chain
 
 import paramiko
 import sshpubkeys
+from cryptography.hazmat.primitives import serialization
+from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
 from itsdangerous import (
     TimedJSONWebSignatureSerializer, JSONWebSignatureSerializer,
     BadSignature, SignatureExpired
 )
-from django.conf import settings
-from django.core.serializers.json import DjangoJSONEncoder
-from django.db.models.fields.files import FileField
+from six import string_types
 
 from .http import http_date
 
@@ -69,22 +68,23 @@ class Signer(metaclass=Singleton):
             return None
 
 
+_supported_paramiko_ssh_key_types = (
+    paramiko.RSAKey,
+    paramiko.DSSKey,
+    paramiko.Ed25519Key,
+    paramiko.ECDSAKey,)
+
+
 def ssh_key_string_to_obj(text, password=None):
     key = None
-    try:
-        key = paramiko.RSAKey.from_private_key(StringIO(text), password=password)
-    except paramiko.SSHException:
-        pass
-    else:
-        return key
-
-    try:
-        key = paramiko.DSSKey.from_private_key(StringIO(text), password=password)
-    except paramiko.SSHException:
-        pass
-    else:
-        return key
-
+    for ssh_key_type in _supported_paramiko_ssh_key_types:
+        try:
+            key = ssh_key_type.from_private_key(StringIO(text), password=password)
+            return key
+        except paramiko.SSHException:
+            pass
+    if key is None:
+        raise ValueError('Invalid private key')
     return key
 
 
@@ -137,17 +137,65 @@ def ssh_key_gen(length=2048, type='rsa', password=None, username='jumpserver', h
 
 
 def validate_ssh_private_key(text, password=None):
-    if isinstance(text, bytes):
-        try:
-            text = text.decode("utf-8")
-        except UnicodeDecodeError:
-            return False
+    key = parse_ssh_private_key_str(text, password=password)
+    return bool(key)
 
-    key = ssh_key_string_to_obj(text, password=password)
-    if key is None:
-        return False
-    else:
-        return True
+
+def parse_ssh_private_key_str(text: bytes, password=None) -> str:
+    private_key = _parse_ssh_private_key(text, password=password)
+    if private_key is None:
+        return ""
+    # 解析之后，转换成 openssh 格式的私钥
+    private_key_bytes = private_key.private_bytes(
+        serialization.Encoding.PEM,
+        serialization.PrivateFormat.OpenSSH,
+        serialization.NoEncryption()
+    )
+    return private_key_bytes.decode('utf-8')
+
+
+def parse_ssh_public_key_str(text: bytes = "", password=None) -> str:
+    private_key = _parse_ssh_private_key(text, password=password)
+    if private_key is None:
+        return ""
+    public_key_bytes = private_key.public_key().public_bytes(
+        serialization.Encoding.OpenSSH,
+        serialization.PublicFormat.OpenSSH,
+    )
+    return public_key_bytes.decode('utf-8')
+
+
+def _parse_ssh_private_key(text, password=None):
+    """
+    text: bytes
+    password: str
+    return:private key types:
+                ec.EllipticCurvePrivateKey,
+                rsa.RSAPrivateKey,
+                dsa.DSAPrivateKey,
+                ed25519.Ed25519PrivateKey,
+    """
+    if isinstance(text, str):
+        try:
+            text = text.encode("utf-8")
+        except UnicodeDecodeError:
+            return None
+    if isinstance(password, str):
+        try:
+            password = password.encode("utf-8")
+        except UnicodeDecodeError:
+            return None
+
+    try:
+        if is_openssh_format_key(text):
+            return serialization.load_ssh_private_key(text, password=password)
+        return serialization.load_pem_private_key(text, password=password)
+    except (ValueError, TypeError) as e:
+        raise e
+
+
+def is_openssh_format_key(text: bytes):
+    return text.startswith(b"-----BEGIN OPENSSH PRIVATE KEY-----")
 
 
 def validate_ssh_public_key(text):

apps/common/utils/geoip/GeoLite2-City.mmdb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:860b4d38beff81667c64da41c026a7dd28c3c93a28ae61fefaa7c26875f35638
+size 73906864

apps/common/utils/http.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 #
-import time
-from email.utils import formatdate
 import calendar
 import threading
+import time
+from email.utils import formatdate
 
 _STRPTIME_LOCK = threading.Lock()
 
@@ -35,3 +35,6 @@ def http_to_unixtime(time_string):
 def iso8601_to_unixtime(time_string):
     """把ISO8601时间字符串（形如，2012-02-24T06:07:48.000Z）转换为UNIX时间，精确到秒。"""
     return to_unixtime(time_string, _ISO8601_FORMAT)
+
+
+

apps/common/utils/integer.py

@@ -0,0 +1,4 @@
+def bit(x):
+    if x < 1:
+        raise ValueError("x must be greater than 1")
+    return 2 ** (x - 1)

apps/common/utils/ip/utils.py

@@ -66,7 +66,7 @@ def contains_ip(ip, ip_group):
             if in_ip_segment(ip, _ip):
                 return True
         else:
-            # is domain name
+            # address / host
             if ip == _ip:
                 return True
 
@@ -75,7 +75,7 @@ def contains_ip(ip, ip_group):
 
 def get_ip_city(ip):
     if not ip or not isinstance(ip, str):
-        return _("Invalid ip")
+        return _("Invalid address")
     if ':' in ip:
         return 'IPv6'
 

apps/common/utils/lock.py

@@ -76,7 +76,6 @@ class DistributedLock(RedisLock):
             # 要创建一个新的锁对象
             with self.__class__(**self.kwargs_copy):
                 return func(*args, **kwds)
-
         return inner
 
     @classmethod
@@ -105,22 +104,21 @@ class DistributedLock(RedisLock):
         if self._reentrant:
             if self.locked_by_current_thread():
                 self._acquired_reentrant_lock = True
-                logger.debug(f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
                 return True
 
-            logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name}')
             acquired = super().acquire(blocking=blocking, timeout=timeout)
             if acquired:
-                logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name}')
                 setattr(thread_local, self.name, self.id)
             else:
-                logger.debug(
-                    f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name}')
             return acquired
         else:
-            logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name}')
             acquired = super().acquire(blocking=blocking, timeout=timeout)
-            logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name}')
             return acquired
 
     @property
@@ -139,17 +137,17 @@ class DistributedLock(RedisLock):
     def _release_on_reentrant_locked_by_brother(self):
         if self._acquired_reentrant_lock:
             self._acquired_reentrant_lock = False
-            logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
             return
         else:
-            self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
 
     def _release_on_reentrant_locked_by_me(self):
-        logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+        logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name}')
 
         id = getattr(thread_local, self.name, None)
         if id != self.id:
-            raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
         try:
             # 这里要保证先删除 thread_local 的标记，
             delattr(thread_local, self.name)
@@ -171,9 +169,9 @@ class DistributedLock(RedisLock):
     def _release(self):
         try:
             self._release_redis_lock()
-            logger.debug(f'Released lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Released lock: lock_id={self.id} lock={self.name}')
         except NotAcquired as e:
-            logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id} error: {e}')
+            logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} error: {e}')
             self._raise_exc(e)
 
     def release(self):
@@ -188,12 +186,12 @@ class DistributedLock(RedisLock):
                     _release = self._release_on_reentrant_locked_by_brother
             else:
                 self._raise_exc_with_log(
-                    f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                    f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name}')
 
         # 处理是否在事务提交时才释放锁
         if self._release_on_transaction_commit:
             logger.debug(
-                f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name}')
             transaction.on_commit(_release)
         else:
             _release()

apps/common/utils/timezone.py

@@ -1,22 +1,21 @@
-import datetime
-
 import pytz
+from datetime import datetime, timedelta, timezone
 from django.utils import timezone as dj_timezone
 from rest_framework.fields import DateTimeField
 
-max = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
+max = datetime.max.replace(tzinfo=timezone.utc)
 
 
-def astimezone(dt: datetime.datetime, tzinfo: pytz.tzinfo.DstTzInfo):
+def astimezone(dt: datetime, tzinfo: pytz.tzinfo.DstTzInfo):
     assert dj_timezone.is_aware(dt)
     return tzinfo.normalize(dt.astimezone(tzinfo))
 
 
-def as_china_cst(dt: datetime.datetime):
+def as_china_cst(dt: datetime):
     return astimezone(dt, pytz.timezone('Asia/Shanghai'))
 
 
-def as_current_tz(dt: datetime.datetime):
+def as_current_tz(dt: datetime):
     return astimezone(dt, dj_timezone.get_current_timezone())
 
 
@@ -36,6 +35,15 @@ def local_now_date_display(fmt='%Y-%m-%d'):
     return local_now().strftime(fmt)
 
 
+def local_zero_hour(fmt='%Y-%m-%d'):
+    return datetime.strptime(local_now().strftime(fmt), fmt)
+
+
+def local_monday():
+    zero_hour_time = local_zero_hour()
+    return zero_hour_time - timedelta(zero_hour_time.weekday())
+
+
 _rest_dt_field = DateTimeField()
 dt_parser = _rest_dt_field.to_internal_value
 dt_formatter = _rest_dt_field.to_representation