fix: 修复存储故障造成的录像获取失败问题

.dockerignore

@@ -7,5 +7,4 @@ django.db
 celerybeat.pid
 ### Vagrant ###
 .vagrant/
-apps/xpack/.git
-
+apps/xpack/.git

.gitattributes

@@ -1,4 +1,3 @@
 *.mmdb filter=lfs diff=lfs merge=lfs -text
 *.mo filter=lfs diff=lfs merge=lfs -text
 *.ipdb filter=lfs diff=lfs merge=lfs -text
-

.github/release-config.yml

@@ -41,5 +41,4 @@ version-resolver:
   default: patch
 template: |
   ## 版本变化  What’s Changed
-  $CHANGES
-
+  $CHANGES

.github/workflows/release-drafter.yml

@@ -21,7 +21,7 @@ jobs:
           TAG=$(basename ${GITHUB_REF})
           VERSION=${TAG/v/}
           wget https://raw.githubusercontent.com/jumpserver/installer/master/quick_start.sh
-          sed -i "s@VERSION=dev@VERSION=v${VERSION}@g" quick_start.sh
+          sed -i "s@Version=.*@Version=v${VERSION}@g" quick_start.sh
           echo "::set-output name=TAG::$TAG"
           echo "::set-output name=VERSION::$VERSION"
       - name: Create Release

.gitignore

@@ -41,4 +41,3 @@ release/*
 releashe
 /apps/script.py
 data/*
-

CODE_OF_CONDUCT.md

@@ -126,4 +126,3 @@ enforcement ladder](https://github.com/mozilla/diversity).
 For answers to common questions about this code of conduct, see the FAQ at
 https://www.contributor-covenant.org/faq. Translations are available at
 https://www.contributor-covenant.org/translations.
-

CONTRIBUTING.md

@@ -23,4 +23,3 @@ When reporting issues, always include:
 
 Because the issues are open to the public, when submitting files, be sure to remove any sensitive information, e.g. user name, password, IP address, and company name. You can
 replace those parts with "REDACTED" or other strings like "****".
-

Dockerfile

@@ -1,67 +1,53 @@
-FROM python:3.8-slim-bullseye as stage-build
-ARG TARGETARCH
-
-ARG VERSION
-ENV VERSION=$VERSION
-
-WORKDIR /opt/jumpserver
-ADD . .
-RUN cd utils && bash -ixeu build.sh
-
-FROM python:3.8-slim-bullseye
-ARG TARGETARCH
+FROM python:3.8-slim
 MAINTAINER JumpServer Team <ibuler@qq.com>
 
 ARG BUILD_DEPENDENCIES="              \
-        g++                           \
-        make                          \
-        pkg-config"
+    g++                               \
+    make                              \
+    pkg-config"
 
 ARG DEPENDENCIES="                    \
-        default-libmysqlclient-dev    \
-        freetds-dev                   \
-        libpq-dev                     \
-        libffi-dev                    \
-        libjpeg-dev                   \
-        libldap2-dev                  \
-        libsasl2-dev                  \
-        libxml2-dev                   \
-        libxmlsec1-dev                \
-        libxmlsec1-openssl            \
-        libaio-dev                    \
-        openssh-client                \
-        sshpass"
+    default-libmysqlclient-dev        \
+    freetds-dev                       \
+    libpq-dev                         \
+    libffi-dev                        \
+    libldap2-dev                      \
+    libsasl2-dev                      \
+    libxml2-dev                       \
+    libxmlsec1-dev                    \
+    libxmlsec1-openssl                \
+    libaio-dev                        \
+    openssh-client                    \
+    sshpass"
 
 ARG TOOLS="                           \
-        ca-certificates               \
-        curl                          \
-        default-mysql-client          \
-        iputils-ping                  \
-        locales                       \
-        patch                         \
-        procps                        \
-        redis-tools                   \
-        telnet                        \
-        vim                           \
-        unzip                         \
-        wget"
+    curl                              \
+    default-mysql-client              \
+    iproute2                          \
+    iputils-ping                      \
+    locales                           \
+    procps                            \
+    redis-tools                       \
+    telnet                            \
+    vim                               \
+    unzip                             \
+    wget"
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \
-    sed -i 's@http://.*.debian.org@http://mirrors.ustc.edu.cn@g' /etc/apt/sources.list \
-    && rm -f /etc/apt/apt.conf.d/docker-clean \
-    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+RUN sed -i 's@http://.*.debian.org@http://mirrors.ustc.edu.cn@g' /etc/apt/sources.list \
     && apt-get update \
     && apt-get -y install --no-install-recommends ${BUILD_DEPENDENCIES} \
     && apt-get -y install --no-install-recommends ${DEPENDENCIES} \
     && apt-get -y install --no-install-recommends ${TOOLS} \
+    && localedef -c -f UTF-8 -i zh_CN zh_CN.UTF-8 \
+    && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
     && mkdir -p /root/.ssh/ \
     && echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \
     && sed -i "s@# alias l@alias l@g" ~/.bashrc \
     && echo "set mouse-=a" > ~/.vimrc \
     && echo "no" | dpkg-reconfigure dash \
-    && echo "zh_CN.UTF-8" | dpkg-reconfigure locales \
     && rm -rf /var/lib/apt/lists/*
 
+ARG TARGETARCH
 ARG ORACLE_LIB_MAJOR=19
 ARG ORACLE_LIB_MINOR=10
 ENV ORACLE_FILE="instantclient-basiclite-linux.${TARGETARCH:-amd64}-${ORACLE_LIB_MAJOR}.${ORACLE_LIB_MINOR}.0.0.0dbru.zip"
@@ -82,20 +68,21 @@ ARG PIP_MIRROR=https://pypi.douban.com/simple
 ENV PIP_MIRROR=$PIP_MIRROR
 ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
 ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR
+# 因为以 jms 或者 jumpserver 开头的 mirror 上可能没有
+RUN pip install --upgrade pip==20.2.4 setuptools==49.6.0 wheel==0.34.2 -i ${PIP_MIRROR} \
+    && pip install --no-cache-dir $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
+    && pip install --no-cache-dir -r requirements/requirements.txt -i ${PIP_MIRROR} \
+    && rm -rf ~/.cache/pip
 
-RUN --mount=type=cache,target=/root/.cache/pip \
-    set -ex \
-    && pip config set global.index-url ${PIP_MIRROR} \
-    && pip install --upgrade pip \
-    && pip install --upgrade setuptools wheel \
-    && pip install Cython==0.29.35 \
-    && pip install --no-build-isolation pymssql \
-    && pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
-    && pip install -r requirements/requirements.txt --use-deprecated=legacy-resolver
+ARG VERSION
+ENV VERSION=$VERSION
 
-COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
-RUN echo > /opt/jumpserver/config.yml \
-    && rm -rf /tmp/build
+ADD . .
+RUN cd utils \
+    && bash -ixeu build.sh \
+    && mv ../release/jumpserver /opt/jumpserver \
+    && rm -rf /tmp/build \
+    && echo > /opt/jumpserver/config.yml
 
 WORKDIR /opt/jumpserver
 VOLUME /opt/jumpserver/data

Dockerfile.loong64

@@ -1,95 +0,0 @@
-FROM python:3.8-slim as stage-build
-ARG TARGETARCH
-
-ARG VERSION
-ENV VERSION=$VERSION
-
-WORKDIR /opt/jumpserver
-ADD . .
-RUN cd utils && bash -ixeu build.sh
-
-FROM python:3.8-slim
-ARG TARGETARCH
-MAINTAINER JumpServer Team <ibuler@qq.com>
-
-ARG BUILD_DEPENDENCIES="              \
-        g++                           \
-        make                          \
-        pkg-config"
-
-ARG DEPENDENCIES="                    \
-        default-libmysqlclient-dev    \
-        freetds-dev                   \
-        libpq-dev                     \
-        libffi-dev                    \
-        libjpeg-dev                   \
-        libldap2-dev                  \
-        libsasl2-dev                  \
-        libxml2-dev                   \
-        libxmlsec1-dev                \
-        libxmlsec1-openssl            \
-        libaio-dev                    \
-        openssh-client                \
-        sshpass"
-
-ARG TOOLS="                           \
-        ca-certificates               \
-        curl                          \
-        default-mysql-client          \
-        iputils-ping                  \
-        locales                       \
-        procps                        \
-        redis-tools                   \
-        telnet                        \
-        vim                           \
-        unzip                         \
-        wget"
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \
-    set -ex \
-    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
-    && apt-get update \
-    && apt-get -y install --no-install-recommends ${BUILD_DEPENDENCIES} \
-    && apt-get -y install --no-install-recommends ${DEPENDENCIES} \
-    && apt-get -y install --no-install-recommends ${TOOLS} \
-    && mkdir -p /root/.ssh/ \
-    && echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \
-    && sed -i "s@# alias l@alias l@g" ~/.bashrc \
-    && echo "set mouse-=a" > ~/.vimrc \
-    && echo "no" | dpkg-reconfigure dash \
-    && echo "zh_CN.UTF-8" | dpkg-reconfigure locales \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /tmp/build
-COPY ./requirements ./requirements
-
-ARG PIP_MIRROR=https://pypi.douban.com/simple
-ENV PIP_MIRROR=$PIP_MIRROR
-ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
-ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    set -ex \
-    && pip config set global.index-url ${PIP_MIRROR} \
-    && pip install --upgrade pip \
-    && pip install --upgrade setuptools wheel \
-    && pip install https://download.jumpserver.org/pypi/simple/cryptography/cryptography-36.0.1-cp38-cp38-linux_loongarch64.whl \
-    && pip install https://download.jumpserver.org/pypi/simple/greenlet/greenlet-1.1.2-cp38-cp38-linux_loongarch64.whl \
-    && pip install $(grep 'PyNaCl' requirements/requirements.txt) \
-    && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=true pip install grpcio \
-    && pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
-    && pip install -r requirements/requirements.txt
-
-COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
-RUN echo > /opt/jumpserver/config.yml \
-    && rm -rf /tmp/build
-
-WORKDIR /opt/jumpserver
-VOLUME /opt/jumpserver/data
-VOLUME /opt/jumpserver/logs
-
-ENV LANG=zh_CN.UTF-8
-
-EXPOSE 8070
-EXPOSE 8080
-ENTRYPOINT ["./entrypoint.sh"]

LICENSE

@@ -671,5 +671,4 @@ into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.
-
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

README.md

@@ -22,7 +22,9 @@ JumpServer 使用 Python 开发，配备了业界领先的 Web Terminal 方案
 
 JumpServer 采纳分布式架构，支持多机房跨区域部署，支持横向扩展，无资产数量及并发限制。
 
+改变世界，从一点点开始 ...
 
+> 如需进一步了解 JumpServer 开源项目，推荐阅读 [JumpServer 的初心和使命](https://mp.weixin.qq.com/s/S6q_2rP_9MwaVwyqLQnXzA)
 
 ### 特色优势
 
@@ -100,7 +102,7 @@ JumpServer 采纳分布式架构，支持多机房跨区域部署，支持横向
 - [沐瞳游戏：通过JumpServer管控多项目分布式资产](https://blog.fit2cloud.com/?p=3213)
 - [携程：JumpServer 堡垒机部署与运营实战](https://blog.fit2cloud.com/?p=851)
 - [大智慧：JumpServer 堡垒机让“大智慧”的混合 IT 运维更智慧](https://blog.fit2cloud.com/?p=882)
-- [小红书：JumpServer 堡垒机大规模资产跨版本迁移之路](https://blog.fit2cloud.com/?p=516)
+- [小红书：的JumpServer堡垒机大规模资产跨版本迁移之路](https://blog.fit2cloud.com/?p=516)
 - [中手游：JumpServer堡垒机助力中手游提升多云环境下安全运维能力](https://blog.fit2cloud.com/?p=732)
 - [中通快递：JumpServer主机安全运维实践](https://blog.fit2cloud.com/?p=708)
 - [东方明珠：JumpServer高效管控异构化、分布式云端资产](https://blog.fit2cloud.com/?p=687)

README_EN.md

@@ -92,3 +92,4 @@ Licensed under The GNU General Public License version 3 (GPLv3)  (the "License")
 https://www.gnu.org/licenses/gpl-3.0.htmll
 
 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+

SECURITY.md

@@ -18,4 +18,3 @@ All security bugs should be reported to the contact as below:
 - ibuler@fit2cloud.com
 - support@fit2cloud.com
 - 400-052-0755
-

Vagrantfile

@@ -0,0 +1,56 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box_check_update = false
+  config.vm.box = "centos/7"
+  config.vm.hostname = "jumpserver"
+  config.vm.network "private_network", ip: "172.17.8.101"
+  config.vm.provider "virtualbox" do |vb|
+    vb.memory = "4096"
+    vb.cpus = 2
+    vb.name = "jumpserver"
+  end
+
+  config.vm.synced_folder ".", "/vagrant", type: "rsync",
+    rsync__verbose: true,
+    rsync__exclude: ['.git*', 'node_modules*','*.log','*.box','Vagrantfile']
+
+  config.vm.provision "shell", inline: <<-SHELL
+## 设置yum的阿里云源
+sudo curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
+sudo sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
+sudo curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
+sudo yum makecache
+
+## 安装依赖包
+sudo yum install -y python36 python36-devel python36-pip \
+		 libtiff-devel libjpeg-devel libzip-devel freetype-devel \
+     lcms2-devel libwebp-devel tcl-devel tk-devel sshpass \
+     openldap-devel mariadb-devel mysql-devel libffi-devel \
+     openssh-clients telnet openldap-clients gcc
+
+## 配置pip阿里云源
+mkdir /home/vagrant/.pip
+cat << EOF | sudo tee -a /home/vagrant/.pip/pip.conf
+[global]
+timeout = 6000
+index-url = https://mirrors.aliyun.com/pypi/simple/
+
+[install]
+use-mirrors = true
+mirrors = https://mirrors.aliyun.com/pypi/simple/
+trusted-host=mirrors.aliyun.com
+EOF
+
+python3.6 -m venv /home/vagrant/venv
+source /home/vagrant/venv/bin/activate
+echo 'source /home/vagrant/venv/bin/activate' >> /home/vagrant/.bash_profile
+  SHELL
+end

apps/acls/models/login_acl.py

@@ -1,7 +1,7 @@
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from .base import BaseACL, BaseACLQuerySet
-from common.utils import get_request_ip_or_data, get_ip_city
+from common.utils import get_request_ip, get_ip_city
 from common.utils.ip import contains_ip
 from common.utils.time_period import contains_time_period
 from common.utils.timezone import local_now_display
@@ -73,7 +73,7 @@ class LoginACL(BaseACL):
         from tickets.models import ApplyLoginTicket
         from orgs.models import Organization
         title = _('Login confirm') + ' {}'.format(self.user)
-        login_ip = get_request_ip_or_data(request) if request else ''
+        login_ip = get_request_ip(request) if request else ''
         login_ip = login_ip or '0.0.0.0'
         login_city = get_ip_city(login_ip)
         login_datetime = local_now_display()

apps/acls/serializers/login_acl.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import ugettext as _
 from rest_framework import serializers
 from common.drf.serializers import BulkModelSerializer
 from common.drf.serializers import MethodSerializer

apps/applications/api/application.py

@@ -4,7 +4,6 @@ from orgs.mixins.api import OrgBulkModelViewSet
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
-
 from common.tree import TreeNodeSerializer
 from common.mixins.api import SuggestionMixin
 from .. import serializers

apps/applications/apps.py

@@ -7,7 +7,3 @@ from django.apps import AppConfig
 class ApplicationsConfig(AppConfig):
     name = 'applications'
     verbose_name = _('Applications')
-
-    def ready(self):
-        from . import signal_handlers
-        super().ready()

apps/applications/const.py

@@ -27,7 +27,6 @@ class AppType(models.TextChoices):
     sqlserver = 'sqlserver', 'SQLServer'
     redis = 'redis', 'Redis'
     mongodb = 'mongodb', 'MongoDB'
-    clickhouse = 'clickhouse', 'ClickHouse'
 
     # remote-app category
     chrome = 'chrome', 'Chrome'
@@ -43,7 +42,7 @@ class AppType(models.TextChoices):
         return {
             AppCategory.db: [
                 cls.mysql, cls.mariadb, cls.oracle, cls.pgsql,
-                cls.sqlserver, cls.redis, cls.mongodb, cls.clickhouse
+                cls.sqlserver, cls.redis, cls.mongodb
             ],
             AppCategory.remote_app: [
                 cls.chrome, cls.mysql_workbench,
@@ -83,4 +82,10 @@ class AppType(models.TextChoices):
 
         if AppCategory.is_xpack(category):
             return True
-        return tp in ['oracle', 'postgresql', 'sqlserver', 'clickhouse']
+        return tp in ['oracle', 'postgresql', 'sqlserver']
+
+
+class OracleVersion(models.TextChoices):
+    version_11g = '11g', '11g'
+    version_12c = '12c', '12c'
+    version_other = 'other', _('Other')

apps/applications/migrations/0024_alter_application_type.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.2.14 on 2022-11-04 07:06
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('applications', '0023_auto_20220715_1556'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='application',
-            name='type',
-            field=models.CharField(choices=[('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('oracle', 'Oracle'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('clickhouse', 'ClickHouse'), ('chrome', 'Chrome'), ('mysql_workbench', 'MySQL Workbench'), ('vmware_client', 'vSphere Client'), ('custom', 'Custom'), ('k8s', 'Kubernetes')], max_length=16, verbose_name='Type'),
-        ),
-    ]

apps/applications/models/application.py

@@ -10,7 +10,9 @@ from common.mixins import CommonModelMixin
 from common.tree import TreeNode
 from common.utils import is_uuid
 from assets.models import Asset, SystemUser
+from ..const import OracleVersion
 
+from ..utils import KubernetesTree
 from .. import const
 
 
@@ -173,7 +175,6 @@ class ApplicationTreeNodeMixin:
         return pid
 
     def as_tree_node(self, pid, k8s_as_tree=False):
-        from ..utils import KubernetesTree
         if self.type == const.AppType.k8s and k8s_as_tree:
             node = KubernetesTree(pid).as_tree_node(self)
         else:
@@ -303,6 +304,15 @@ class Application(CommonModelMixin, OrgModelMixin, ApplicationTreeNodeMixin):
             target_ip = self.attrs.get('host')
         return target_ip
 
+    def get_target_protocol_for_oracle(self):
+        """ Oracle 类型需要单独处理，因为要携带版本号 """
+        if not self.is_type(self.APP_TYPE.oracle):
+            return
+        version = self.attrs.get('version', OracleVersion.version_12c)
+        if version == OracleVersion.version_other:
+            return
+        return 'oracle_%s' % version
+
 
 class ApplicationUser(SystemUser):
     class Meta:

apps/applications/serializers/attrs/application_type/__init__.py

@@ -6,7 +6,6 @@ from .pgsql import *
 from .sqlserver import *
 from .redis import *
 from .mongodb import *
-from .clickhouse import *
 
 from .chrome import *
 from .mysql_workbench import *

apps/applications/serializers/attrs/application_type/chrome.py

@@ -23,7 +23,7 @@ class ChromeSerializer(RemoteAppSerializer):
     )
     chrome_password = EncryptedField(
         max_length=128, allow_blank=True, required=False,
-        label=_('Chrome password'), allow_null=True, encrypted_key='chrome_password'
+        label=_('Chrome password'), allow_null=True
     )
 
 

apps/applications/serializers/attrs/application_type/clickhouse.py

@@ -1,16 +0,0 @@
-from rest_framework import serializers
-from django.utils.translation import ugettext_lazy as _
-
-from ..application_category import DBSerializer
-
-__all__ = ['ClickHouseSerializer']
-
-
-class ClickHouseSerializer(DBSerializer):
-    port = serializers.IntegerField(
-        default=9000, label=_('Port'), allow_null=True,
-        help_text=_(
-            'Typically, the port is 9000，'
-            'the HTTP interface and the native interface use different ports'
-        ),
-    )

apps/applications/serializers/attrs/application_type/oracle.py

@@ -2,9 +2,15 @@ from rest_framework import serializers
 from django.utils.translation import ugettext_lazy as _
 
 from ..application_category import DBSerializer
+from applications.const import OracleVersion
 
 __all__ = ['OracleSerializer']
 
 
 class OracleSerializer(DBSerializer):
+    version = serializers.ChoiceField(
+        choices=OracleVersion.choices, default=OracleVersion.version_12c,
+        allow_null=True, label=_('Version'),
+        help_text=_('Magnus currently supports only 11g and 12c connections')
+    )
     port = serializers.IntegerField(default=1521, label=_('Port'), allow_null=True)

apps/applications/serializers/attrs/attrs.py

@@ -31,7 +31,6 @@ type_serializer_classes_mapping = {
     const.AppType.sqlserver.value: application_type.SQLServerSerializer,
     const.AppType.redis.value: application_type.RedisSerializer,
     const.AppType.mongodb.value: application_type.MongoDBSerializer,
-    const.AppType.clickhouse.value: application_type.ClickHouseSerializer,
     # cloud
     const.AppType.k8s.value: application_type.K8SSerializer
 }

apps/applications/signal_handlers.py

@@ -1,2 +0,0 @@
-# -*- coding: utf-8 -*-
-#

apps/applications/utils/kubernetes_util.py

@@ -1,14 +1,18 @@
 # -*- coding: utf-8 -*-
+from urllib3.exceptions import MaxRetryError
 from urllib.parse import urlencode
 
-from kubernetes import client
 from kubernetes.client import api_client
 from kubernetes.client.api import core_v1_api
+from kubernetes import client
+from kubernetes.client.exceptions import ApiException
+
 from rest_framework.generics import get_object_or_404
 
-from assets.models import SystemUser
-from common.tree import TreeNode
 from common.utils import get_logger
+from common.tree import TreeNode
+from assets.models import SystemUser
+
 from .. import const
 
 logger = get_logger(__file__)
@@ -19,8 +23,7 @@ class KubernetesClient:
         self.url = url
         self.token = token
 
-    @property
-    def api(self):
+    def get_api(self):
         configuration = client.Configuration()
         configuration.host = self.url
         configuration.verify_ssl = False
@@ -29,46 +32,63 @@ class KubernetesClient:
         api = core_v1_api.CoreV1Api(c)
         return api
 
-    def get_namespaces(self):
-        namespaces = []
-        resp = self.api.list_namespace()
-        for ns in resp.items:
-            namespaces.append(ns.metadata.name)
-        return namespaces
+    def get_namespace_list(self):
+        api = self.get_api()
+        namespace_list = []
+        for ns in api.list_namespace().items:
+            namespace_list.append(ns.metadata.name)
+        return namespace_list
 
-    def get_pods(self, namespace):
-        pods = []
-        resp = self.api.list_namespaced_pod(namespace)
-        for pd in resp.items:
-            pods.append(pd.metadata.name)
-        return pods
+    def get_services(self):
+        api = self.get_api()
+        ret = api.list_service_for_all_namespaces(watch=False)
+        for i in ret.items:
+            print("%s \t%s \t%s \t%s \t%s \n" % (
+                i.kind, i.metadata.namespace, i.metadata.name, i.spec.cluster_ip, i.spec.ports))
 
-    def get_containers(self, namespace, pod_name):
-        containers = []
-        resp = self.api.read_namespaced_pod(pod_name, namespace)
-        for container in resp.spec.containers:
-            containers.append(container.name)
-        return containers
+    def get_pod_info(self, namespace, pod):
+        api = self.get_api()
+        resp = api.read_namespaced_pod(namespace=namespace, name=pod)
+        return resp
 
-    @classmethod
-    def run(cls, asset, secret, tp='namespace'):
-        k8s_url = f'{asset.address}'
-        k8s = cls(k8s_url, secret)
-        func_name = f'get_{tp}s'
-        if hasattr(k8s, func_name):
-            return getattr(k8s, func_name)()
-        return []
+    def get_pod_logs(self, namespace, pod):
+        api = self.get_api()
+        log_content = api.read_namespaced_pod_log(pod, namespace, pretty=True, tail_lines=200)
+        return log_content
 
-    @classmethod
-    def get_kubernetes_data(cls, app_id, system_user_id, tp, *args):
+    def get_pods(self):
+        api = self.get_api()
+        try:
+            ret = api.list_pod_for_all_namespaces(watch=False, _request_timeout=(3, 3))
+        except MaxRetryError:
+            logger.warning('Kubernetes connection timed out')
+            return
+        except ApiException as e:
+            if e.status == 401:
+                logger.warning('Kubernetes User not authenticated')
+            else:
+                logger.warning(e)
+            return
+        data = {}
+        for i in ret.items:
+            namespace = i.metadata.namespace
+            pod_info = {
+                'pod_name': i.metadata.name,
+                'containers': [j.name for j in i.spec.containers]
+            }
+            if namespace in data:
+                data[namespace].append(pod_info)
+            else:
+                data[namespace] = [pod_info, ]
+        return data
+
+    @staticmethod
+    def get_kubernetes_data(app_id, system_user_id):
         from ..models import Application
         app = get_object_or_404(Application, id=app_id)
         system_user = get_object_or_404(SystemUser, id=system_user_id)
-        k8s = cls(app.attrs['cluster'], system_user.token)
-        func_name = f'get_{tp}s'
-        if hasattr(k8s, func_name):
-            return getattr(k8s, func_name)(*args)
-        return []
+        k8s = KubernetesClient(app.attrs['cluster'], system_user.token)
+        return k8s.get_pods()
 
 
 class KubernetesTree:
@@ -98,10 +118,11 @@ class KubernetesTree:
         )
         return node
 
-    def as_namespace_pod_tree_node(self, name, meta, type, is_container=False):
+    def as_namespace_pod_tree_node(self, name, meta, type, counts=0, is_container=False):
         from ..models import ApplicationTreeNodeMixin
         i = ApplicationTreeNodeMixin.create_tree_id(self.tree_id, type, name)
         meta.update({type: name})
+        name = name if is_container else f'{name}({counts})'
         node = self.create_tree_node(
             i, self.tree_id, name, type, meta, icon='cloud', is_container=is_container
         )
@@ -136,30 +157,30 @@ class KubernetesTree:
         system_user_id = parent_info.get('system_user_id')
 
         tree_nodes = []
+        data = KubernetesClient.get_kubernetes_data(app_id, system_user_id)
+        if not data:
+            return tree_nodes
+
         if pod_name:
-            tp = 'container'
-            containers = KubernetesClient.get_kubernetes_data(
-                app_id, system_user_id, tp, namespace, pod_name
-            )
-            for container in containers:
+            for container in next(
+                    filter(
+                        lambda x: x['pod_name'] == pod_name, data[namespace]
+                    )
+            )['containers']:
                 container_node = self.as_namespace_pod_tree_node(
-                    container, parent_info, tp, is_container=True
+                    container, parent_info, 'container', is_container=True
                 )
                 tree_nodes.append(container_node)
         elif namespace:
-            tp = 'pod'
-            pods = KubernetesClient.get_kubernetes_data(app_id, system_user_id, tp, namespace)
-            for pod in pods:
-                pod_node = self.as_namespace_pod_tree_node(
-                    pod, parent_info, tp
+            for pod in data[namespace]:
+                pod_nodes = self.as_namespace_pod_tree_node(
+                    pod['pod_name'], parent_info, 'pod', len(pod['containers'])
                 )
-                tree_nodes.append(pod_node)
+                tree_nodes.append(pod_nodes)
         elif system_user_id:
-            tp = 'namespace'
-            namespaces = KubernetesClient.get_kubernetes_data(app_id, system_user_id, tp)
-            for namespace in namespaces:
+            for namespace, pods in data.items():
                 namespace_node = self.as_namespace_pod_tree_node(
-                    namespace, parent_info, tp
+                    namespace, parent_info, 'namespace', len(pods)
                 )
                 tree_nodes.append(namespace_node)
         return tree_nodes

apps/assets/api/system_user.py

@@ -208,7 +208,7 @@ class SystemUserTaskApi(generics.CreateAPIView):
 
 class SystemUserCommandFilterRuleListApi(generics.ListAPIView):
     rbac_perms = {
-        'list': 'assets.view_commandfilterule',
+        'list': 'assets.view_commandfilterule'
     }
 
     def get_serializer_class(self):
@@ -223,14 +223,12 @@ class SystemUserCommandFilterRuleListApi(generics.ListAPIView):
         if not system_user:
             system_user_id = self.request.query_params.get('system_user_id')
         asset_id = self.request.query_params.get('asset_id')
-        node_id = self.request.query_params.get('node_id')
         application_id = self.request.query_params.get('application_id')
         rules = CommandFilterRule.get_queryset(
             user_id=user_id,
             user_group_id=user_group_id,
             system_user_id=system_user_id,
             asset_id=asset_id,
-            node_id=node_id,
             application_id=application_id
         )
         return rules

apps/assets/migrations/0022_auto_20181012_1717.py

@@ -21,8 +21,8 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=64, verbose_name='Name')),
                 ('is_active', models.BooleanField(default=True, verbose_name='Is active')),
                 ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
-                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
-                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('date_created', models.DateTimeField(auto_now_add=True)),
+                ('date_updated', models.DateTimeField(auto_now=True)),
                 ('created_by', models.CharField(blank=True, default='', max_length=128, verbose_name='Created by')),
             ],
             options={

apps/assets/migrations/0084_auto_20220112_1959.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
-                ('is_periodic', models.BooleanField(default=False, verbose_name='Periodic perform')),
+                ('is_periodic', models.BooleanField(default=False)),
                 ('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')),
                 ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')),
                 ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),

apps/assets/migrations/0092_commandfilter_nodes.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.2.15 on 2022-10-09 09:55
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0091_auto_20220629_1826'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='commandfilter',
-            name='nodes',
-            field=models.ManyToManyField(blank=True, related_name='cmd_filters', to='assets.Node', verbose_name='Nodes'),
-        ),
-    ]

apps/assets/migrations/0093_alter_systemuser_protocol.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.2.14 on 2022-11-04 07:06
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0092_commandfilter_nodes'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='systemuser',
-            name='protocol',
-            field=models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('clickhouse', 'ClickHouse'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol'),
-        ),
-    ]

apps/assets/models/asset.py

@@ -2,8 +2,9 @@
 # -*- coding: utf-8 -*-
 #
 
-import logging
 import uuid
+import logging
+from functools import reduce
 from collections import OrderedDict
 
 from django.db import models
@@ -13,6 +14,7 @@ from rest_framework.exceptions import ValidationError
 from common.db.fields import JsonDictTextField
 from common.utils import lazyproperty
 from orgs.mixins.models import OrgModelMixin, OrgManager
+
 from .base import AbsConnectivity
 
 __all__ = ['Asset', 'ProtocolsMixin', 'Platform', 'AssetQuerySet']
@@ -114,9 +116,9 @@ class NodesRelationMixin:
         nodes = []
         for node in self.get_nodes():
             _nodes = node.get_ancestors(with_self=True)
-            nodes.extend(list(_nodes))
+            nodes.append(_nodes)
         if flat:
-            nodes = list(set([node.id for node in nodes]))
+            nodes = list(reduce(lambda x, y: set(x) | set(y), nodes))
         return nodes
 
 

apps/assets/models/base.py

@@ -1,27 +1,28 @@
 # -*- coding: utf-8 -*-
 #
+import io
 import os
 import uuid
 from hashlib import md5
 
 import sshpubkeys
-from django.conf import settings
 from django.core.cache import cache
 from django.db import models
-from django.db.models import QuerySet
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
+from django.conf import settings
+from django.db.models import QuerySet
 
-from common.db import fields
-from common.utils import random_string
+from common.utils import random_string, signer
 from common.utils import (
     ssh_key_string_to_obj, ssh_key_gen, get_logger, lazyproperty
 )
-from common.utils.encode import (
-    parse_ssh_public_key_str, parse_ssh_private_key_str
-)
+from common.utils.encode import ssh_pubkey_gen
+from common.validators import alphanumeric
+from common.db import fields
 from orgs.mixins.models import OrgModelMixin
 
+
 logger = get_logger(__file__)
 
 
@@ -63,16 +64,16 @@ class AuthMixin:
 
     @property
     def ssh_key_fingerprint(self):
-        public_key = None
         if self.public_key:
             public_key = self.public_key
         elif self.private_key:
             try:
-                public_key = parse_ssh_public_key_str(self.private_key, password=self.password)
+                public_key = ssh_pubkey_gen(private_key=self.private_key, password=self.password)
             except IOError as e:
                 return str(e)
-        if not public_key:
+        else:
             return ''
+
         public_key_obj = sshpubkeys.SSHKey(public_key)
         fingerprint = public_key_obj.hash_md5()
         return fingerprint
@@ -87,29 +88,24 @@ class AuthMixin:
 
     @property
     def private_key_file(self):
-        if not self.private_key:
-            return None
-        private_key_str = self.get_private_key()
-        if not private_key_str:
+        if not self.private_key_obj:
             return None
         project_dir = settings.PROJECT_DIR
         tmp_dir = os.path.join(project_dir, 'tmp')
         key_name = '.' + md5(self.private_key.encode('utf-8')).hexdigest()
         key_path = os.path.join(tmp_dir, key_name)
         if not os.path.exists(key_path):
-            with open(key_path, 'w') as f:
-                f.write(private_key_str)
+            self.private_key_obj.write_private_key_file(key_path)
             os.chmod(key_path, 0o400)
         return key_path
 
     def get_private_key(self):
-        if not self.private_key:
+        if not self.private_key_obj:
             return None
-        private_key_str = parse_ssh_private_key_str(self.private_key, password=self.password)
-        if not private_key_str and self.password:
-            # 由于历史原因，密码可能是真实的密码，而非私钥的 passphrase，所以这里再尝试一次
-            private_key_str = parse_ssh_private_key_str(self.private_key)
-        return private_key_str
+        string_io = io.StringIO()
+        self.private_key_obj.write_private_key(string_io)
+        private_key = string_io.getvalue()
+        return private_key
 
     @property
     def public_key_obj(self):
@@ -238,3 +234,4 @@ class BaseUser(OrgModelMixin, AuthMixin):
 
     class Meta:
         abstract = True
+

apps/assets/models/cmd_filter.py

@@ -10,7 +10,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from users.models import User, UserGroup
 from applications.models import Application
-from ..models import SystemUser, Asset, Node
+from ..models import SystemUser, Asset
 
 from common.utils import lazyproperty, get_logger, get_object_or_none
 from orgs.mixins.models import OrgModelMixin
@@ -33,10 +33,6 @@ class CommandFilter(OrgModelMixin):
         'users.UserGroup', related_name='cmd_filters', blank=True,
         verbose_name=_("User group"),
     )
-    nodes = models.ManyToManyField(
-        'assets.Node', related_name='cmd_filters', blank=True,
-        verbose_name=_("Nodes")
-    )
     assets = models.ManyToManyField(
         'assets.Asset', related_name='cmd_filters', blank=True,
         verbose_name=_("Asset")
@@ -50,8 +46,8 @@ class CommandFilter(OrgModelMixin):
     )
     is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
     comment = models.TextField(blank=True, default='', verbose_name=_("Comment"))
-    date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created'))
-    date_updated = models.DateTimeField(auto_now=True, verbose_name=_('Date updated'))
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_updated = models.DateTimeField(auto_now=True)
     created_by = models.CharField(
         max_length=128, blank=True, default='', verbose_name=_('Created by')
     )
@@ -193,8 +189,7 @@ class CommandFilterRule(OrgModelMixin):
 
     @classmethod
     def get_queryset(cls, user_id=None, user_group_id=None, system_user_id=None,
-                     asset_id=None, node_id=None, application_id=None, org_id=None):
-        # user & user_group
+                     asset_id=None, application_id=None, org_id=None):
         user_groups = []
         user = get_object_or_none(User, pk=user_id)
         if user:
@@ -203,18 +198,8 @@ class CommandFilterRule(OrgModelMixin):
         if user_group:
             org_id = user_group.org_id
             user_groups.append(user_group)
-
-        # asset & node
-        nodes = []
-        asset = get_object_or_none(Asset, pk=asset_id)
-        if asset:
-            nodes.extend(asset.get_all_nodes())
-        node = get_object_or_none(Node, pk=node_id)
-        if node:
-            org_id = node.org_id
-            nodes.extend(list(node.get_ancestors(with_self=True)))
-
         system_user = get_object_or_none(SystemUser, pk=system_user_id)
+        asset = get_object_or_none(Asset, pk=asset_id)
         application = get_object_or_none(Application, pk=application_id)
         q = Q()
         if user:
@@ -227,8 +212,6 @@ class CommandFilterRule(OrgModelMixin):
         if asset:
             org_id = asset.org_id
             q |= Q(assets=asset)
-        if nodes:
-            q |= Q(nodes__in=set(nodes))
         if application:
             org_id = application.org_id
             q |= Q(applications=application)

apps/assets/models/user.py

@@ -4,16 +4,16 @@
 
 import logging
 
-from django.core.cache import cache
-from django.core.validators import MinValueValidator, MaxValueValidator
 from django.db import models
-from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
+from django.core.validators import MinValueValidator, MaxValueValidator
+from django.core.cache import cache
 
-from common.utils import signer, get_object_or_none, is_uuid
+from common.utils import signer, get_object_or_none
+from .base import BaseUser
 from .asset import Asset
 from .authbook import AuthBook
-from .base import BaseUser
+
 
 __all__ = ['AdminUser', 'SystemUser']
 logger = logging.getLogger(__name__)
@@ -34,7 +34,6 @@ class ProtocolMixin:
         sqlserver = 'sqlserver', 'SQLServer'
         redis = 'redis', 'Redis'
         mongodb = 'mongodb', 'MongoDB'
-        clickhouse = 'clickhouse', 'ClickHouse'
         k8s = 'k8s', 'K8S'
 
     SUPPORT_PUSH_PROTOCOLS = [Protocol.ssh, Protocol.rdp]
@@ -47,7 +46,7 @@ class ProtocolMixin:
     ]
     APPLICATION_CATEGORY_DB_PROTOCOLS = [
         Protocol.mysql, Protocol.mariadb, Protocol.oracle,
-        Protocol.postgresql, Protocol.sqlserver, Protocol.clickhouse,
+        Protocol.postgresql, Protocol.sqlserver,
         Protocol.redis, Protocol.mongodb
     ]
     APPLICATION_CATEGORY_CLOUD_PROTOCOLS = [
@@ -187,22 +186,22 @@ class AuthMixin:
         if username == '':
             username = self.username
 
-        not_stu_query = Q(asset=asset, username=username, systemuser__isnull=True)
-        stu_query = Q(asset=asset, systemuser=self)
-        not_stu_qs = AuthBook.objects.filter(not_stu_query).order_by('-date_created')
-        stu_qs = AuthBook.objects.filter(stu_query).order_by('-date_created')
+        authbook = AuthBook.objects.filter(
+            asset=asset, username=username, systemuser__isnull=True
+        ).order_by('-date_created').first()
 
-        authbook = not_stu_qs.first()
         if not authbook:
-            authbook = stu_qs.first()
+            authbook = AuthBook.objects.filter(
+                asset=asset, systemuser=self
+            ).order_by('-date_created').first()
 
         if not authbook:
             return None
-        authbook.load_auth()
 
-        self.password = authbook.password or self.password or ''
-        self.private_key = authbook.private_key or self.private_key or ''
-        self.public_key = authbook.public_key or self.public_key or ''
+        authbook.load_auth()
+        self.password = authbook.password
+        self.private_key = authbook.private_key
+        self.public_key = authbook.public_key
 
     def load_asset_more_auth(self, asset_id=None, username=None, user_id=None):
         from users.models import User
@@ -249,19 +248,12 @@ class SystemUser(ProtocolMixin, AuthMixin, BaseUser):
     users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users"))
     groups = models.ManyToManyField('users.UserGroup', blank=True, verbose_name=_("User groups"))
     type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_('Type'))
-    priority = models.IntegerField(
-        default=81, verbose_name=_("Priority"),
-        help_text=_("1-100, the lower the value will be match first"),
-        validators=[MinValueValidator(1), MaxValueValidator(100)]
-    )
-    protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh',
-                                verbose_name=_('Protocol'))
+    priority = models.IntegerField(default=81, verbose_name=_("Priority"), help_text=_("1-100, the lower the value will be match first"), validators=[MinValueValidator(1), MaxValueValidator(100)])
+    protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices, default='ssh', verbose_name=_('Protocol'))
     auto_push = models.BooleanField(default=True, verbose_name=_('Auto push'))
     sudo = models.TextField(default='/bin/whoami', verbose_name=_('Sudo'))
-    shell = models.CharField(max_length=64, default='/bin/bash', verbose_name=_('Shell'))
-    login_mode = models.CharField(
-        choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode')
-    )
+    shell = models.CharField(max_length=64,  default='/bin/bash', verbose_name=_('Shell'))
+    login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode'))
     sftp_root = models.CharField(default='tmp', max_length=128, verbose_name=_("SFTP Root"))
     token = models.TextField(default='', verbose_name=_('Token'))
     home = models.CharField(max_length=4096, default='', verbose_name=_('Home'), blank=True)
@@ -269,9 +261,7 @@ class SystemUser(ProtocolMixin, AuthMixin, BaseUser):
     ad_domain = models.CharField(default='', max_length=256)
     # linux su 命令 (switch user)
     su_enabled = models.BooleanField(default=False, verbose_name=_('User switch'))
-    su_from = models.ForeignKey(
-        'self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from")
-    )
+    su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from"))
 
     def __str__(self):
         username = self.username
@@ -331,20 +321,9 @@ class SystemUser(ProtocolMixin, AuthMixin, BaseUser):
         assets = Asset.objects.filter(id__in=asset_ids)
         return assets
 
-    def filter_contain_protocol_assets(self, assets_or_ids):
-        if not assets_or_ids:
-            return assets_or_ids
-        if is_uuid(assets_or_ids[0]):
-            assets = Asset.objects.filter(id__in=assets_or_ids)
-        else:
-            assets = assets_or_ids
-        assets = [asset for asset in assets if self.protocol in asset.protocols_as_dict]
-        return assets
-
     def add_related_assets(self, assets_or_ids):
-        assets = self.filter_contain_protocol_assets(assets_or_ids)
-        self.assets.add(*tuple(assets))
-        self.add_related_assets_to_su_from_if_need(assets)
+        self.assets.add(*tuple(assets_or_ids))
+        self.add_related_assets_to_su_from_if_need(assets_or_ids)
 
     def add_related_assets_to_su_from_if_need(self, assets_or_ids):
         if self.protocol not in [self.Protocol.ssh.value]:

apps/assets/notifications.py

@@ -20,6 +20,6 @@ class AccountBackupExecutionTaskMsg(object):
                  "please go to personal information -> file encryption password to set the encryption password").format(name)
 
     def publish(self, attachment_list=None):
-        send_mail_attachment_async(
+        send_mail_attachment_async.delay(
             self.subject, self.message, [self.user.email], attachment_list
         )

apps/assets/serializers/base.py

@@ -1,24 +1,24 @@
 # -*- coding: utf-8 -*-
 #
+from io import StringIO
 
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
-from assets.models import Type
+from common.utils import ssh_pubkey_gen, ssh_private_key_gen, validate_ssh_private_key
 from common.drf.fields import EncryptedField
-from common.utils import validate_ssh_private_key, parse_ssh_private_key_str, parse_ssh_public_key_str
+from assets.models import Type
 from .utils import validate_password_for_ansible
 
 
 class AuthSerializer(serializers.ModelSerializer):
     password = EncryptedField(required=False, allow_blank=True, allow_null=True, max_length=1024, label=_('Password'))
-    private_key = EncryptedField(required=False, allow_blank=True, allow_null=True, max_length=16384,
-                                 label=_('Private key'))
+    private_key = EncryptedField(required=False, allow_blank=True, allow_null=True, max_length=16384, label=_('Private key'))
 
     def gen_keys(self, private_key=None, password=None):
         if private_key is None:
             return None, None
-        public_key = parse_ssh_public_key_str(text=private_key, password=password)
+        public_key = ssh_pubkey_gen(private_key=private_key, password=password)
         return private_key, public_key
 
     def save(self, **kwargs):
@@ -57,7 +57,10 @@ class AuthSerializerMixin(serializers.ModelSerializer):
         if not valid:
             raise serializers.ValidationError(_("private key invalid or passphrase error"))
 
-        private_key = parse_ssh_private_key_str(private_key, password=passphrase)
+        private_key = ssh_private_key_gen(private_key, password=passphrase)
+        string_io = StringIO()
+        private_key.write_private_key(string_io)
+        private_key = string_io.getvalue()
         return private_key
 
     def validate_public_key(self, public_key):

apps/assets/serializers/cmd_filter.py

@@ -21,7 +21,7 @@ class CommandFilterSerializer(BulkOrgResourceModelSerializer):
             'comment', 'created_by',
         ]
         fields_fk = ['rules']
-        fields_m2m = ['users', 'user_groups', 'system_users', 'nodes', 'assets', 'applications']
+        fields_m2m = ['users', 'user_groups', 'system_users', 'assets', 'applications']
         fields = fields_small + fields_fk + fields_m2m
         extra_kwargs = {
             'rules': {'read_only': True},

apps/assets/serializers/system_user.py

@@ -1,16 +1,16 @@
-from django.db.models import Count
-from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
+from django.utils.translation import ugettext_lazy as _
+from django.db.models import Count
 
+from common.mixins.serializers import BulkSerializerMixin
+from common.utils import ssh_pubkey_gen
 from common.drf.fields import EncryptedField
 from common.drf.serializers import SecretReadableMixin
-from common.mixins.serializers import BulkSerializerMixin
-from common.utils import parse_ssh_public_key_str
 from common.validators import alphanumeric_re, alphanumeric_cn_re, alphanumeric_win_re
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
-from .base import AuthSerializerMixin
-from .utils import validate_password_for_ansible
 from ..models import SystemUser, Asset
+from .utils import validate_password_for_ansible
+from .base import AuthSerializerMixin
 
 __all__ = [
     'SystemUserSerializer', 'MiniSystemUserSerializer',
@@ -36,6 +36,9 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     token = EncryptedField(
         label=_('Token'), required=False, write_only=True, style={'base_template': 'textarea.html'}
     )
+    applications_amount = serializers.IntegerField(
+        source='apps_amount', read_only=True, label=_('Apps amount')
+    )
 
     class Meta:
         model = SystemUser
@@ -50,7 +53,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
             'su_enabled', 'su_from',
             'date_created', 'date_updated', 'comment', 'created_by',
         ]
-        fields_m2m = ['cmd_filters', 'nodes']
+        fields_m2m = ['cmd_filters', 'assets_amount', 'applications_amount', 'nodes']
         fields = fields_small + fields_m2m
         extra_kwargs = {
             'cmd_filters': {"required": False, 'label': _('Command filter')},
@@ -211,7 +214,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         elif attrs.get('private_key'):
             private_key = attrs['private_key']
             password = attrs.get('password')
-            public_key = parse_ssh_public_key_str(private_key, password=password)
+            public_key = ssh_pubkey_gen(private_key, password=password, username=username)
             attrs['public_key'] = public_key
         return attrs
 
@@ -238,6 +241,7 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     def setup_eager_loading(cls, queryset):
         """ Perform necessary eager loading of data. """
         queryset = queryset \
+            .annotate(assets_amount=Count("assets")) \
             .prefetch_related('nodes', 'cmd_filters')
         return queryset
 

apps/assets/serializers/utils.py

@@ -1,8 +1,6 @@
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
-from common.utils import validate_ssh_private_key, parse_ssh_private_key_str
-
 
 def validate_password_for_ansible(password):
     """ 校验 Ansible 不支持的特殊字符 """
@@ -17,9 +15,3 @@ def validate_password_for_ansible(password):
     if '"' in password:
         raise serializers.ValidationError(_('Password can not contains `"` '))
 
-
-def validate_ssh_key(ssh_key, passphrase=None):
-    valid = validate_ssh_private_key(ssh_key, password=passphrase)
-    if not valid:
-        raise serializers.ValidationError(_("private key invalid or passphrase error"))
-    return parse_ssh_private_key_str(ssh_key, passphrase)

apps/assets/signal_handlers/node_assets_mapping.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 #
+import os
+import threading
 
 from django.db.models.signals import (
     m2m_changed, post_save, post_delete
@@ -7,14 +9,14 @@ from django.db.models.signals import (
 from django.dispatch import receiver
 from django.utils.functional import LazyObject
 
-from assets.models import Asset, Node
 from common.signals import django_ready
-from common.utils import get_logger
 from common.utils.connection import RedisPubSub
+from common.utils import get_logger
+from assets.models import Asset, Node
 from orgs.models import Organization
 
-logger = get_logger(__file__)
 
+logger = get_logger(__file__)
 
 # clear node assets mapping for memory
 # ------------------------------------
@@ -76,4 +78,9 @@ def subscribe_node_assets_mapping_expire(sender, **kwargs):
         Node.expire_node_all_asset_ids_mapping_from_memory(org_id)
         Node.expire_node_all_asset_ids_mapping_from_memory(root_org_id)
 
-    node_assets_mapping_for_memory_pub_sub.subscribe(handle_node_relation_change)
+    def keep_subscribe_node_assets_relation():
+        node_assets_mapping_for_memory_pub_sub.subscribe(handle_node_relation_change)
+
+    t = threading.Thread(target=keep_subscribe_node_assets_relation)
+    t.daemon = True
+    t.start()

apps/assets/tasks/utils.py

@@ -50,10 +50,6 @@ def clean_ansible_task_hosts(assets, system_user=None):
     for asset in assets:
         if not check_asset_can_run_ansible(asset):
             continue
-        # 资产平台不包含系统用户的协议, 不推送
-        if system_user and system_user.protocol not in asset.protocols_as_dict:
-            logger.info(_('Asset protocol not support system user protocol, skipped: {}').format(system_user.protocol))
-            continue
         cleaned_assets.append(asset)
     if not cleaned_assets:
         logger.info(_("No assets matched, stop task"))

apps/audits/api.py

@@ -1,29 +1,21 @@
 # -*- coding: utf-8 -*-
 #
-from importlib import import_module
-
-from rest_framework.mixins import ListModelMixin, CreateModelMixin, RetrieveModelMixin
+from rest_framework.mixins import ListModelMixin, CreateModelMixin
 from django.db.models import F, Value
 from django.db.models.functions import Concat
-from django.conf import settings
 from rest_framework.permissions import IsAuthenticated
 from rest_framework import generics
 
 from common.drf.api import JMSReadOnlyModelViewSet
-from common.plugins.es import QuerySet as ESQuerySet
 from common.drf.filters import DatetimeRangeFilter
 from common.api import CommonGenericViewSet
 from orgs.mixins.api import OrgGenericViewSet, OrgBulkModelViewSet, OrgRelationMixin
 from orgs.utils import current_org
 from ops.models import CommandExecution
 from . import filters
-from .backends import TYPE_ENGINE_MAPPING
 from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog
 from .serializers import FTPLogSerializer, UserLoginLogSerializer, CommandExecutionSerializer
-from .serializers import (
-    OperateLogSerializer, OperateLogActionDetailSerializer,
-    PasswordChangeLogSerializer, CommandExecutionHostsRelationSerializer
-)
+from .serializers import OperateLogSerializer, PasswordChangeLogSerializer, CommandExecutionHostsRelationSerializer
 
 
 class FTPLogViewSet(CreateModelMixin,
@@ -76,7 +68,7 @@ class MyLoginLogAPIView(UserLoginCommonMixin, generics.ListAPIView):
         return qs
 
 
-class OperateLogViewSet(RetrieveModelMixin, ListModelMixin, OrgGenericViewSet):
+class OperateLogViewSet(ListModelMixin, OrgGenericViewSet):
     model = OperateLog
     serializer_class = OperateLogSerializer
     extra_filter_backends = [DatetimeRangeFilter]
@@ -87,22 +79,6 @@ class OperateLogViewSet(RetrieveModelMixin, ListModelMixin, OrgGenericViewSet):
     search_fields = ['resource']
     ordering = ['-datetime']
 
-    def get_serializer_class(self):
-        if self.request.query_params.get('type') == 'action_detail':
-            return OperateLogActionDetailSerializer
-        return super().get_serializer_class()
-
-    def get_queryset(self):
-        qs = OperateLog.objects.all()
-        es_config = settings.OPERATE_LOG_ELASTICSEARCH_CONFIG
-        if es_config:
-            engine_mod = import_module(TYPE_ENGINE_MAPPING['es'])
-            store = engine_mod.OperateLogStore(es_config)
-            if store.ping(timeout=2):
-                qs = ESQuerySet(store)
-                qs.model = OperateLog
-        return qs
-
 
 class PasswordChangeLogViewSet(ListModelMixin, CommonGenericViewSet):
     queryset = PasswordChangeLog.objects.all()

apps/audits/backends/__init__.py

@@ -1,18 +0,0 @@
-from importlib import import_module
-
-from django.conf import settings
-
-
-TYPE_ENGINE_MAPPING = {
-    'db': 'audits.backends.db',
-    'es': 'audits.backends.es',
-}
-
-
-def get_operate_log_storage(default=False):
-    engine_mod = import_module(TYPE_ENGINE_MAPPING['db'])
-    es_config = settings.OPERATE_LOG_ELASTICSEARCH_CONFIG
-    if not default and es_config:
-        engine_mod = import_module(TYPE_ENGINE_MAPPING['es'])
-    storage = engine_mod.OperateLogStore(es_config)
-    return storage

apps/audits/backends/db.py

@@ -1,38 +0,0 @@
-# ~*~ coding: utf-8 ~*~
-from django.utils.translation import ugettext_lazy as _
-
-from audits.models import OperateLog
-
-
-class OperateLogStore(object):
-    def __init__(self, config):
-        self.model = OperateLog
-        self.max_length = 1024
-        self.max_length_tip_msg = _(
-            'The text content is too long. Use Elasticsearch to store operation logs'
-        )
-
-    @staticmethod
-    def ping(timeout=None):
-        return True
-
-    def save(self, **kwargs):
-        log_id = kwargs.get('id', '')
-        before = kwargs.get('before') or {}
-        after = kwargs.get('after') or {}
-        if len(str(before)) > self.max_length:
-            before = {_('Tips'): self.max_length_tip_msg}
-        if len(str(after)) > self.max_length:
-            after = {_('Tips'): self.max_length_tip_msg}
-
-        op_log = self.model.objects.filter(pk=log_id).first()
-        if op_log is not None:
-            raw_after = op_log.after or {}
-            raw_before = op_log.before or {}
-            raw_before.update(before)
-            raw_after.update(after)
-            op_log.before = raw_before
-            op_log.after = raw_after
-            op_log.save()
-        else:
-            self.model.objects.create(**kwargs)

apps/audits/backends/es.py

@@ -1,85 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-import uuid
-
-from common.utils.timezone import local_now_display
-from common.utils import get_logger
-from common.utils.encode import Singleton
-from common.plugins.es import ES
-
-
-logger = get_logger(__file__)
-
-
-class OperateLogStore(ES, metaclass=Singleton):
-    def __init__(self, config):
-        properties = {
-            "id": {
-                "type": "keyword"
-            },
-            "user": {
-                "type": "keyword"
-            },
-            "action": {
-                "type": "keyword"
-            },
-            "resource_type": {
-                "type": "keyword"
-            },
-            "org_id": {
-                "type": "keyword"
-            },
-            "datetime": {
-                "type": "date",
-                "format": "yyyy-MM-dd HH:mm:ss"
-            }
-        }
-        exact_fields = {}
-        match_fields = {
-            'id', 'user', 'action', 'resource_type',
-            'resource', 'remote_addr', 'org_id'
-        }
-        keyword_fields = {
-            'id', 'user', 'action', 'resource_type', 'org_id'
-        }
-        if not config.get('INDEX'):
-            config['INDEX'] = 'jumpserver_operate_log'
-        super().__init__(config, properties, keyword_fields, exact_fields, match_fields)
-        self.pre_use_check()
-
-    @staticmethod
-    def make_data(data):
-        op_id = data.get('id', str(uuid.uuid4()))
-        datetime_param = data.get('datetime', local_now_display())
-        data = {
-            'id': op_id, 'user': data['user'], 'action': data['action'],
-            'resource_type': data['resource_type'], 'resource': data['resource'],
-            'remote_addr': data['remote_addr'], 'datetime': datetime_param,
-            'before': data['before'], 'after': data['after'], 'org_id': data['org_id']
-        }
-        return data
-
-    def save(self, **kwargs):
-        log_id = kwargs.get('id', '')
-        before = kwargs.get('before') or {}
-        after = kwargs.get('after') or {}
-
-        op_log = self.get({'id': log_id})
-        if op_log is not None:
-            data = {'doc': {}}
-            raw_after = op_log.get('after') or {}
-            raw_before = op_log.get('before') or {}
-            raw_before.update(before)
-            raw_after.update(after)
-            data['doc']['before'] = raw_before
-            data['doc']['after'] = raw_after
-            self.es.update(
-                index=self.index, doc_type=self.doc_type,
-                id=op_log.get('es_id'), body=data, refresh=True
-            )
-        else:
-            data = self.make_data(kwargs)
-            self.es.index(
-                index=self.index, doc_type=self.doc_type, body=data,
-                refresh=True
-            )

apps/audits/const.py

@@ -7,30 +7,19 @@ DEFAULT_CITY = _("Unknown")
 MODELS_NEED_RECORD = (
     # users
     'User', 'UserGroup',
-    # authentication
-    'AccessKey', 'TempToken',
     # acls
     'LoginACL', 'LoginAssetACL', 'LoginConfirmSetting',
     # assets
     'Asset', 'Node', 'AdminUser', 'SystemUser', 'Domain', 'Gateway', 'CommandFilterRule',
-    'CommandFilter', 'Platform', 'Label',
+    'CommandFilter', 'Platform', 'AuthBook',
     # applications
     'Application',
-    # account
-    'AuthBook',
     # orgs
     'Organization',
     # settings
     'Setting',
     # perms
     'AssetPermission', 'ApplicationPermission',
-    # notifications
-    'SystemMsgSubscription', 'UserMsgSubscription',
-    # Terminal
-    'Terminal', 'Endpoint', 'EndpointRule', 'CommandStorage', 'ReplayStorage',
-    # rbac
-    'Role', 'SystemRole', 'OrgRole', 'RoleBinding', 'OrgRoleBinding', 'SystemRoleBinding',
     # xpack
-    'License', 'Account', 'SyncInstanceTask', 'ChangeAuthPlan', 'ApplicationChangeAuthPlan',
-    'GatherUserTask', 'Interface',
+    'License', 'Account', 'SyncInstanceTask', 'ChangeAuthPlan', 'GatherUserTask',
 )

apps/audits/handler.py

@@ -1,190 +0,0 @@
-from datetime import datetime
-
-from django.db import transaction
-from django.core.cache import cache
-from django.utils.translation import ugettext_lazy as _
-
-from common.utils import get_request_ip_or_data, get_logger
-from common.utils.timezone import as_current_tz
-from common.utils.encode import Singleton
-from common.local import encrypted_field_set
-from settings.serializers import SettingsSerializer
-from jumpserver.utils import current_request
-from audits.models import OperateLog
-from orgs.utils import get_current_org_id
-
-from .backends import get_operate_log_storage
-
-logger = get_logger(__name__)
-
-
-class ModelClient:
-    @staticmethod
-    def save(**kwargs):
-        log_id = kwargs.get('id', '')
-        op_log = OperateLog.objects.filter(pk=log_id).first()
-        if op_log is not None:
-            raw_after = op_log.after or {}
-            raw_before = op_log.before or {}
-            cur_before = kwargs.get('before') or {}
-            cur_after = kwargs.get('after') or {}
-            raw_before.update(cur_before)
-            raw_after.update(cur_after)
-            op_log.before = raw_before
-            op_log.after = raw_after
-            op_log.save()
-        else:
-            OperateLog.objects.create(**kwargs)
-
-
-class OperatorLogHandler(metaclass=Singleton):
-    CACHE_KEY = 'OPERATOR_LOG_CACHE_KEY'
-
-    def __init__(self):
-        self.log_client = self.get_storage_client()
-
-    @staticmethod
-    def get_storage_client():
-        client = get_operate_log_storage()
-        return client
-
-    @staticmethod
-    def _consistent_type_to_str(value1, value2):
-        if isinstance(value1, datetime):
-            value1 = as_current_tz(value1).strftime('%Y-%m-%d %H:%M:%S')
-        if isinstance(value2, datetime):
-            value2 = as_current_tz(value2).strftime('%Y-%m-%d %H:%M:%S')
-        return value1, value2
-
-    def _look_for_two_dict_change(self, left_dict, right_dict):
-        # 以右边的字典为基础
-        before, after = {}, {}
-        for key, value in right_dict.items():
-            pre_value = left_dict.get(key, '')
-            pre_value, value = self._consistent_type_to_str(pre_value, value)
-            if sorted(str(value)) == sorted(str(pre_value)):
-                continue
-            if pre_value:
-                before[key] = pre_value
-            if value:
-                after[key] = value
-        return before, after
-
-    def cache_instance_before_data(self, instance_dict):
-        instance_id = instance_dict.get('id')
-        if instance_id is None:
-            return
-
-        key = '%s_%s' % (self.CACHE_KEY, instance_id)
-        cache.set(key, instance_dict, 3 * 60)
-
-    def get_instance_dict_from_cache(self, instance_id):
-        if instance_id is None:
-            return None
-
-        key = '%s_%s' % (self.CACHE_KEY, instance_id)
-        cache_instance = cache.get(key, {})
-        log_id = cache_instance.get('operate_log_id')
-        return log_id, cache_instance
-
-    def get_instance_current_with_cache_diff(self, current_instance):
-        log_id, before, after = None, None, None
-        instance_id = current_instance.get('id')
-        if instance_id is None:
-            return log_id, before, after
-
-        log_id, cache_instance = self.get_instance_dict_from_cache(instance_id)
-        if not cache_instance:
-            return log_id, before, after
-
-        before, after = self._look_for_two_dict_change(
-            cache_instance, current_instance
-        )
-        return log_id, before, after
-
-    @staticmethod
-    def get_resource_display_from_setting(resource):
-        resource_display = None
-        setting_serializer = SettingsSerializer()
-        label = setting_serializer.get_field_label(resource)
-        if label is not None:
-            resource_display = label
-        return resource_display
-
-    def get_resource_display(self, resource):
-        resource_display = str(resource)
-        return_value = self.get_resource_display_from_setting(resource_display)
-        if return_value is not None:
-            resource_display = return_value
-        return resource_display
-
-    @staticmethod
-    def serialized_value(value: (list, tuple)):
-        if len(value) == 0:
-            return ''
-        if isinstance(value[0], str):
-            return ','.join(value)
-        return ','.join([i['value'] for i in value if i.get('value')])
-
-    def __data_processing(self, dict_item, loop=True):
-        encrypt_value = '******'
-        for key, value in dict_item.items():
-            if isinstance(value, bool):
-                value = _('Yes') if value else _('No')
-            elif isinstance(value, (list, tuple)):
-                value = self.serialized_value(value)
-            elif isinstance(value, dict) and loop:
-                self.__data_processing(value, loop=False)
-            if key in encrypted_field_set:
-                value = encrypt_value
-            dict_item[key] = value
-        return dict_item
-
-    def data_processing(self, before, after):
-        if before:
-            before = self.__data_processing(before)
-        if after:
-            after = self.__data_processing(after)
-        return before, after
-
-    def create_or_update_operate_log(
-            self, action, resource_type, resource=None,
-            force=False, log_id=None, before=None, after=None
-    ):
-        user = current_request.user if current_request else None
-        if not user or not user.is_authenticated:
-            return
-
-        remote_addr = get_request_ip_or_data(current_request)
-        resource_display = self.get_resource_display(resource)
-        before, after = self.data_processing(before, after)
-        if not force and not any([before, after]):
-            # 前后都没变化，没必要生成日志，除非手动强制保存
-            return
-
-        data = {
-            'id': log_id, "user": str(user), 'action': action,
-            'resource_type': str(resource_type), 'resource': resource_display,
-            'remote_addr': remote_addr, 'before': before, 'after': after,
-            'org_id': get_current_org_id(),
-        }
-        with transaction.atomic():
-            if self.log_client.ping(timeout=1):
-                client = self.log_client
-            else:
-                logger.info('Switch default operate log storage save.')
-                client = get_operate_log_storage(default=True)
-
-            try:
-                client.save(**data)
-            except Exception as e:
-                error_msg = 'An error occurred saving OperateLog.' \
-                            'Error: %s, Data: %s' % (e, data)
-                logger.error(error_msg)
-
-
-op_handler = OperatorLogHandler()
-create_or_update_operate_log = op_handler.create_or_update_operate_log
-cache_instance_before_data = op_handler.cache_instance_before_data
-get_instance_current_with_cache_diff = op_handler.get_instance_current_with_cache_diff
-get_instance_dict_from_cache = op_handler.get_instance_dict_from_cache

apps/audits/migrations/0015_auto_20221011_1745.py

@@ -1,24 +0,0 @@
-# Generated by Django 3.2.14 on 2022-10-11 09:45
-
-import common.db.encoder
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('audits', '0014_auto_20220505_1902'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='operatelog',
-            name='after',
-            field=models.JSONField(default=dict, encoder=common.db.encoder.ModelJSONFieldEncoder, null=True),
-        ),
-        migrations.AddField(
-            model_name='operatelog',
-            name='before',
-            field=models.JSONField(default=dict, encoder=common.db.encoder.ModelJSONFieldEncoder, null=True),
-        ),
-    ]

apps/audits/models.py

@@ -4,9 +4,8 @@ from django.db import models
 from django.db.models import Q
 from django.utils.translation import gettext, ugettext_lazy as _
 from django.utils import timezone
-
 from common.utils import lazyproperty
-from common.db.encoder import ModelJSONFieldEncoder
+
 from orgs.mixins.models import OrgModelMixin, Organization
 from orgs.utils import current_org
 
@@ -66,8 +65,6 @@ class OperateLog(OrgModelMixin):
     resource = models.CharField(max_length=128, verbose_name=_("Resource"))
     remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     datetime = models.DateTimeField(auto_now=True, verbose_name=_('Datetime'), db_index=True)
-    before = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True)
-    after = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True)
 
     def __str__(self):
         return "<{}> {} <{}>".format(self.user, self.action, self.resource)
@@ -81,21 +78,6 @@ class OperateLog(OrgModelMixin):
             self.org_id = Organization.ROOT_ID
         return super(OperateLog, self).save(*args, **kwargs)
 
-    @classmethod
-    def from_dict(cls, d):
-        self = cls()
-        for k, v in d.items():
-            setattr(self, k, v)
-        return self
-
-    @classmethod
-    def from_multi_dict(cls, l):
-        operate_logs = []
-        for d in l:
-            operate_log = cls.from_dict(d)
-            operate_logs.append(operate_log)
-        return operate_logs
-
     class Meta:
         verbose_name = _("Operate log")
 

apps/audits/serializers.py

@@ -47,12 +47,6 @@ class UserLoginLogSerializer(serializers.ModelSerializer):
         }
 
 
-class OperateLogActionDetailSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = models.OperateLog
-        fields = ('before', 'after')
-
-
 class OperateLogSerializer(serializers.ModelSerializer):
     action_display = serializers.CharField(source='get_action_display', label=_('Action'))
 

apps/audits/signal_handlers.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 #
-import uuid
+import time
 
 from django.db.models.signals import (
-    post_save, m2m_changed, pre_delete, pre_save
+    post_save, m2m_changed, pre_delete
 )
 from django.dispatch import receiver
 from django.conf import settings
@@ -16,29 +16,22 @@ from django.utils import translation
 from rest_framework.renderers import JSONRenderer
 from rest_framework.request import Request
 
-from users.models import User
-from assets.models import Asset, SystemUser, CommandFilter
-from terminal.models import Session, Command
-from perms.models import AssetPermission, ApplicationPermission
-from rbac.models import Role
-
-from audits.utils import model_to_dict_for_operate_log as model_to_dict
-from audits.handler import (
-    get_instance_current_with_cache_diff, cache_instance_before_data,
-    create_or_update_operate_log, get_instance_dict_from_cache
-)
+from assets.models import Asset, SystemUser
 from authentication.signals import post_auth_failed, post_auth_success
 from authentication.utils import check_different_city_login_if_need
 from jumpserver.utils import current_request
+from users.models import User
 from users.signals import post_user_change_password
-from .utils import write_login_log
+from terminal.models import Session, Command
+from .utils import write_login_log, create_operate_log
 from . import models, serializers
 from .models import OperateLog
-from .const import MODELS_NEED_RECORD
+from orgs.utils import current_org
+from perms.models import AssetPermission, ApplicationPermission
 from terminal.backends.command.serializers import SessionCommandSerializer
 from terminal.serializers import SessionSerializer
-from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR, SKIP_SIGNAL
-from common.utils import get_request_ip_or_data, get_logger, get_syslogger
+from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR
+from common.utils import get_request_ip, get_logger, get_syslogger
 from common.utils.encode import data_to_json
 
 logger = get_logger(__name__)
@@ -69,6 +62,70 @@ class AuthBackendLabelMapping(LazyObject):
 
 AUTH_BACKEND_LABEL_MAPPING = AuthBackendLabelMapping()
 
+
+M2M_NEED_RECORD = {
+    User.groups.through._meta.object_name: (
+        _('User and Group'),
+        _('{User} JOINED {UserGroup}'),
+        _('{User} LEFT {UserGroup}')
+    ),
+    SystemUser.assets.through._meta.object_name: (
+        _('Asset and SystemUser'),
+        _('{Asset} ADD {SystemUser}'),
+        _('{Asset} REMOVE {SystemUser}')
+    ),
+    Asset.nodes.through._meta.object_name: (
+        _('Node and Asset'),
+        _('{Node} ADD {Asset}'),
+        _('{Node} REMOVE {Asset}')
+    ),
+    AssetPermission.users.through._meta.object_name: (
+        _('User asset permissions'),
+        _('{AssetPermission} ADD {User}'),
+        _('{AssetPermission} REMOVE {User}'),
+    ),
+    AssetPermission.user_groups.through._meta.object_name: (
+        _('User group asset permissions'),
+        _('{AssetPermission} ADD {UserGroup}'),
+        _('{AssetPermission} REMOVE {UserGroup}'),
+    ),
+    AssetPermission.assets.through._meta.object_name: (
+        _('Asset permission'),
+        _('{AssetPermission} ADD {Asset}'),
+        _('{AssetPermission} REMOVE {Asset}'),
+    ),
+    AssetPermission.nodes.through._meta.object_name: (
+        _('Node permission'),
+        _('{AssetPermission} ADD {Node}'),
+        _('{AssetPermission} REMOVE {Node}'),
+    ),
+    AssetPermission.system_users.through._meta.object_name: (
+        _('Asset permission and SystemUser'),
+        _('{AssetPermission} ADD {SystemUser}'),
+        _('{AssetPermission} REMOVE {SystemUser}'),
+    ),
+    ApplicationPermission.users.through._meta.object_name: (
+        _('User application permissions'),
+        _('{ApplicationPermission} ADD {User}'),
+        _('{ApplicationPermission} REMOVE {User}'),
+    ),
+    ApplicationPermission.user_groups.through._meta.object_name: (
+        _('User group application permissions'),
+        _('{ApplicationPermission} ADD {UserGroup}'),
+        _('{ApplicationPermission} REMOVE {UserGroup}'),
+    ),
+    ApplicationPermission.applications.through._meta.object_name: (
+        _('Application permission'),
+        _('{ApplicationPermission} ADD {Application}'),
+        _('{ApplicationPermission} REMOVE {Application}'),
+    ),
+    ApplicationPermission.system_users.through._meta.object_name: (
+        _('Application permission and SystemUser'),
+        _('{ApplicationPermission} ADD {SystemUser}'),
+        _('{ApplicationPermission} REMOVE {SystemUser}'),
+    ),
+}
+
 M2M_ACTION = {
     POST_ADD: OperateLog.ACTION_CREATE,
     POST_REMOVE: OperateLog.ACTION_DELETE,
@@ -80,115 +137,60 @@ M2M_ACTION = {
 def on_m2m_changed(sender, action, instance, reverse, model, pk_set, **kwargs):
     if action not in M2M_ACTION:
         return
-    if not instance:
+
+    user = current_request.user if current_request else None
+    if not user or not user.is_authenticated:
         return
 
-    resource_type = instance._meta.verbose_name
-    current_instance = model_to_dict(instance, include_model_fields=False)
+    sender_name = sender._meta.object_name
+    if sender_name in M2M_NEED_RECORD:
+        org_id = current_org.id
+        remote_addr = get_request_ip(current_request)
+        user = str(user)
+        resource_type, resource_tmpl_add, resource_tmpl_remove = M2M_NEED_RECORD[sender_name]
+        action = M2M_ACTION[action]
+        if action == OperateLog.ACTION_CREATE:
+            resource_tmpl = resource_tmpl_add
+        elif action == OperateLog.ACTION_DELETE:
+            resource_tmpl = resource_tmpl_remove
 
-    instance_id = current_instance.get('id')
-    log_id, before_instance = get_instance_dict_from_cache(instance_id)
+        to_create = []
+        objs = model.objects.filter(pk__in=pk_set)
 
-    field_name = str(model._meta.verbose_name)
-    objs = model.objects.filter(pk__in=pk_set)
-    objs_display = [str(o) for o in objs]
-    action = M2M_ACTION[action]
-    changed_field = current_instance.get(field_name, [])
+        instance_name = instance._meta.object_name
+        instance_value = str(instance)
 
-    after, before, before_value = None, None, None
-    if action == OperateLog.ACTION_CREATE:
-        before_value = list(set(changed_field) - set(objs_display))
-    elif action == OperateLog.ACTION_DELETE:
-        before_value = list(
-            set(changed_field).symmetric_difference(set(objs_display))
-        )
+        model_name = model._meta.object_name
 
-    if changed_field:
-        after = {field_name: changed_field}
-    if before_value:
-        before = {field_name: before_value}
+        for obj in objs:
+            resource = resource_tmpl.format(**{
+                instance_name: instance_value,
+                model_name: str(obj)
+            })[:128]  # `resource` 字段只有 128 个字符长 😔
 
-    if sorted(str(before)) == sorted(str(after)):
-        return
-
-    create_or_update_operate_log(
-        OperateLog.ACTION_UPDATE, resource_type,
-        resource=instance, log_id=log_id, before=before, after=after
-    )
-
-
-def signal_of_operate_log_whether_continue(sender, instance, created, update_fields=None):
-    condition = True
-    if not instance:
-        condition = False
-    if instance and getattr(instance, SKIP_SIGNAL, False):
-        condition = False
-    # 终端模型的 create 事件由系统产生，不记录
-    if instance._meta.object_name == 'Terminal' and created:
-        condition = False
-    # last_login 改变是最后登录日期, 每次登录都会改变
-    if instance._meta.object_name == 'User' and \
-            update_fields and 'last_login' in update_fields:
-        condition = False
-    # 不在记录白名单中，跳过
-    if sender._meta.object_name not in MODELS_NEED_RECORD:
-        condition = False
-    return condition
-
-
-@receiver(pre_save)
-def on_object_pre_create_or_update(sender, instance=None, raw=False, using=None, update_fields=None, **kwargs):
-    ok = signal_of_operate_log_whether_continue(
-        sender, instance, False, update_fields
-    )
-    if not ok:
-        return
-    instance_before_data = {'id': instance.id}
-    raw_instance = type(instance).objects.filter(pk=instance.id).first()
-    if raw_instance:
-        instance_before_data = model_to_dict(raw_instance)
-    operate_log_id = str(uuid.uuid4())
-    instance_before_data['operate_log_id'] = operate_log_id
-    setattr(instance, 'operate_log_id', operate_log_id)
-    cache_instance_before_data(instance_before_data)
+            to_create.append(OperateLog(
+                user=user, action=action, resource_type=resource_type,
+                resource=resource, remote_addr=remote_addr, org_id=org_id
+            ))
+        OperateLog.objects.bulk_create(to_create)
 
 
 @receiver(post_save)
 def on_object_created_or_update(sender, instance=None, created=False, update_fields=None, **kwargs):
-    ok = signal_of_operate_log_whether_continue(
-        sender, instance, created, update_fields
-    )
-    if not ok:
+    # last_login 改变是最后登录日期, 每次登录都会改变
+    if instance._meta.object_name == 'User' and \
+            update_fields and 'last_login' in update_fields:
         return
-
-    log_id, before, after = None, None, None
     if created:
         action = models.OperateLog.ACTION_CREATE
-        after = model_to_dict(instance)
-        log_id = getattr(instance, 'operate_log_id', None)
     else:
         action = models.OperateLog.ACTION_UPDATE
-        current_instance = model_to_dict(instance)
-        log_id, before, after = get_instance_current_with_cache_diff(current_instance)
-
-    resource_type = sender._meta.verbose_name
-    create_or_update_operate_log(
-        action, resource_type, resource=instance,
-        log_id=log_id, before=before, after=after
-    )
+    create_operate_log(action, sender, instance)
 
 
 @receiver(pre_delete)
 def on_object_delete(sender, instance=None, **kwargs):
-    ok = signal_of_operate_log_whether_continue(sender, instance, False)
-    if not ok:
-        return
-
-    resource_type = sender._meta.verbose_name
-    create_or_update_operate_log(
-        models.OperateLog.ACTION_DELETE, resource_type,
-        resource=instance, before=model_to_dict(instance)
-    )
+    create_operate_log(models.OperateLog.ACTION_DELETE, sender, instance)
 
 
 @receiver(post_user_change_password, sender=User)
@@ -197,7 +199,7 @@ def on_user_change_password(sender, user=None, **kwargs):
         remote_addr = '127.0.0.1'
         change_by = 'System'
     else:
-        remote_addr = get_request_ip_or_data(current_request)
+        remote_addr = get_request_ip(current_request)
         if not current_request.user.is_authenticated:
             change_by = str(user)
         else:
@@ -249,7 +251,7 @@ def get_login_backend(request):
 
 def generate_data(username, request, login_type=None):
     user_agent = request.META.get('HTTP_USER_AGENT', '')
-    login_ip = get_request_ip_or_data(request) or '0.0.0.0'
+    login_ip = get_request_ip(request) or '0.0.0.0'
 
     if login_type is None and isinstance(request, Request):
         login_type = request.META.get('HTTP_X_JMS_LOGIN_TYPE', 'U')

apps/audits/utils.py

@@ -1,16 +1,14 @@
 import csv
 import codecs
 
-from itertools import chain
-from datetime import datetime
-
 from django.http import HttpResponse
-from django.db import models
+from django.db import transaction
+from django.utils import translation
 
-from settings.serializers import SettingsSerializer
-from common.utils.timezone import as_current_tz
-from common.utils import validate_ip, get_ip_city, get_logger
-from .const import DEFAULT_CITY
+from audits.models import OperateLog
+from common.utils import validate_ip, get_ip_city, get_request_ip, get_logger
+from jumpserver.utils import current_request
+from .const import DEFAULT_CITY, MODELS_NEED_RECORD
 
 
 logger = get_logger(__name__)
@@ -48,59 +46,23 @@ def write_login_log(*args, **kwargs):
     UserLoginLog.objects.create(**kwargs)
 
 
-def get_resource_display(resource):
-    resource_display = str(resource)
-    setting_serializer = SettingsSerializer()
-    label = setting_serializer.get_field_label(resource_display)
-    if label is not None:
-        resource_display = label
-    return resource_display
+def create_operate_log(action, sender, resource):
+    user = current_request.user if current_request else None
+    if not user or not user.is_authenticated:
+        return
+    model_name = sender._meta.object_name
+    if model_name not in MODELS_NEED_RECORD:
+        return
+    with translation.override('en'):
+        resource_type = sender._meta.verbose_name
+    remote_addr = get_request_ip(current_request)
 
-
-def model_to_dict_for_operate_log(
-        instance, include_model_fields=True, include_related_fields=True
-):
-    model_need_continue_fields = ['date_updated']
-    m2m_need_continue_fields = ['history_passwords']
-    opts = instance._meta
-    data = {}
-    for f in chain(opts.concrete_fields, opts.private_fields):
-        if isinstance(f, (models.FileField, models.ImageField)):
-            continue
-
-        if getattr(f, 'attname', None) in model_need_continue_fields:
-            continue
-
-        value = getattr(instance, f.name) or getattr(instance, f.attname)
-        if not isinstance(value, bool) and not value:
-            continue
-
-        if getattr(f, 'primary_key', False):
-            f.verbose_name = 'id'
-        elif isinstance(value, list):
-            value = [str(v) for v in value]
-        elif isinstance(value, datetime):
-            value = as_current_tz(value).strftime('%Y-%m-%d %H:%M:%S')
-
-        if include_model_fields or getattr(f, 'primary_key', False):
-            data[str(f.verbose_name)] = value
-
-    if include_related_fields:
-        for f in chain(opts.many_to_many, opts.related_objects):
-            value = []
-            if instance.pk is not None:
-                related_name = getattr(f, 'attname', '') or getattr(f, 'related_name', '')
-                if not related_name or related_name in m2m_need_continue_fields:
-                    continue
-                try:
-                    value = [str(i) for i in getattr(instance, related_name).all()]
-                except:
-                    pass
-            if not value:
-                continue
-            try:
-                field_key = getattr(f, 'verbose_name', None) or f.related_model._meta.verbose_name
-                data[str(field_key)] = value
-            except:
-                pass
-    return data
+    data = {
+        "user": str(user), 'action': action, 'resource_type': resource_type,
+        'resource': str(resource), 'remote_addr': remote_addr,
+    }
+    with transaction.atomic():
+        try:
+            OperateLog.objects.create(**data)
+        except Exception as e:
+            logger.error("Create operate log error: {}".format(e))

apps/authentication/api/connection_token.py

@@ -1,27 +1,26 @@
 import abc
-import base64
-import json
 import os
+import json
+import base64
 import urllib.parse
-
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
-from rest_framework import status
-from rest_framework.decorators import action
 from rest_framework.exceptions import PermissionDenied
-from rest_framework.request import Request
+from rest_framework.decorators import action
 from rest_framework.response import Response
+from rest_framework import status
+from rest_framework.request import Request
 
 from common.drf.api import JMSModelViewSet
 from common.http import is_true
 from orgs.mixins.api import RootOrgViewMixin
 from perms.models.base import Action
 from terminal.models import EndpointRule
-from ..models import ConnectionToken
 from ..serializers import (
     ConnectionTokenSerializer, ConnectionTokenSecretSerializer,
     SuperConnectionTokenSerializer, ConnectionTokenDisplaySerializer,
 )
+from ..models import ConnectionToken
 
 __all__ = ['ConnectionTokenViewSet', 'SuperConnectionTokenViewSet']
 
@@ -63,15 +62,12 @@ class ConnectionTokenMixin:
 
     def get_smart_endpoint(self, protocol, asset=None, application=None):
         if asset:
-            target_instance = asset
             target_ip = asset.get_target_ip()
         elif application:
-            target_instance = application
             target_ip = application.get_target_ip()
         else:
-            target_instance = None
             target_ip = ''
-        endpoint = EndpointRule.match_endpoint(target_instance, target_ip, protocol, self.request)
+        endpoint = EndpointRule.match_endpoint(target_ip, protocol, self.request)
         return endpoint
 
     @staticmethod
@@ -89,7 +85,7 @@ class ConnectionTokenMixin:
             filename, ssh_token = self.get_ssh_token(token)
         else:
             raise ValueError('Protocol not support: {}'.format(protocol))
-        filename = urllib.parse.unquote(filename)
+
         return {
             "filename": filename,
             "protocol": protocol,
@@ -125,7 +121,6 @@ class ConnectionTokenMixin:
             'bookmarktype:i': '3',
             'use redirection server name:i': '0',
             'smart sizing:i': '1',
-            'disableconnectionsharing:i': '1',
             # 'drivestoredirect:s': '*',
             # 'domain:s': ''
             # 'alternate shell:s:': '||MySQLWorkbench',
@@ -167,9 +162,6 @@ class ConnectionTokenMixin:
         rdp_options['session bpp:i'] = os.getenv('JUMPSERVER_COLOR_DEPTH', '32')
         rdp_options['audiomode:i'] = self.parse_env_bool('JUMPSERVER_DISABLE_AUDIO', 'false', '2', '0')
 
-        if token.asset and token.asset.platform.meta.get('console', None) == 'true':
-            rdp_options['administrative session:i:'] = '1'
-
         if token.asset:
             name = token.asset.hostname
         elif token.application and token.application.category_remote_app:

apps/authentication/api/login_confirm.py

@@ -20,7 +20,6 @@ class TicketStatusApi(mixins.AuthMixin, APIView):
         try:
             self.check_user_login_confirm()
             self.request.session['auth_third_party_done'] = 1
-            self.request.session.pop('auth_third_party_required', '')
             return Response({"msg": "ok"})
         except errors.LoginConfirmOtherError as e:
             reason = e.msg

apps/authentication/api/password.py

@@ -1,71 +1,13 @@
 from rest_framework.generics import CreateAPIView
 from rest_framework.response import Response
-from rest_framework.permissions import AllowAny
-from django.utils.translation import ugettext as _
-from django.template.loader import render_to_string
-from django.core.cache import cache
-from django.shortcuts import reverse
 
-from common.utils.verify_code import SendAndVerifyCodeUtil
+from authentication.serializers import PasswordVerifySerializer
 from common.permissions import IsValidUser
-from common.utils.random import random_string
-from common.utils import get_object_or_none
-from authentication.serializers import (
-    PasswordVerifySerializer, ResetPasswordCodeSerializer
-)
-from settings.utils import get_login_title
-from users.models import User
 from authentication.mixins import authenticate
 from authentication.errors import PasswordInvalid
 from authentication.mixins import AuthMixin
 
 
-class UserResetPasswordSendCodeApi(CreateAPIView):
-    permission_classes = (AllowAny,)
-    serializer_class = ResetPasswordCodeSerializer
-
-    @staticmethod
-    def is_valid_user(**kwargs):
-        user = get_object_or_none(User, **kwargs)
-        if not user:
-            err_msg = _('User does not exist: {}').format(_("No user matched"))
-            return None, err_msg
-        if not user.is_local:
-            err_msg = _(
-                'The user is from {}, please go to the corresponding system to change the password'
-            ).format(user.get_source_display())
-            return None, err_msg
-        return user, None
-
-    def create(self, request, *args, **kwargs):
-        token = request.GET.get('token')
-        userinfo = cache.get(token)
-        if not userinfo:
-            return reverse('authentication:forgot-previewing')
-
-        serializer = self.get_serializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-        username = userinfo.get('username')
-        form_type = serializer.validated_data['form_type']
-        code = random_string(6, lower=False, upper=False)
-        other_args = {}
-
-        target = serializer.validated_data[form_type]
-        query_key = 'phone' if form_type == 'sms' else form_type
-        user, err = self.is_valid_user(username=username, **{query_key: target})
-        if not user:
-            return Response({'error': err}, status=400)
-
-        subject = '%s: %s' % (get_login_title(), _('Forgot password'))
-        context = {
-            'user': user, 'title': subject, 'code': code,
-        }
-        message = render_to_string('authentication/_msg_reset_password_code.html', context)
-        other_args['subject'], other_args['message'] = subject, message
-        SendAndVerifyCodeUtil(target, code, backend=form_type, **other_args).gen_and_send_async()
-        return Response({'data': 'ok'}, status=200)
-
-
 class UserPasswordVerifyApi(AuthMixin, CreateAPIView):
     permission_classes = (IsValidUser,)
     serializer_class = PasswordVerifySerializer

apps/authentication/backends/oauth2/backends.py

@@ -2,7 +2,6 @@
 #
 import requests
 
-from django.utils.translation import gettext_lazy as _
 from django.contrib.auth import get_user_model
 from django.utils.http import urlencode
 from django.conf import settings
@@ -91,12 +90,8 @@ class OAuth2Backend(JMSModelBackend):
                 request, path=reverse(settings.AUTH_OAUTH2_AUTH_LOGIN_CALLBACK_URL_NAME)
             )
         }
-        if '?' in settings.AUTH_OAUTH2_ACCESS_TOKEN_ENDPOINT:
-            separator = '&'
-        else:
-            separator = '?'
-        access_token_url = '{url}{separator}{query}'.format(
-            url=settings.AUTH_OAUTH2_ACCESS_TOKEN_ENDPOINT, separator=separator, query=urlencode(query_dict)
+        access_token_url = '{url}?{query}'.format(
+            url=settings.AUTH_OAUTH2_ACCESS_TOKEN_ENDPOINT, query=urlencode(query_dict)
         )
         token_method = settings.AUTH_OAUTH2_ACCESS_TOKEN_METHOD.lower()
         requests_func = getattr(requests, token_method, requests.get)
@@ -119,16 +114,12 @@ class OAuth2Backend(JMSModelBackend):
 
         headers = {
             'Accept': 'application/json',
-            'Authorization': 'Bearer {}'.format(response_data.get('access_token', ''))
+            'Authorization': 'token {}'.format(response_data.get('access_token', ''))
         }
 
         logger.debug(log_prompt.format('Get userinfo endpoint'))
-        if '?' in settings.AUTH_OAUTH2_PROVIDER_USERINFO_ENDPOINT:
-            separator = '&'
-        else:
-            separator = '?'
-        userinfo_url = '{url}{separator}{query}'.format(
-            url=settings.AUTH_OAUTH2_PROVIDER_USERINFO_ENDPOINT, separator=separator,
+        userinfo_url = '{url}?{query}'.format(
+            url=settings.AUTH_OAUTH2_PROVIDER_USERINFO_ENDPOINT,
             query=urlencode(query_dict)
         )
         userinfo_response = requests.get(userinfo_url, headers=headers)

apps/authentication/backends/oauth2/views.py

@@ -26,13 +26,8 @@ class OAuth2AuthRequestView(View):
             )
         }
 
-        if '?' in settings.AUTH_OAUTH2_PROVIDER_AUTHORIZATION_ENDPOINT:
-            separator = '&'
-        else:
-            separator = '?'
-        redirect_url = '{url}{separator}{query}'.format(
+        redirect_url = '{url}?{query}'.format(
             url=settings.AUTH_OAUTH2_PROVIDER_AUTHORIZATION_ENDPOINT,
-            separator=separator,
             query=urlencode(query_dict)
         )
         logger.debug(log_prompt.format('Redirect login url'))

apps/authentication/backends/oidc/backends.py

@@ -88,7 +88,7 @@ class OIDCAuthCodeBackend(OIDCBaseBackend):
     """
 
     @ssl_verification
-    def authenticate(self, request, nonce=None, code_verifier=None, **kwargs):
+    def authenticate(self, request, nonce=None, **kwargs):
         """ Authenticates users in case of the OpenID Connect Authorization code flow. """
         log_prompt = "Process authenticate [OIDCAuthCodeBackend]: {}"
         logger.debug(log_prompt.format('start'))
@@ -134,8 +134,6 @@ class OIDCAuthCodeBackend(OIDCBaseBackend):
                 request, path=reverse(settings.AUTH_OPENID_AUTH_LOGIN_CALLBACK_URL_NAME)
             )
         }
-        if settings.AUTH_OPENID_PKCE and code_verifier:
-            token_payload['code_verifier'] = code_verifier
         if settings.AUTH_OPENID_CLIENT_AUTH_METHOD == 'client_secret_post':
             token_payload.update({
                 'client_id': settings.AUTH_OPENID_CLIENT_ID,

apps/authentication/backends/oidc/views.py

@@ -9,10 +9,7 @@
 
 """
 
-import base64
-import hashlib
 import time
-import secrets
 
 from django.conf import settings
 from django.contrib import auth
@@ -41,19 +38,6 @@ class OIDCAuthRequestView(View):
 
     http_method_names = ['get', ]
 
-    @staticmethod
-    def gen_code_verifier(length=128):
-        # length range 43 ~ 128
-        return secrets.token_urlsafe(length-32)
-
-    @staticmethod
-    def gen_code_challenge(code_verifier, code_challenge_method):
-        if code_challenge_method == 'plain':
-            return code_verifier
-        h = hashlib.sha256(code_verifier.encode('ascii')).digest()
-        b = base64.urlsafe_b64encode(h)
-        return b.decode('ascii')[:-1]
-
     def get(self, request):
         """ Processes GET requests. """
 
@@ -72,16 +56,6 @@ class OIDCAuthRequestView(View):
             )
         })
 
-        if settings.AUTH_OPENID_PKCE:
-            code_verifier = self.gen_code_verifier()
-            code_challenge_method = settings.AUTH_OPENID_CODE_CHALLENGE_METHOD or 'S256'
-            code_challenge = self.gen_code_challenge(code_verifier, code_challenge_method)
-            authentication_request_params.update({
-                'code_challenge_method': code_challenge_method,
-                'code_challenge': code_challenge
-            })
-            request.session['oidc_auth_code_verifier'] = code_verifier
-
         # States should be used! They are recommended in order to maintain state between the
         # authentication request and the callback.
         if settings.AUTH_OPENID_USE_STATE:
@@ -164,9 +138,8 @@ class OIDCAuthCallbackView(View):
 
             # Authenticates the end-user.
             next_url = request.session.get('oidc_auth_next_url', None)
-            code_verifier = request.session.get('oidc_auth_code_verifier', None)
             logger.debug(log_prompt.format('Process authenticate'))
-            user = auth.authenticate(nonce=nonce, request=request, code_verifier=code_verifier)
+            user = auth.authenticate(nonce=nonce, request=request)
             if user and user.is_valid:
                 logger.debug(log_prompt.format('Login: {}'.format(user)))
                 auth.login(self.request, user)

apps/authentication/backends/pubkey.py

@@ -1,9 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-from django.conf import settings
 from django.contrib.auth import get_user_model
+from django.conf import settings
 
-from common.permissions import ServiceAccountSignaturePermission
 from .base import JMSBaseAuthBackend
 
 UserModel = get_user_model()
@@ -19,10 +18,6 @@ class PublicKeyAuthBackend(JMSBaseAuthBackend):
     def authenticate(self, request, username=None, public_key=None, **kwargs):
         if not public_key:
             return None
-
-        permission = ServiceAccountSignaturePermission()
-        if not permission.has_permission(request, None):
-            return None
         if username is None:
             username = kwargs.get(UserModel.USERNAME_FIELD)
         try:
@@ -31,7 +26,7 @@ class PublicKeyAuthBackend(JMSBaseAuthBackend):
             return None
         else:
             if user.check_public_key(public_key) and \
-                    self.user_can_authenticate(user):
+                  self.user_can_authenticate(user):
                 return user
 
     def get_user(self, user_id):

apps/authentication/backends/saml2/views.py

@@ -271,10 +271,7 @@ class Saml2AuthCallbackView(View, PrepareRequestMixin):
             auth.login(self.request, user)
 
         logger.debug(log_prompt.format('Redirect'))
-        redir = post_data.get('RelayState')
-        if not redir or len(redir) == 0:
-          redir = "/"
-        next_url = saml_instance.redirect_to(redir)
+        next_url = saml_instance.redirect_to(post_data.get('RelayState', '/'))
         return HttpResponseRedirect(next_url)
 
     @csrf_exempt

apps/authentication/const.py

@@ -2,7 +2,7 @@ from django.db.models import TextChoices
 
 from authentication.confirm import CONFIRM_BACKENDS
 from .confirm import ConfirmMFA, ConfirmPassword, ConfirmReLogin
-from .mfa import MFAOtp, MFASms, MFARadius, MFACustom
+from .mfa import MFAOtp, MFASms, MFARadius
 
 RSA_PRIVATE_KEY = 'rsa_private_key'
 RSA_PUBLIC_KEY = 'rsa_public_key'
@@ -35,4 +35,3 @@ class MFAType(TextChoices):
     OTP = MFAOtp.name, MFAOtp.display_name
     SMS = MFASms.name, MFASms.display_name
     Radius = MFARadius.name, MFARadius.display_name
-    Custom = MFACustom.name, MFACustom.display_name

apps/authentication/forms.py

@@ -62,7 +62,7 @@ class CustomCaptchaTextInput(CaptchaTextInput):
 
 
 class CaptchaMixin(forms.Form):
-    captcha = CaptchaField(widget=CustomCaptchaTextInput, label=_('Captcha'))
+    captcha = CaptchaField(widget=CustomCaptchaTextInput)
 
 
 class ChallengeMixin(forms.Form):

apps/authentication/mfa/__init__.py

@@ -1,4 +1,5 @@
 from .otp import MFAOtp, otp_failed_msg
 from .sms import MFASms
 from .radius import MFARadius
-from .custom import MFACustom
+
+MFA_BACKENDS = [MFAOtp, MFASms, MFARadius]

apps/authentication/mfa/custom.py

@@ -1,59 +0,0 @@
-from django.conf import settings
-from django.utils.module_loading import import_string
-from django.utils.translation import ugettext_lazy as _
-
-from common.utils import get_logger
-from .base import BaseMFA
-
-logger = get_logger(__file__)
-
-mfa_custom_method = None
-
-if settings.MFA_CUSTOM:
-    """ 保证自定义认证方法在服务运行时不能被更改，只在第一次调用时加载一次 """
-    try:
-        mfa_custom_method_path = 'data.mfa.main.check_code'
-        mfa_custom_method = import_string(mfa_custom_method_path)
-    except Exception as e:
-        logger.warning('Import custom auth method failed: {}, Maybe not enabled'.format(e))
-
-custom_failed_msg = _("MFA Custom code invalid")
-
-
-class MFACustom(BaseMFA):
-    name = 'mfa_custom'
-    display_name = 'Custom'
-    placeholder = _("MFA custom verification code")
-
-    def check_code(self, code):
-        assert self.is_authenticated()
-        ok = False
-        try:
-            ok = mfa_custom_method(user=self.user, code=code)
-        except Exception as exc:
-            logger.error('Custom authenticate error: {}'.format(exc))
-        msg = '' if ok else custom_failed_msg
-        return ok, msg
-
-    def is_active(self):
-        return True
-
-    @staticmethod
-    def global_enabled():
-        return settings.MFA_CUSTOM and callable(mfa_custom_method)
-
-    def get_enable_url(self) -> str:
-        return ''
-
-    def can_disable(self):
-        return False
-
-    def disable(self):
-        return ''
-
-    @staticmethod
-    def help_text_of_disable():
-        return _("MFA custom global enabled, cannot disable")
-
-    def get_disable_url(self) -> str:
-        return ''

apps/authentication/mfa/sms.py

@@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 
 from .base import BaseMFA
-from common.utils.verify_code import SendAndVerifyCodeUtil
+from common.sdk.sms import SendAndVerifySMSUtil
 
 sms_failed_msg = _("SMS verify code invalid")
 
@@ -15,7 +15,7 @@ class MFASms(BaseMFA):
     def __init__(self, user):
         super().__init__(user)
         phone = user.phone if self.is_authenticated() else ''
-        self.sms = SendAndVerifyCodeUtil(phone, backend=self.name)
+        self.sms = SendAndVerifySMSUtil(phone)
 
     def check_code(self, code):
         assert self.is_authenticated()
@@ -37,7 +37,7 @@ class MFASms(BaseMFA):
         return True
 
     def send_challenge(self):
-        self.sms.gen_and_send_async()
+        self.sms.gen_and_send()
 
     @staticmethod
     def global_enabled():

apps/authentication/middleware.py

@@ -9,7 +9,7 @@ from django.contrib.auth import logout as auth_logout
 
 from apps.authentication import mixins
 from common.utils import gen_key_pair
-from common.utils import get_request_ip_or_data
+from common.utils import get_request_ip
 from .signals import post_auth_failed
 
 
@@ -61,19 +61,7 @@ class ThirdPartyLoginMiddleware(mixins.AuthMixin):
             return response
         if not request.session.get('auth_third_party_required'):
             return response
-
-        white_urls = [
-            'jsi18n/', '/static/',
-            'login/guard', 'login/wait-confirm',
-            'login-confirm-ticket/status',
-            'settings/public/open',
-            'core/auth/login', 'core/auth/logout'
-        ]
-        for url in white_urls:
-            if request.path.find(url) > -1:
-                return response
-
-        ip = get_request_ip_or_data(request)
+        ip = get_request_ip(request)
         try:
             self.request = request
             self._check_login_acl(request.user, ip)
@@ -100,6 +88,7 @@ class ThirdPartyLoginMiddleware(mixins.AuthMixin):
                 guard_url = "%s?%s" % (guard_url, args)
             response = redirect(guard_url)
         finally:
+            request.session.pop('auth_third_party_required', '')
             return response
 
 

apps/authentication/migrations/0001_initial.py

@@ -1,6 +1,5 @@
 # Generated by Django 2.1.7 on 2019-02-28 08:07
 
-import common.db.models
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
@@ -28,7 +27,7 @@ class Migration(migrations.Migration):
                  models.UUIDField(default=uuid.uuid4, editable=False,
                                   verbose_name='AccessKeySecret')),
                 ('user', models.ForeignKey(
-                    on_delete=common.db.models.CASCADE_SIGNAL_SKIP,
+                    on_delete=django.db.models.deletion.CASCADE,
                     related_name='access_keys',
                     to=settings.AUTH_USER_MODEL, verbose_name='User')),
             ],

apps/authentication/migrations/0002_auto_20190729_1423.py

@@ -15,7 +15,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='accesskey',
             name='date_created',
-            field=models.DateTimeField(auto_now_add=True, default=datetime.datetime(2019, 7, 29, 6, 23, 54, 115123, tzinfo=utc), verbose_name='Date created'),
+            field=models.DateTimeField(auto_now_add=True, default=datetime.datetime(2019, 7, 29, 6, 23, 54, 115123, tzinfo=utc)),
             preserve_default=False,
         ),
         migrations.AddField(

apps/authentication/migrations/0006_auto_20211227_1059.py

@@ -1,8 +1,8 @@
 # Generated by Django 3.1.13 on 2021-12-27 02:59
 
-import common.db.models
 from django.conf import settings
 from django.db import migrations, models
+import django.db.models.deletion
 
 
 class Migration(migrations.Migration):
@@ -16,6 +16,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='ssotoken',
             name='user',
-            field=models.ForeignKey(db_constraint=False, on_delete=common.db.models.CASCADE_SIGNAL_SKIP, to=settings.AUTH_USER_MODEL, verbose_name='User'),
+            field=models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User'),
         ),
     ]

apps/authentication/mixins.py

@@ -18,7 +18,7 @@ from django.contrib.auth import (
 from django.core.exceptions import ImproperlyConfigured
 from django.shortcuts import reverse, redirect, get_object_or_404
 
-from common.utils import get_request_ip_or_data, get_request_ip, get_logger, bulk_get, FlashMessageUtil
+from common.utils import get_request_ip, get_logger, bulk_get, FlashMessageUtil
 from acls.models import LoginACL
 from users.models import User
 from users.utils import LoginBlockUtil, MFABlockUtils, LoginIpBlockUtil
@@ -92,12 +92,13 @@ auth.authenticate = authenticate
 
 class CommonMixin:
     request: Request
-    _ip = ''
 
     def get_request_ip(self):
-        if not self._ip:
-            self._ip = get_request_ip_or_data(self.request)
-        return self._ip
+        ip = ''
+        if hasattr(self.request, 'data'):
+            ip = self.request.data.get('remote_addr', '')
+        ip = ip or get_request_ip(self.request)
+        return ip
 
     def raise_credential_error(self, error):
         raise self.partial_credential_error(error=error)
@@ -211,8 +212,7 @@ class MFAMixin:
         self._do_check_user_mfa(code, mfa_type, user=user)
 
     def check_user_mfa_if_need(self, user):
-        if self.request.session.get('auth_mfa') and \
-                self.request.session.get('auth_mfa_username') == user.username:
+        if self.request.session.get('auth_mfa'):
             return
         if not user.mfa_enabled:
             return
@@ -220,16 +220,14 @@ class MFAMixin:
         active_mfa_names = user.active_mfa_backends_mapper.keys()
         raise errors.MFARequiredError(mfa_types=tuple(active_mfa_names))
 
-    def mark_mfa_ok(self, mfa_type, user):
+    def mark_mfa_ok(self, mfa_type):
         self.request.session['auth_mfa'] = 1
-        self.request.session['auth_mfa_username'] = user.username
         self.request.session['auth_mfa_time'] = time.time()
         self.request.session['auth_mfa_required'] = 0
         self.request.session['auth_mfa_type'] = mfa_type
-        MFABlockUtils(user.username, self.get_request_ip()).clean_failed_count()
 
     def clean_mfa_mark(self):
-        keys = ['auth_mfa', 'auth_mfa_time', 'auth_mfa_required', 'auth_mfa_type', 'auth_mfa_username']
+        keys = ['auth_mfa', 'auth_mfa_time', 'auth_mfa_required', 'auth_mfa_type']
         for k in keys:
             self.request.session.pop(k, '')
 
@@ -264,7 +262,7 @@ class MFAMixin:
             ok, msg = mfa_backend.check_code(code)
 
         if ok:
-            self.mark_mfa_ok(mfa_type, user)
+            self.mark_mfa_ok(mfa_type)
             return
 
         raise errors.MFAFailedError(
@@ -371,7 +369,7 @@ class AuthACLMixin:
     def check_user_login_confirm(self):
         ticket = self.get_ticket()
         if not ticket:
-            raise errors.LoginConfirmOtherError('', "Not found", '')
+            raise errors.LoginConfirmOtherError('', "Not found")
         elif ticket.is_state(ticket.State.approved):
             self.request.session["auth_confirm_required"] = ''
             return
@@ -514,20 +512,4 @@ class AuthMixin(CommonMixin, AuthPreCheckMixin, AuthACLMixin, MFAMixin, AuthPost
         args = self.request.META.get('QUERY_STRING', '')
         if args:
             guard_url = "%s?%s" % (guard_url, args)
-        response = redirect(guard_url)
-        self.set_browser_default_language_if_need(response)
-        return response
-
-    def set_browser_default_language_if_need(self, response):
-        # en, ja, zh-CN,zh;q=0.9
-        browser_lang = self.request.headers.get('Accept-Language', '')
-        # 浏览器首选语言
-        if browser_lang.startswith('en'):
-            browser_lang = 'en'
-        elif browser_lang.startswith('ja'):
-            browser_lang = 'ja'
-        else:
-            browser_lang = 'zh'
-        request_lang = self.request.LANGUAGE_CODE
-        lang = request_lang or browser_lang
-        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang)
+        return redirect(guard_url)

apps/authentication/models.py

@@ -16,10 +16,10 @@ class AccessKey(models.Model):
                           default=uuid.uuid4, editable=False)
     secret = models.UUIDField(verbose_name='AccessKeySecret',
                               default=uuid.uuid4, editable=False)
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('User'),
-                             on_delete=models.CASCADE_SIGNAL_SKIP, related_name='access_keys')
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, verbose_name='User',
+                             on_delete=models.CASCADE, related_name='access_keys')
     is_active = models.BooleanField(default=True, verbose_name=_('Active'))
-    date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created'))
+    date_created = models.DateTimeField(auto_now_add=True)
 
     def get_id(self):
         return str(self.id)
@@ -51,7 +51,7 @@ class SSOToken(models.JMSBaseModel):
     """
     authkey = models.UUIDField(primary_key=True, default=uuid.uuid4, verbose_name=_('Token'))
     expired = models.BooleanField(default=False, verbose_name=_('Expired'))
-    user = models.ForeignKey('users.User', on_delete=models.CASCADE_SIGNAL_SKIP, verbose_name=_('User'), db_constraint=False)
+    user = models.ForeignKey('users.User', on_delete=models.CASCADE, verbose_name=_('User'), db_constraint=False)
 
     class Meta:
         verbose_name = _('SSO token')

apps/authentication/serializers/password_mfa.py

@@ -1,38 +1,15 @@
 # -*- coding: utf-8 -*-
 #
-from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
 from common.drf.fields import EncryptedField
 
 __all__ = [
     'MFAChallengeSerializer', 'MFASelectTypeSerializer',
-    'PasswordVerifySerializer', 'ResetPasswordCodeSerializer',
+    'PasswordVerifySerializer',
 ]
 
 
-class ResetPasswordCodeSerializer(serializers.Serializer):
-    form_type = serializers.ChoiceField(
-        choices=[('sms', _('SMS')), ('email', _('Email'))], default='email'
-    )
-    email = serializers.CharField(allow_blank=True)
-    sms = serializers.CharField(allow_blank=True)
-
-    def create(self, attrs):
-        error = []
-        validate_backends = {
-            'email': _('Email'), 'sms': _('SMS')
-        }
-        form_type = attrs.get('form_type', 'email')
-        validate_backend_input = attrs.get(form_type)
-        if not validate_backend_input:
-            error.append(_('The {} cannot be empty').format(
-                validate_backends.get(validate_backend_input))
-            )
-        if error:
-            raise serializers.ValidationError(error)
-
-
 class PasswordVerifySerializer(serializers.Serializer):
     password = EncryptedField()
 

apps/authentication/templates/authentication/_msg_reset_password_code.html

@@ -1,21 +0,0 @@
-{% load i18n %}
-
-<div style="width: 100%; text-align: center">
-    <table style="margin: 0 auto; border: 1px solid #ccc; border-collapse: collapse; width: 60%">
-        <tr style="background-color: #1ab394; color: white">
-            <th style="height: 80px;">{{ title }}</th>
-        </tr>
-        <tr style="border: 1px solid #eee;">
-            <td style="height: 50px;">{% trans 'Hello' %} {{ user.name }},</td>
-        </tr>
-        <tr style="border: 1px solid #eee">
-            <td style="height: 50px;">{% trans 'Verify code' %}: <span style="font-weight: bold;">{{ code }}</span></td>
-        </tr>
-        <tr style="border: 1px solid #eee;">
-            <td style="height: 30px;"> {% trans 'Copy the verification code to the Reset Password page to reset the password.' %} </td>
-        </tr>
-        <tr style="border: 1px solid #eee">
-            <td style="height: 30px;">{% trans 'The validity period of the verification code is one minute' %}</td>
-        </tr>
-    </table>
-</div>

apps/authentication/urls/api_urls.py

@@ -32,8 +32,7 @@ urlpatterns = [
     path('mfa/verify/', api.MFAChallengeVerifyApi.as_view(), name='mfa-verify'),
     path('mfa/challenge/', api.MFAChallengeVerifyApi.as_view(), name='mfa-challenge'),
     path('mfa/select/', api.MFASendCodeApi.as_view(), name='mfa-select'),
-    path('mfa/send-code/', api.MFASendCodeApi.as_view(), name='mfa-send-code'),
-    path('password/reset-code/', api.UserResetPasswordSendCodeApi.as_view(), name='reset-password-code'),
+    path('mfa/send-code/', api.MFASendCodeApi.as_view(), name='mfa-send-codej'),
     path('password/verify/', api.UserPasswordVerifyApi.as_view(), name='user-password-verify'),
     path('login-confirm-ticket/status/', api.TicketStatusApi.as_view(), name='login-confirm-ticket-status'),
 ]

apps/authentication/urls/view_urls.py

@@ -18,7 +18,6 @@ urlpatterns = [
     path('logout/', views.UserLogoutView.as_view(), name='logout'),
 
     # 原来在users中的
-    path('password/forget/previewing/', users_view.UserForgotPasswordPreviewingView.as_view(), name='forgot-previewing'),
     path('password/forgot/', users_view.UserForgotPasswordView.as_view(), name='forgot-password'),
     path('password/reset/', users_view.UserResetPasswordView.as_view(), name='reset-password'),
     path('password/verify/', users_view.UserVerifyPasswordView.as_view(), name='user-verify-password'),

apps/authentication/utils.py

@@ -6,7 +6,7 @@ from urllib.parse import urljoin, urlparse
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from common.utils import validate_ip, get_ip_city, get_request_ip_or_data
+from common.utils import validate_ip, get_ip_city, get_request_ip
 from common.utils import get_logger
 from audits.models import UserLoginLog
 from audits.const import DEFAULT_CITY
@@ -19,7 +19,7 @@ def check_different_city_login_if_need(user, request):
     if not settings.SECURITY_CHECK_DIFFERENT_CITY_LOGIN:
         return
 
-    ip = get_request_ip_or_data(request) or '0.0.0.0'
+    ip = get_request_ip(request) or '0.0.0.0'
     if not (ip and validate_ip(ip)):
         city = DEFAULT_CITY
     else:

apps/authentication/views/dingtalk.py

@@ -20,12 +20,13 @@ from authentication import errors
 from authentication.mixins import AuthMixin
 from authentication.const import ConfirmType
 from common.sdk.im.dingtalk import DingTalk
-from common.utils.common import get_request_ip_or_data
+from common.utils.common import get_request_ip
 from authentication.notifications import OAuthBindMessage
 from .mixins import METAMixin
 
 logger = get_logger(__file__)
 
+
 DINGTALK_STATE_SESSION_KEY = '_dingtalk_state'
 
 
@@ -174,7 +175,7 @@ class DingTalkQRBindCallbackView(DingTalkQRMixin, View):
                 return response
             raise e
 
-        ip = get_request_ip_or_data(request)
+        ip = get_request_ip(request)
         OAuthBindMessage(user, ip, _('DingTalk'), user_id).publish_async()
         msg = _('Binding DingTalk successfully')
         response = self.get_success_response(redirect_url, msg, msg)
@@ -199,7 +200,7 @@ class DingTalkEnableStartView(UserVerifyPasswordView):
 class DingTalkQRLoginView(DingTalkQRMixin, METAMixin, View):
     permission_classes = (AllowAny,)
 
-    def get(self, request: HttpRequest):
+    def get(self,  request: HttpRequest):
         redirect_url = request.GET.get('redirect_url') or reverse('index')
         next_url = self.get_next_url_from_meta() or reverse('index')
 
@@ -257,7 +258,7 @@ class DingTalkQRLoginCallbackView(AuthMixin, DingTalkQRMixin, View):
 class DingTalkOAuthLoginView(DingTalkOAuthMixin, View):
     permission_classes = (AllowAny,)
 
-    def get(self, request: HttpRequest):
+    def get(self,  request: HttpRequest):
         redirect_url = request.GET.get('redirect_url')
 
         redirect_uri = reverse('authentication:dingtalk-oauth-login-callback', external=True)

apps/authentication/views/feishu.py

@@ -16,7 +16,7 @@ from common.utils.django import reverse, get_object_or_none
 from common.mixins.views import UserConfirmRequiredExceptionMixin, PermissionsMixin
 from common.permissions import UserConfirmation
 from common.sdk.im.feishu import FeiShu, URL
-from common.utils.common import get_request_ip_or_data
+from common.utils.common import get_request_ip
 from authentication import errors
 from authentication.const import ConfirmType
 from authentication.mixins import AuthMixin
@@ -24,6 +24,7 @@ from authentication.notifications import OAuthBindMessage
 
 logger = get_logger(__file__)
 
+
 FEISHU_STATE_SESSION_KEY = '_feishu_state'
 
 
@@ -59,7 +60,7 @@ class FeiShuQRMixin(UserConfirmRequiredExceptionMixin, PermissionsMixin, View):
             'state': state,
             'redirect_uri': redirect_uri,
         }
-        url = URL().authen + '?' + urlencode(params)
+        url = URL.AUTHEN + '?' + urlencode(params)
         return url
 
     @staticmethod
@@ -139,7 +140,7 @@ class FeiShuQRBindCallbackView(FeiShuQRMixin, View):
                 return response
             raise e
 
-        ip = get_request_ip_or_data(request)
+        ip = get_request_ip(request)
         OAuthBindMessage(user, ip, _('FeiShu'), user_id).publish_async()
         msg = _('Binding FeiShu successfully')
         response = self.get_success_response(redirect_url, msg, msg)
@@ -164,7 +165,7 @@ class FeiShuEnableStartView(UserVerifyPasswordView):
 class FeiShuQRLoginView(FeiShuQRMixin, View):
     permission_classes = (AllowAny,)
 
-    def get(self, request: HttpRequest):
+    def get(self,  request: HttpRequest):
         redirect_url = request.GET.get('redirect_url') or reverse('index')
         redirect_uri = reverse('authentication:feishu-qr-login-callback', external=True)
         redirect_uri += '?' + urlencode({

apps/authentication/views/login.py

@@ -115,7 +115,7 @@ class UserLoginContextMixin:
 
     @staticmethod
     def get_forgot_password_url():
-        forgot_password_url = reverse('authentication:forgot-previewing')
+        forgot_password_url = reverse('authentication:forgot-password')
         forgot_password_url = settings.FORGOT_PASSWORD_URL or forgot_password_url
         return forgot_password_url
 

apps/authentication/views/mfa.py

@@ -32,14 +32,11 @@ class UserLoginMFAView(mixins.AuthMixin, FormView):
         return super().get(*args, **kwargs)
 
     def form_valid(self, form):
-        from users.utils import MFABlockUtils
         code = form.cleaned_data.get('code')
         mfa_type = form.cleaned_data.get('mfa_type')
 
         try:
             self._do_check_user_mfa(code, mfa_type)
-            user, ip = self.get_user_from_session(), self.get_request_ip()
-            MFABlockUtils(user.username, ip).clean_failed_count()
             return redirect_to_guard_view('mfa_ok')
         except (errors.MFAFailedError, errors.BlockMFAError) as e:
             form.add_error('code', e.msg)

apps/authentication/views/wecom.py

@@ -16,7 +16,7 @@ from common.utils.django import reverse, get_object_or_none
 from common.sdk.im.wecom import URL
 from common.sdk.im.wecom import WeCom
 from common.mixins.views import UserConfirmRequiredExceptionMixin, PermissionsMixin
-from common.utils.common import get_request_ip_or_data
+from common.utils.common import get_request_ip
 from common.permissions import UserConfirmation
 from authentication import errors
 from authentication.mixins import AuthMixin
@@ -26,6 +26,7 @@ from .mixins import METAMixin
 
 logger = get_logger(__file__)
 
+
 WECOM_STATE_SESSION_KEY = '_wecom_state'
 
 
@@ -173,7 +174,7 @@ class WeComQRBindCallbackView(WeComQRMixin, View):
                 return response
             raise e
 
-        ip = get_request_ip_or_data(request)
+        ip = get_request_ip(request)
         OAuthBindMessage(user, ip, _('WeCom'), wecom_userid).publish_async()
         msg = _('Binding WeCom successfully')
         response = self.get_success_response(redirect_url, msg, msg)
@@ -195,7 +196,7 @@ class WeComEnableStartView(UserVerifyPasswordView):
 class WeComQRLoginView(WeComQRMixin, METAMixin, View):
     permission_classes = (AllowAny,)
 
-    def get(self, request: HttpRequest):
+    def get(self,  request: HttpRequest):
         redirect_url = request.GET.get('redirect_url') or reverse('index')
         next_url = self.get_next_url_from_meta() or reverse('index')
         redirect_uri = reverse('authentication:wecom-qr-login-callback', external=True)
@@ -252,7 +253,7 @@ class WeComQRLoginCallbackView(AuthMixin, WeComQRMixin, View):
 class WeComOAuthLoginView(WeComOAuthMixin, View):
     permission_classes = (AllowAny,)
 
-    def get(self, request: HttpRequest):
+    def get(self,  request: HttpRequest):
         redirect_url = request.GET.get('redirect_url')
 
         redirect_uri = reverse('authentication:wecom-oauth-login-callback', external=True)

apps/common/const/signals.py

@@ -15,5 +15,3 @@ POST_CLEAR = 'post_clear'
 
 POST_PREFIX = 'post'
 PRE_PREFIX = 'pre'
-
-SKIP_SIGNAL = 'skip_signal'

apps/common/db/fields.py

@@ -6,7 +6,6 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import force_text
 from django.core.validators import MinValueValidator, MaxValueValidator
 from common.utils import signer, crypto
-from common.local import add_encrypted_field_set
 
 
 __all__ = [
@@ -150,10 +149,6 @@ class EncryptMixin:
 class EncryptTextField(EncryptMixin, models.TextField):
     description = _("Encrypt field using Secret Key")
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        add_encrypted_field_set(self.verbose_name)
-
 
 class EncryptCharField(EncryptMixin, models.CharField):
     @staticmethod
@@ -168,7 +163,6 @@ class EncryptCharField(EncryptMixin, models.CharField):
     def __init__(self, *args, **kwargs):
         self.change_max_length(kwargs)
         super().__init__(*args, **kwargs)
-        add_encrypted_field_set(self.verbose_name)
 
     def deconstruct(self):
         name, path, args, kwargs = super().deconstruct()
@@ -180,15 +174,11 @@ class EncryptCharField(EncryptMixin, models.CharField):
 
 
 class EncryptJsonDictTextField(EncryptMixin, JsonDictTextField):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        add_encrypted_field_set(self.verbose_name)
+    pass
 
 
 class EncryptJsonDictCharField(EncryptMixin, JsonDictCharField):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        add_encrypted_field_set(self.verbose_name)
+    pass
 
 
 class PortField(models.IntegerField):

apps/common/db/models.py

@@ -19,8 +19,6 @@ from django.db.models import QuerySet
 from django.db.models.functions import Concat
 from django.utils.translation import ugettext_lazy as _
 
-from ..const.signals import SKIP_SIGNAL
-
 
 class Choice(str):
     def __new__(cls, value, label=''):  # `deepcopy` 的时候不会传 `label`
@@ -126,9 +124,6 @@ class JMSModel(JMSBaseModel):
     class Meta:
         abstract = True
 
-    def __str__(self):
-        return str(self.id)
-
 
 def concated_display(name1, name2):
     return Concat(F(name1), Value('('), F(name2), Value(')'))
@@ -243,14 +238,3 @@ class MultiTableChildQueryset(QuerySet):
             self._batched_insert(objs, self.model._meta.local_fields, batch_size)
 
         return objs
-
-
-def CASCADE_SIGNAL_SKIP(collector, field, sub_objs, using):
-    # 级联删除时，操作日志标记不保存，以免用户混淆
-    try:
-        for obj in sub_objs:
-            setattr(obj, SKIP_SIGNAL, True)
-    except:
-        pass
-
-    CASCADE(collector, field, sub_objs, using)

apps/common/drf/fields.py

@@ -4,7 +4,6 @@
 from rest_framework import serializers
 
 from common.utils import decrypt_password
-from common.local import add_encrypted_field_set
 
 __all__ = [
     'ReadableHiddenField', 'EncryptedField'
@@ -32,9 +31,7 @@ class EncryptedField(serializers.CharField):
         if write_only is None:
             write_only = True
         kwargs['write_only'] = write_only
-        encrypted_key = kwargs.pop('encrypted_key', None)
         super().__init__(**kwargs)
-        add_encrypted_field_set(encrypted_key or self.label)
 
     def to_internal_value(self, value):
         value = super().to_internal_value(value)

apps/common/local.py

@@ -1,13 +1,7 @@
 from werkzeug.local import Local
 
 thread_local = Local()
-encrypted_field_set = {'password'}
 
 
 def _find(attr):
     return getattr(thread_local, attr, None)
-
-
-def add_encrypted_field_set(label):
-    if label:
-        encrypted_field_set.add(str(label))

apps/common/management/commands/services/services/flower.py

@@ -9,10 +9,6 @@ class FlowerService(BaseService):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    @property
-    def db_file(self):
-        return os.path.join(BASE_DIR, 'data', 'flower')
-
     @property
     def cmd(self):
         print("\n- Start Flower as Task Monitor")
@@ -23,13 +19,11 @@ class FlowerService(BaseService):
             'celery',
             '-A', 'ops',
             'flower',
-            '-logging=info',
-            '-db={}'.format(self.db_file),
+            '-l', 'INFO',
             '--url_prefix=/core/flower',
             '--auto_refresh=False',
             '--max_tasks=1000',
-            '--persistent=True',
-            '--state_save_interval=600000'
+            '--tasks_columns=uuid,name,args,state,received,started,runtime,worker'
         ]
         return cmd
 

apps/common/mixins/views.py

@@ -7,7 +7,7 @@ from rest_framework import permissions
 from rest_framework.request import Request
 
 from common.exceptions import UserConfirmRequired
-from audits.handler import create_or_update_operate_log
+from audits.utils import create_operate_log
 from audits.models import OperateLog
 
 __all__ = ["PermissionsMixin", "RecordViewLogMixin", "UserConfirmRequiredExceptionMixin"]
@@ -62,18 +62,10 @@ class RecordViewLogMixin:
     def list(self, request, *args, **kwargs):
         response = super().list(request, *args, **kwargs)
         resource = self.get_resource_display(request)
-        resource_type = self.model._meta.verbose_name
-        create_or_update_operate_log(
-            self.ACTION, resource_type, force=True,
-            resource=resource
-        )
+        create_operate_log(self.ACTION, self.model, resource)
         return response
 
     def retrieve(self, request, *args, **kwargs):
         response = super().retrieve(request, *args, **kwargs)
-        resource_type = self.model._meta.verbose_name
-        create_or_update_operate_log(
-            self.ACTION, resource_type, force=True,
-            resource=self.get_object()
-        )
+        create_operate_log(self.ACTION, self.model, self.get_object())
         return response

apps/common/permissions.py

@@ -81,38 +81,3 @@ class UserConfirmation(permissions.BasePermission):
         min_level = ConfirmType.values.index(confirm_type) + 1
         name = 'UserConfirmationLevel{}TTL{}'.format(min_level, ttl)
         return type(name, (cls,), {'min_level': min_level, 'ttl': ttl, 'confirm_type': confirm_type})
-
-
-class ServiceAccountSignaturePermission(permissions.BasePermission):
-    def has_permission(self, request, view):
-        from authentication.models import AccessKey
-        from common.utils.crypto import get_aes_crypto
-        signature = request.META.get('HTTP_X_JMS_SVC', '')
-        if not signature or not signature.startswith('Sign'):
-            return False
-        data = signature[4:].strip()
-        if not data or ':' not in data:
-            return False
-        ak_id, time_sign = data.split(':', 1)
-        if not ak_id or not time_sign:
-            return False
-        ak = AccessKey.objects.filter(id=ak_id).first()
-        if not ak or not ak.is_active:
-            return False
-        if not ak.user or not ak.user.is_active or not ak.user.is_service_account:
-            return False
-        aes = get_aes_crypto(str(ak.secret).replace('-', ''), mode='ECB')
-        try:
-            timestamp = aes.decrypt(time_sign)
-            if not timestamp or not timestamp.isdigit():
-                return False
-            timestamp = int(timestamp)
-            interval = abs(int(time.time()) - timestamp)
-            if interval > 30:
-                return False
-            return True
-        except Exception:
-            return False
-
-    def has_object_permission(self, request, view, obj):
-        return False

apps/common/plugins/es.py

@@ -1,428 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-import datetime
-import inspect
-
-from collections.abc import Iterable
-from functools import reduce, partial
-from itertools import groupby
-from uuid import UUID
-
-from django.utils.translation import gettext_lazy as _
-from django.db.models import QuerySet as DJQuerySet
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-from elasticsearch.exceptions import RequestError, NotFoundError
-
-from common.utils.common import lazyproperty
-from common.utils import get_logger
-from common.utils.timezone import local_now_date_display
-from common.exceptions import JMSException
-
-
-logger = get_logger(__file__)
-
-
-class InvalidElasticsearch(JMSException):
-    default_code = 'invalid_elasticsearch'
-    default_detail = _('Invalid elasticsearch config')
-
-
-class NotSupportElasticsearch8(JMSException):
-    default_code = 'not_support_elasticsearch8'
-    default_detail = _('Not Support Elasticsearch8')
-
-
-class ES(object):
-    def __init__(self, config, properties, keyword_fields, exact_fields=None, match_fields=None):
-
-        self.config = config
-        hosts = self.config.get('HOSTS')
-        kwargs = self.config.get('OTHER', {})
-
-        ignore_verify_certs = kwargs.pop('IGNORE_VERIFY_CERTS', False)
-        if ignore_verify_certs:
-            kwargs['verify_certs'] = None
-        self.es = Elasticsearch(hosts=hosts, max_retries=0, **kwargs)
-        self.index_prefix = self.config.get('INDEX') or 'jumpserver'
-        self.is_index_by_date = bool(self.config.get('INDEX_BY_DATE', False))
-
-        self.index = None
-        self.query_index = None
-        self.properties = properties
-        self.exact_fields, self.match_fields, self.keyword_fields = set(), set(), set()
-
-        if isinstance(keyword_fields, Iterable):
-            self.keyword_fields.update(keyword_fields)
-        if isinstance(exact_fields, Iterable):
-            self.exact_fields.update(exact_fields)
-        if isinstance(match_fields, Iterable):
-            self.match_fields.update(match_fields)
-
-        self.init_index()
-        self.doc_type = self.config.get("DOC_TYPE") or '_doc'
-        if self.is_new_index_type():
-            self.doc_type = '_doc'
-            self.exact_fields.update(self.keyword_fields)
-        else:
-            self.match_fields.update(self.keyword_fields)
-
-    def init_index(self):
-        if self.is_index_by_date:
-            date = local_now_date_display()
-            self.index = '%s-%s' % (self.index_prefix, date)
-            self.query_index = '%s-alias' % self.index_prefix
-        else:
-            self.index = self.config.get("INDEX") or 'jumpserver'
-            self.query_index = self.config.get("INDEX") or 'jumpserver'
-
-    def is_new_index_type(self):
-        if not self.ping(timeout=2):
-            return False
-
-        info = self.es.info()
-        version = info['version']['number'].split('.')[0]
-
-        if version == '8':
-            raise NotSupportElasticsearch8
-
-        try:
-            # 获取索引信息，如果没有定义，直接返回
-            data = self.es.indices.get_mapping(self.index)
-        except NotFoundError:
-            return False
-
-        try:
-            if version == '6':
-                # 检测索引是不是新的类型 es6
-                properties = data[self.index]['mappings']['data']['properties']
-            else:
-                # 检测索引是不是新的类型 es7 default index type: _doc
-                properties = data[self.index]['mappings']['properties']
-
-            for keyword in self.keyword_fields:
-                if not properties[keyword]['type'] == 'keyword':
-                    break
-            else:
-                return True
-        except KeyError:
-            return False
-
-    def pre_use_check(self):
-        if not self.ping(timeout=3):
-            raise InvalidElasticsearch
-        self._ensure_index_exists()
-
-    def _ensure_index_exists(self):
-        info = self.es.info()
-        version = info['version']['number'].split('.')[0]
-        if version == '6':
-            mappings = {'mappings': {'data': {'properties': self.properties}}}
-        else:
-            mappings = {'mappings': {'properties': self.properties}}
-
-        if self.is_index_by_date:
-            mappings['aliases'] = {
-                self.query_index: {}
-            }
-
-        try:
-            self.es.indices.create(self.index, body=mappings)
-            return
-        except RequestError as e:
-            if e.error == 'resource_already_exists_exception':
-                logger.warning(e)
-            else:
-                logger.exception(e)
-
-    def make_data(self, data):
-        return []
-
-    def save(self, **kwargs):
-        data = self.make_data(kwargs)
-        return self.es.index(index=self.index, doc_type=self.doc_type, body=data)
-
-    def bulk_save(self, command_set, raise_on_error=True):
-        actions = []
-        for command in command_set:
-            data = dict(
-                _index=self.index,
-                _type=self.doc_type,
-                _source=self.make_data(command),
-            )
-            actions.append(data)
-        return bulk(self.es, actions, index=self.index, raise_on_error=raise_on_error)
-
-    def get(self, query: dict):
-        item = None
-        data = self.filter(query, size=1)
-        if len(data) >= 1:
-            item = data[0]
-        return item
-
-    def filter(self, query: dict, from_=None, size=None, sort=None):
-        try:
-            data = self._filter(query, from_, size, sort)
-        except Exception as e:
-            logger.error('ES filter error: {}'.format(e))
-            data = []
-        return data
-
-    def _filter(self, query: dict, from_=None, size=None, sort=None):
-        body = self.get_query_body(**query)
-
-        data = self.es.search(
-            index=self.query_index, doc_type=self.doc_type, body=body,
-            from_=from_, size=size, sort=sort
-        )
-        source_data = []
-        for item in data['hits']['hits']:
-            if item:
-                item['_source'].update({'es_id': item['_id']})
-                source_data.append(item['_source'])
-
-        return source_data
-
-    def count(self, **query):
-        try:
-            body = self.get_query_body(**query)
-            data = self.es.count(index=self.query_index, doc_type=self.doc_type, body=body)
-            count = data["count"]
-        except Exception as e:
-            logger.error('ES count error: {}'.format(e))
-            count = 0
-        return count
-
-    def __getattr__(self, item):
-        return getattr(self.es, item)
-
-    def all(self):
-        """返回所有数据"""
-        raise NotImplementedError("Not support")
-
-    def ping(self, timeout=None):
-        try:
-            return self.es.ping(request_timeout=timeout)
-        except Exception:
-            return False
-
-    @staticmethod
-    def handler_time_field(data):
-        datetime__gte = data.get('datetime__gte')
-        datetime__lte = data.get('datetime__lte')
-        datetime_range = {}
-
-        if datetime__gte:
-            if isinstance(datetime__gte, datetime.datetime):
-                datetime__gte = datetime__gte.strftime('%Y-%m-%d %H:%M:%S')
-            datetime_range['gte'] = datetime__gte
-        if datetime__lte:
-            if isinstance(datetime__lte, datetime.datetime):
-                datetime__lte = datetime__lte.strftime('%Y-%m-%d %H:%M:%S')
-            datetime_range['lte'] = datetime__lte
-        return 'datetime', datetime_range
-
-    def get_query_body(self, **kwargs):
-        new_kwargs = {}
-        for k, v in kwargs.items():
-            if isinstance(v, UUID):
-                v = str(v)
-            if k == 'pk':
-                k = 'id'
-            new_kwargs[k] = v
-        kwargs = new_kwargs
-
-        index_in_field = 'id__in'
-        exact_fields = self.exact_fields
-        match_fields = self.match_fields
-
-        match = {}
-        exact = {}
-        index = {}
-
-        if index_in_field in kwargs:
-            index['values'] = kwargs[index_in_field]
-
-        for k, v in kwargs.items():
-            if k in exact_fields:
-                exact[k] = v
-            elif k in match_fields:
-                match[k] = v
-
-        # 处理时间
-        time_field_name, time_range = self.handler_time_field(kwargs)
-
-        # 处理组织
-        should = []
-        org_id = match.get('org_id')
-
-        real_default_org_id = '00000000-0000-0000-0000-000000000002'
-        root_org_id = '00000000-0000-0000-0000-000000000000'
-
-        if org_id == root_org_id:
-            match.pop('org_id')
-        elif org_id in (real_default_org_id, ''):
-            match.pop('org_id')
-            should.append({
-                'bool': {
-                    'must_not': [
-                        {
-                            'wildcard': {'org_id': '*'}
-                        }
-                    ]}
-            })
-            should.append({'match': {'org_id': real_default_org_id}})
-
-        # 构建 body
-        body = {
-            'query': {
-                'bool': {
-                    'must': [
-                        {'match': {k: v}} for k, v in match.items()
-                    ],
-                    'should': should,
-                    'filter': [
-                                  {
-                                      'term': {k: v}
-                                  } for k, v in exact.items()
-                              ] + [
-                                  {
-                                      'range': {
-                                          time_field_name: time_range
-                                      }
-                                  }
-                              ] + [
-                                  {
-                                      'ids': {k: v}
-                                  } for k, v in index.items()
-                              ]
-                }
-            },
-        }
-        return body
-
-
-class QuerySet(DJQuerySet):
-    default_days_ago = 7
-    max_result_window = 10000
-
-    def __init__(self, es_instance):
-        self._method_calls = []
-        self._slice = None  # (from_, size)
-        self._storage = es_instance
-
-        # 命令列表模糊搜索时报错
-        super().__init__()
-
-    @lazyproperty
-    def _grouped_method_calls(self):
-        _method_calls = {k: list(v) for k, v in groupby(self._method_calls, lambda x: x[0])}
-        return _method_calls
-
-    @lazyproperty
-    def _filter_kwargs(self):
-        _method_calls = self._grouped_method_calls
-        filter_calls = _method_calls.get('filter')
-        if not filter_calls:
-            return {}
-        names, multi_args, multi_kwargs = zip(*filter_calls)
-        kwargs = reduce(lambda x, y: {**x, **y}, multi_kwargs, {})
-
-        striped_kwargs = {}
-        for k, v in kwargs.items():
-            k = k.replace('__exact', '')
-            k = k.replace('__startswith', '')
-            k = k.replace('__icontains', '')
-            striped_kwargs[k] = v
-        return striped_kwargs
-
-    @lazyproperty
-    def _sort(self):
-        order_by = self._grouped_method_calls.get('order_by')
-        if order_by:
-            for call in reversed(order_by):
-                fields = call[1]
-                if fields:
-                    field = fields[-1]
-
-                    if field.startswith('-'):
-                        direction = 'desc'
-                    else:
-                        direction = 'asc'
-                    field = field.lstrip('-+')
-                    sort = f'{field}:{direction}'
-                    return sort
-
-    def __execute(self):
-        _filter_kwargs = self._filter_kwargs
-        _sort = self._sort
-        from_, size = self._slice or (None, None)
-        data = self._storage.filter(_filter_kwargs, from_=from_, size=size, sort=_sort)
-        return self.model.from_multi_dict(data)
-
-    def __stage_method_call(self, item, *args, **kwargs):
-        _clone = self.__clone()
-        _clone._method_calls.append((item, args, kwargs))
-        return _clone
-
-    def __clone(self):
-        uqs = QuerySet(self._storage)
-        uqs._method_calls = self._method_calls.copy()
-        uqs._slice = self._slice
-        uqs.model = self.model
-        return uqs
-
-    def get(self, **kwargs):
-        kwargs.update(self._filter_kwargs)
-        return self._storage.get(kwargs)
-
-    def count(self, limit_to_max_result_window=True):
-        filter_kwargs = self._filter_kwargs
-        count = self._storage.count(**filter_kwargs)
-        if limit_to_max_result_window:
-            count = min(count, self.max_result_window)
-        return count
-
-    def __getattribute__(self, item):
-        if any((
-                item.startswith('__'),
-                item in QuerySet.__dict__,
-        )):
-            return object.__getattribute__(self, item)
-
-        origin_attr = object.__getattribute__(self, item)
-        if not inspect.ismethod(origin_attr):
-            return origin_attr
-
-        attr = partial(self.__stage_method_call, item)
-        return attr
-
-    def __getitem__(self, item):
-        max_window = self.max_result_window
-        if isinstance(item, slice):
-            if self._slice is None:
-                clone = self.__clone()
-                from_ = item.start or 0
-                if item.stop is None:
-                    size = self.max_result_window - from_
-                else:
-                    size = item.stop - from_
-
-                if from_ + size > max_window:
-                    if from_ >= max_window:
-                        from_ = max_window
-                        size = 0
-                    else:
-                        size = max_window - from_
-                clone._slice = (from_, size)
-                return clone
-        return self.__execute()[item]
-
-    def __repr__(self):
-        return self.__execute().__repr__()
-
-    def __iter__(self):
-        return iter(self.__execute())
-
-    def __len__(self):
-        return self.count()

apps/common/sdk/im/feishu/__init__.py

@@ -3,7 +3,6 @@ import json
 from django.utils.translation import ugettext_lazy as _
 from rest_framework.exceptions import APIException
 
-from django.conf import settings
 from common.utils.common import get_logger
 from common.sdk.im.utils import digest
 from common.sdk.im.mixin import RequestMixin, BaseRequest
@@ -12,30 +11,14 @@ logger = get_logger(__name__)
 
 
 class URL:
+    AUTHEN = 'https://open.feishu.cn/open-apis/authen/v1/index'
+
+    GET_TOKEN = 'https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal/'
+
     # https://open.feishu.cn/document/ukTMukTMukTM/uEDO4UjLxgDO14SM4gTN
-    @property
-    def host(self):
-        if settings.FEISHU_VERSION == 'feishu':
-            h = 'https://open.feishu.cn'
-        else:
-            h = 'https://open.larksuite.com'
-        return h
+    GET_USER_INFO_BY_CODE = 'https://open.feishu.cn/open-apis/authen/v1/access_token'
 
-    @property
-    def authen(self):
-        return f'{self.host}/open-apis/authen/v1/index'
-
-    @property
-    def get_token(self):
-        return f'{self.host}/open-apis/auth/v3/tenant_access_token/internal/'
-
-    @property
-    def get_user_info_by_code(self):
-        return f'{self.host}/open-apis/authen/v1/access_token'
-
-    @property
-    def send_message(self):
-        return f'{self.host}/open-apis/im/v1/messages'
+    SEND_MESSAGE = 'https://open.feishu.cn/open-apis/im/v1/messages'
 
 
 class ErrorCode:
@@ -68,7 +51,7 @@ class FeishuRequests(BaseRequest):
 
     def request_access_token(self):
         data = {'app_id': self._app_id, 'app_secret': self._app_secret}
-        response = self.raw_request('post', url=URL().get_token, data=data)
+        response = self.raw_request('post', url=URL.GET_TOKEN, data=data)
         self.check_errcode_is_0(response)
 
         access_token = response['tenant_access_token']
@@ -103,7 +86,7 @@ class FeiShu(RequestMixin):
             'code': code
         }
 
-        data = self._requests.post(URL().get_user_info_by_code, json=body, check_errcode_is_0=False)
+        data = self._requests.post(URL.GET_USER_INFO_BY_CODE, json=body, check_errcode_is_0=False)
 
         self._requests.check_errcode_is_0(data)
         return data['data']['user_id']
@@ -124,7 +107,7 @@ class FeiShu(RequestMixin):
 
             try:
                 logger.info(f'Feishu send text: user_ids={user_ids} msg={msg}')
-                self._requests.post(URL().send_message, params=params, json=body)
+                self._requests.post(URL.SEND_MESSAGE, params=params, json=body)
             except APIException as e:
                 # 只处理可预知的错误
                 logger.exception(e)

apps/common/sdk/sms/__init__.py

@@ -1 +1,2 @@
 from .endpoint import SMS, BACKENDS
+from .utils import SendAndVerifySMSUtil

apps/common/sdk/sms/exceptions.py

@@ -1,21 +0,0 @@
-from django.utils.translation import gettext_lazy as _
-
-from common.exceptions import JMSException
-
-
-class CodeExpired(JMSException):
-    default_code = 'verify_code_expired'
-    default_detail = _('The verification code has expired. Please resend it')
-
-
-class CodeError(JMSException):
-    default_code = 'verify_code_error'
-    default_detail = _('The verification code is incorrect')
-
-
-class CodeSendTooFrequently(JMSException):
-    default_code = 'code_send_too_frequently'
-    default_detail = _('Please wait {} seconds before sending')
-
-    def __init__(self, ttl):
-        super().__init__(detail=self.default_detail.format(ttl))

apps/common/sdk/sms/utils.py

@@ -0,0 +1,90 @@
+import random
+
+from django.core.cache import cache
+from django.utils.translation import gettext_lazy as _
+
+from .endpoint import SMS
+from common.utils import get_logger
+from common.exceptions import JMSException
+
+logger = get_logger(__file__)
+
+
+class CodeExpired(JMSException):
+    default_code = 'verify_code_expired'
+    default_detail = _('The verification code has expired. Please resend it')
+
+
+class CodeError(JMSException):
+    default_code = 'verify_code_error'
+    default_detail = _('The verification code is incorrect')
+
+
+class CodeSendTooFrequently(JMSException):
+    default_code = 'code_send_too_frequently'
+    default_detail = _('Please wait {} seconds before sending')
+
+    def __init__(self, ttl):
+        super().__init__(detail=self.default_detail.format(ttl))
+
+
+class SendAndVerifySMSUtil:
+    KEY_TMPL = 'auth-verify-code-{}'
+    TIMEOUT = 60
+
+    def __init__(self, phone, key_suffix=None, timeout=None):
+        self.phone = phone
+        self.code = ''
+        self.timeout = timeout or self.TIMEOUT
+        self.key_suffix = key_suffix or str(phone)
+        self.key = self.KEY_TMPL.format(self.key_suffix)
+
+    def gen_and_send(self):
+        """
+        生成，保存，发送
+        """
+        ttl = self.ttl()
+        if ttl > 0:
+            logger.error('Send sms too frequently, delay {}'.format(ttl))
+            raise CodeSendTooFrequently(ttl)
+
+        try:
+            code = self.generate()
+            self.send(code)
+        except JMSException:
+            self.clear()
+            raise
+
+    def generate(self):
+        code = ''.join(random.sample('0123456789', 4))
+        self.code = code
+        return code
+
+    def clear(self):
+        cache.delete(self.key)
+
+    def send(self, code):
+        """
+        发送信息的方法，如果有错误直接抛出 api 异常
+        """
+        sms = SMS()
+        sms.send_verify_code(self.phone, code)
+        cache.set(self.key, self.code, self.timeout)
+        logger.info(f'Send sms verify code to {self.phone}: {code}')
+
+    def verify(self, code):
+        right = cache.get(self.key)
+        if not right:
+            raise CodeExpired
+
+        if right != code:
+            raise CodeError
+
+        self.clear()
+        return True
+
+    def ttl(self):
+        return cache.ttl(self.key)
+
+    def get_code(self):
+        return cache.get(self.key)

apps/common/utils/common.py

@@ -76,7 +76,6 @@ def setattr_bulk(seq, key, value):
     def set_attr(obj):
         setattr(obj, key, value)
         return obj
-
     return map(set_attr, seq)
 
 
@@ -98,12 +97,12 @@ def capacity_convert(size, expect='auto', rate=1000):
     rate_mapping = (
         ('K', rate),
         ('KB', rate),
-        ('M', rate ** 2),
-        ('MB', rate ** 2),
-        ('G', rate ** 3),
-        ('GB', rate ** 3),
-        ('T', rate ** 4),
-        ('TB', rate ** 4),
+        ('M', rate**2),
+        ('MB', rate**2),
+        ('G', rate**3),
+        ('GB', rate**3),
+        ('T', rate**4),
+        ('TB', rate**4),
     )
 
     rate_mapping = OrderedDict(rate_mapping)
@@ -118,7 +117,7 @@ def capacity_convert(size, expect='auto', rate=1000):
 
     if expect == 'auto':
         for unit, rate_ in rate_mapping.items():
-            if rate > std_size / rate_ >= 1 or unit == "T":
+            if rate > std_size/rate_ >= 1 or unit == "T":
                 expect = unit
                 break
 
@@ -154,23 +153,18 @@ def is_uuid(seq):
 
 def get_request_ip(request):
     x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR', '').split(',')
+
     if x_forwarded_for and x_forwarded_for[0]:
         login_ip = x_forwarded_for[0]
-        return login_ip
-
-    login_ip = request.META.get('REMOTE_ADDR', '')
+    else:
+        login_ip = request.META.get('REMOTE_ADDR', '')
     return login_ip
 
 
 def get_request_ip_or_data(request):
-    from common.permissions import ServiceAccountSignaturePermission
-
     ip = ''
-
-    if hasattr(request, 'data') and request.data.get('remote_addr', ''):
-        permission = ServiceAccountSignaturePermission()
-        if permission.has_permission(request, None):
-            ip = request.data.get('remote_addr', '')
+    if hasattr(request, 'data'):
+        ip = request.data.get('remote_addr', '')
     ip = ip or get_request_ip(request)
     return ip
 
@@ -201,7 +195,6 @@ def with_cache(func):
         res = func(*args, **kwargs)
         cache[key] = res
         return res
-
     return wrapper
 
 
@@ -223,7 +216,6 @@ def timeit(func):
         msg = "End call {}, using: {:.1f}ms".format(name, using)
         logger.debug(msg)
         return result
-
     return wrapper
 
 
@@ -318,7 +310,7 @@ class Time:
     def print(self):
         last, *timestamps = self._timestamps
         for timestamp, msg in zip(timestamps, self._msgs):
-            logger.debug(f'TIME_IT: {msg} {timestamp - last}')
+            logger.debug(f'TIME_IT: {msg} {timestamp-last}')
             last = timestamp
 
 
@@ -374,7 +366,7 @@ def pretty_string(data: str, max_length=128, ellipsis_str='...'):
 
 
 def group_by_count(it, count):
-    return [it[i:i + count] for i in range(0, len(it), count)]
+    return [it[i:i+count] for i in range(0, len(it), count)]
 
 
 def test_ip_connectivity(host, port, timeout=0.5):
@@ -397,8 +389,3 @@ def static_or_direct(logo_path):
         return static(logo_path)
     else:
         return logo_path
-
-
-def make_dirs(name, mode=0o755, exist_ok=False):
-    """ 默认权限设置为 0o755 """
-    return os.makedirs(name, mode=mode, exist_ok=exist_ok)