From 181eb621c01f9a1cde2a3245b40c809d5803be11 Mon Sep 17 00:00:00 2001
From: fit2bot <68588906+fit2bot@users.noreply.github.com>
Date: Mon, 19 Aug 2024 16:04:00 +0800
Subject: [PATCH] perf: Remove kubernetes tree api (#13995)
* perf: Remove kubernetes tree api
* perf: Update Dockerfile with new base image tag
---------
Co-authored-by: feng <1304903146@qq.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: feng626 <57284900+feng626@users.noreply.github.com>
---
apps/assets/utils/__init__.py | 1 -
apps/assets/utils/k8s.py | 177 ------------------
.../user_permission/tree/node_with_asset.py | 63 +------
apps/perms/urls/user_permission.py | 3 -
poetry.lock | 33 +---
pyproject.toml | 1 -
6 files changed, 2 insertions(+), 276 deletions(-)
delete mode 100644 apps/assets/utils/k8s.py
diff --git a/apps/assets/utils/__init__.py b/apps/assets/utils/__init__.py
index 9f588b6d2..97431995d 100644
--- a/apps/assets/utils/__init__.py
+++ b/apps/assets/utils/__init__.py
@@ -1,2 +1 @@
-from .k8s import *
from .node import *
diff --git a/apps/assets/utils/k8s.py b/apps/assets/utils/k8s.py
deleted file mode 100644
index 8e1069566..000000000
--- a/apps/assets/utils/k8s.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# -*- coding: utf-8 -*-
-from urllib.parse import urlencode, urlparse
-
-from kubernetes import client
-from kubernetes.client import api_client
-from kubernetes.client.api import core_v1_api
-from sshtunnel import SSHTunnelForwarder, BaseSSHTunnelForwarderError
-
-from common.utils import get_logger
-from ..const import CloudTypes, Category
-
-logger = get_logger(__file__)
-
-
-class KubernetesClient:
- def __init__(self, asset, token):
- self.url = asset.address
- self.token = token or ''
- self.server = self.get_gateway_server(asset)
-
- @property
- def api(self):
- configuration = client.Configuration()
- scheme = urlparse(self.url).scheme
- if not self.server:
- host = self.url
- else:
- host = f'{scheme}://127.0.0.1:{self.server.local_bind_port}'
- configuration.host = host
- configuration.verify_ssl = False
- configuration.api_key = {"authorization": "Bearer " + self.token}
- c = api_client.ApiClient(configuration=configuration)
- api = core_v1_api.CoreV1Api(c)
- return api
-
- def get_namespaces(self):
- namespaces = []
- resp = self.api.list_namespace()
- for ns in resp.items:
- namespaces.append(ns.metadata.name)
- return namespaces
-
- def get_pods(self, namespace):
- pods = []
- resp = self.api.list_namespaced_pod(namespace)
- for pd in resp.items:
- pods.append(pd.metadata.name)
- return pods
-
- def get_containers(self, namespace, pod_name):
- containers = []
- resp = self.api.read_namespaced_pod(pod_name, namespace)
- for container in resp.spec.containers:
- containers.append(container.name)
- return containers
-
- @staticmethod
- def get_gateway_server(asset):
- gateway = None
- if not asset.is_gateway and asset.domain:
- gateway = asset.domain.select_gateway()
-
- if not gateway:
- return
-
- remote_bind_address = (
- urlparse(asset.address).hostname,
- urlparse(asset.address).port or 443
- )
- server = SSHTunnelForwarder(
- (gateway.address, gateway.port),
- ssh_username=gateway.username,
- ssh_password=gateway.password,
- ssh_pkey=gateway.private_key_path,
- remote_bind_address=remote_bind_address
- )
- try:
- server.start()
- except BaseSSHTunnelForwarderError:
- err_msg = 'Gateway is not active: %s' % asset.get('name', '')
- print('\033[31m %s \033[0m\n' % err_msg)
- return server
-
- def run(self, tp, *args):
- func_name = f'get_{tp}s'
- data = []
- if hasattr(self, func_name):
- try:
- data = getattr(self, func_name)(*args)
- except Exception as e:
- logger.error(f'K8S tree get {tp} error: {e}')
-
- if self.server:
- self.server.stop()
- return data
-
-
-class KubernetesTree:
- def __init__(self, asset, secret):
- self.asset = asset
- self.secret = secret
-
- def as_asset_tree_node(self):
- i = str(self.asset.id)
- name = str(self.asset)
- node = self.create_tree_node(
- i, i, name, 'asset', icon='k8s', is_open=True,
- )
- return node
-
- def as_namespace_node(self, name, tp):
- i = urlencode({'namespace': name})
- pid = str(self.asset.id)
- node = self.create_tree_node(i, pid, name, tp, icon='cloud')
- return node
-
- def as_pod_tree_node(self, namespace, name, tp):
- pid = urlencode({'namespace': namespace})
- i = urlencode({'namespace': namespace, 'pod': name})
- node = self.create_tree_node(i, pid, name, tp, icon='cloud')
- return node
-
- def as_container_tree_node(self, namespace, pod, name, tp):
- pid = urlencode({'namespace': namespace, 'pod': pod})
- i = urlencode({'namespace': namespace, 'pod': pod, 'container': name})
- node = self.create_tree_node(
- i, pid, name, tp, icon='cloud', is_container=True
- )
- return node
-
- @staticmethod
- def create_tree_node(id_, pid, name, identity, icon='', is_container=False, is_open=False):
- node = {
- 'id': id_,
- 'name': name,
- 'title': name,
- 'pId': pid,
- 'isParent': not is_container,
- 'open': is_open,
- 'iconSkin': icon,
- 'meta': {
- 'type': 'k8s',
- 'data': {
- 'category': Category.CLOUD,
- 'type': CloudTypes.K8S,
- 'identity': identity
- }
- }
- }
- return node
-
- def async_tree_node(self, namespace, pod):
- tree = []
- k8s_client = KubernetesClient(self.asset, self.secret)
- if pod:
- tp = 'container'
- containers = k8s_client.run(
- tp, namespace, pod
- )
- for container in containers:
- container_node = self.as_container_tree_node(
- namespace, pod, container, tp
- )
- tree.append(container_node)
- elif namespace:
- tp = 'pod'
- pods = k8s_client.run(tp, namespace)
- for pod in pods:
- pod_node = self.as_pod_tree_node(namespace, pod, tp)
- tree.append(pod_node)
- else:
- tp = 'namespace'
- namespaces = k8s_client.run(tp)
- for namespace in namespaces:
- namespace_node = self.as_namespace_node(namespace, tp)
- tree.append(namespace_node)
- return tree
diff --git a/apps/perms/api/user_permission/tree/node_with_asset.py b/apps/perms/api/user_permission/tree/node_with_asset.py
index 114c5475e..dbba179f0 100644
--- a/apps/perms/api/user_permission/tree/node_with_asset.py
+++ b/apps/perms/api/user_permission/tree/node_with_asset.py
@@ -1,31 +1,22 @@
import abc
-from urllib.parse import parse_qsl
from django.conf import settings
from django.db.models import F, Value, CharField
-from rest_framework.exceptions import PermissionDenied, NotFound
from rest_framework.generics import ListAPIView
-from rest_framework.generics import get_object_or_404
-from rest_framework.request import Request
from rest_framework.response import Response
-from accounts.const import AliasAccount
from assets.api import SerializeToTreeNodeMixin
from assets.models import Asset
-from assets.utils import KubernetesTree
-from authentication.models import ConnectionToken
-from common.exceptions import JMSException
from common.utils import get_object_or_none, lazyproperty
from common.utils.common import timeit
from perms.hands import Node
from perms.models import PermNode
-from perms.utils import PermAssetDetailUtil, UserPermNodeUtil
from perms.utils import UserPermAssetUtil
+from perms.utils import UserPermNodeUtil
from .mixin import RebuildTreeMixin
from ..mixin import SelfOrPKUserMixin
__all__ = [
- 'UserGrantedK8sAsTreeApi',
'UserPermedNodesWithAssetsAsTreeApi',
'UserPermedNodeChildrenWithAssetsAsTreeApi',
'UserPermedNodeChildrenWithAssetsAsCategoryTreeApi',
@@ -218,55 +209,3 @@ class UserPermedNodeChildrenWithAssetsAsCategoryTreeApi(BaseUserNodeWithAssetAsT
return self.get_assets()
else:
return []
-
-
-class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView):
- """ 用户授权的K8s树 """
-
- def get_token(self):
- token_id = self.request.query_params.get('token')
- token = get_object_or_404(ConnectionToken, pk=token_id)
- if token.is_expired:
- raise PermissionDenied('Token is expired')
- token.renewal()
- return token
-
- def get_account_secret(self, token: ConnectionToken):
- util = PermAssetDetailUtil(self.user, token.asset)
- accounts = util.get_permed_accounts_for_user()
- account_name = token.account
-
- if account_name in [AliasAccount.INPUT, AliasAccount.USER]:
- return token.input_secret
- else:
- accounts = filter(lambda x: x.name == account_name, accounts)
- accounts = list(accounts)
- if not accounts:
- raise NotFound('Account is not found')
- account = accounts[0]
- return account.secret
-
- @staticmethod
- def get_namespace_and_pod(key):
- namespace_and_pod = dict(parse_qsl(key))
- pod = namespace_and_pod.get('pod')
- namespace = namespace_and_pod.get('namespace')
- return namespace, pod
-
- def list(self, request: Request, *args, **kwargs):
- token = self.get_token()
- asset = token.asset
- secret = self.get_account_secret(token)
- key = self.request.query_params.get('key')
- namespace, pod = self.get_namespace_and_pod(key)
-
- tree = []
- k8s_tree_instance = KubernetesTree(asset, secret)
- if not any([namespace, pod]) and not key:
- asset_node = k8s_tree_instance.as_asset_tree_node()
- tree.append(asset_node)
- try:
- tree.extend(k8s_tree_instance.async_tree_node(namespace, pod))
- return Response(data=tree)
- except Exception as e:
- raise JMSException(e)
diff --git a/apps/perms/urls/user_permission.py b/apps/perms/urls/user_permission.py
index 0ba99087e..e17ee3218 100644
--- a/apps/perms/urls/user_permission.py
+++ b/apps/perms/urls/user_permission.py
@@ -47,9 +47,6 @@ user_permission_urlpatterns = [
path('<str:user>/nodes/all-with-assets/tree/',
api.UserPermedNodesWithAssetsAsTreeApi.as_view(),
name='user-nodes-with-assets-as-tree'),
- path('<str:user>/nodes/children-with-k8s/tree/',
- api.UserGrantedK8sAsTreeApi.as_view(),
- name='user-nodes-children-with-k8s-as-tree'),
]
user_group_permission_urlpatterns = [
diff --git a/poetry.lock b/poetry.lock
index dd4ac20dc..051a97173 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3500,37 +3500,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
-[[package]]
-name = "kubernetes"
-version = "27.2.0"
-description = "Kubernetes python client"
-optional = false
-python-versions = ">=3.6"
-files = [
- {file = "kubernetes-27.2.0-py2.py3-none-any.whl", hash = "sha256:0f9376329c85cf07615ed6886bf9bf21eb1cbfc05e14ec7b0f74ed8153cd2815"},
- {file = "kubernetes-27.2.0.tar.gz", hash = "sha256:d479931c6f37561dbfdf28fc5f46384b1cb8b28f9db344ed4a232ce91990825a"},
-]
-
-[package.dependencies]
-certifi = ">=14.05.14"
-google-auth = ">=1.0.1"
-oauthlib = ">=3.2.2"
-python-dateutil = ">=2.5.3"
-pyyaml = ">=5.4.1"
-requests = "*"
-requests-oauthlib = "*"
-six = ">=1.9.0"
-urllib3 = ">=1.24.2"
-websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0"
-
-[package.extras]
-adal = ["adal (>=1.0.2)"]
-
-[package.source]
-type = "legacy"
-url = "https://mirrors.aliyun.com/pypi/simple"
-reference = "aliyun"
-
[[package]]
name = "ldap3"
version = "2.9.1"
@@ -7701,4 +7670,4 @@ reference = "aliyun"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
-content-hash = "2dc429e66e78ab1e264e26da60df6a67cb8d5e501d80297332d673646bc0cf13"
+content-hash = "9acfafd75bf7dbb7e0dffb54b7f11f6b09aa4ceff769d193a3906d03ae796ccc"
diff --git a/pyproject.toml b/pyproject.toml
index f0ad19047..9b2047f6c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -126,7 +126,6 @@ django-auth-ldap = "4.4.0"
boto3 = "1.28.9"
botocore = "1.31.9"
s3transfer = "0.6.1"
-kubernetes = "27.2.0"
mysqlclient = "2.2.4"
pymssql = "2.2.8"
django-redis = "5.3.0"