From 100bfe0304f82c8b87b6b989debb3690ec55ecdb Mon Sep 17 00:00:00 2001 From: feng626 <1304903146@qq.com> Date: Mon, 30 Aug 2021 16:48:46 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E8=8E=B7=E5=8F=96k8s?= =?UTF-8?q?=20pod=20namespace=20container=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/applications/api/application.py | 2 +- apps/applications/api/mixin.py | 66 +++++-- apps/applications/models/application.py | 78 ++++++-- apps/applications/utils/__init__.py | 4 + apps/applications/utils/kubernetes_util.py | 186 ++++++++++++++++++ apps/common/tree.py | 2 + apps/orgs/models.py | 4 +- .../api/application/user_permission/common.py | 1 - .../user_permission_applications.py | 7 +- requirements/requirements.txt | 4 +- 10 files changed, 309 insertions(+), 45 deletions(-) create mode 100644 apps/applications/utils/__init__.py create mode 100644 apps/applications/utils/kubernetes_util.py diff --git a/apps/applications/api/application.py b/apps/applications/api/application.py index 9428df39f..ef00fe8c1 100644 --- a/apps/applications/api/application.py +++ b/apps/applications/api/application.py @@ -1,6 +1,6 @@ # coding: utf-8 # - +from django.shortcuts import get_object_or_404 from orgs.mixins.api import OrgBulkModelViewSet from rest_framework.decorators import action from rest_framework.response import Response diff --git a/apps/applications/api/mixin.py b/apps/applications/api/mixin.py index 59bc51467..e17755ea8 100644 --- a/apps/applications/api/mixin.py +++ b/apps/applications/api/mixin.py @@ -1,13 +1,21 @@ +from urllib.parse import urlencode, parse_qsl + from django.utils.translation import ugettext as _ +from rest_framework.generics import get_object_or_404 from common.tree import TreeNode from orgs.models import Organization +from assets.models import SystemUser +from applications.utils import KubernetesClient, KubernetesTree +from perms.utils.application.permission import get_application_system_user_ids + from ..models import Application __all__ = ['SerializeApplicationToTreeNodeMixin'] class SerializeApplicationToTreeNodeMixin: + @staticmethod def filter_organizations(applications): organization_ids = set(applications.values_list('org_id', flat=True)) @@ -31,25 +39,47 @@ class SerializeApplicationToTreeNodeMixin: }) return node - def serialize_applications_with_org(self, applications): + def serialize_applications_with_org(self, applications, tree_id, parent_info, user): + tree_nodes = [] if not applications: - return [] - root_node = self.create_root_node() - tree_nodes = [root_node] - organizations = self.filter_organizations(applications) + return tree_nodes - for i, org in enumerate(organizations): - # 组织节点 - org_node = org.as_tree_node(pid=root_node.id) - tree_nodes.append(org_node) - org_applications = applications.filter(org_id=org.id) - count = org_applications.count() - org_node.name += '({})'.format(count) + if not tree_id: + root_node = self.create_root_node() + tree_nodes.append(root_node) + organizations = self.filter_organizations(applications) + for i, org in enumerate(organizations): + tree_id = urlencode({'org_id': str(org.id)}) + apps = applications.filter(org_id=org.id) + # 组织节点 + org_node = org.as_tree_node(oid=tree_id, pid=root_node.id) + org_node.name += '({})'.format(apps.count()) + tree_nodes.append(org_node) + category_type_nodes = Application.create_category_type_tree_nodes( + apps, tree_id, show_empty=False + ) + tree_nodes += category_type_nodes - # 各应用节点 - apps_nodes = Application.create_tree_nodes( - queryset=org_applications, root_node=org_node, - show_empty=False - ) - tree_nodes += apps_nodes + for app in apps: + app_node = app.as_tree_node(tree_id, is_luna=True) + tree_nodes.append(app_node) + return tree_nodes + + parent_info = dict(parse_qsl(parent_info)) + pod_name = parent_info.get('pod') + app_id = parent_info.get('app_id') + namespace = parent_info.get('namespace') + system_user_id = parent_info.get('system_user_id') + if app_id and not any([pod_name, namespace, system_user_id]): + app = get_object_or_404(Application, id=app_id) + system_user_ids = get_application_system_user_ids(user, app) + system_users = SystemUser.objects.filter(id__in=system_user_ids).order_by('priority') + for system_user in system_users: + system_user_node = KubernetesTree(tree_id).as_system_user_tree_node( + system_user, parent_info + ) + tree_nodes.append(system_user_node) + return tree_nodes + + tree_nodes = KubernetesTree(tree_id).async_tree_node(parent_info) return tree_nodes diff --git a/apps/applications/models/application.py b/apps/applications/models/application.py index 5df8e3dfd..725736aeb 100644 --- a/apps/applications/models/application.py +++ b/apps/applications/models/application.py @@ -1,4 +1,5 @@ from collections import defaultdict +from urllib.parse import urlencode, parse_qsl from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -7,6 +8,8 @@ from orgs.mixins.models import OrgModelMixin from common.mixins import CommonModelMixin from common.tree import TreeNode from assets.models import Asset, SystemUser + +from ..utils import KubernetesTree from .. import const @@ -16,6 +19,13 @@ class ApplicationTreeNodeMixin: type: str category: str + @staticmethod + def create_tree_id(pid, type, v): + i = dict(parse_qsl(pid)) + i[type] = v + tree_id = urlencode(i) + return tree_id + @classmethod def create_choice_node(cls, c, id_, pid, tp, opened=False, counts=None, show_empty=True, show_count=True): @@ -65,13 +75,13 @@ class ApplicationTreeNodeMixin: return node @classmethod - def create_category_tree_nodes(cls, root_node, counts=None, show_empty=True, show_count=True): + def create_category_tree_nodes(cls, pid, counts=None, show_empty=True, show_count=True): nodes = [] categories = const.AppType.category_types_mapper().keys() for category in categories: - i = root_node.id + '_' + category.value + i = cls.create_tree_id(pid, 'category', category.value) node = cls.create_choice_node( - category, i, pid=root_node.id, tp='category', + category, i, pid=pid, tp='category', counts=counts, opened=False, show_empty=show_empty, show_count=show_count ) @@ -81,17 +91,20 @@ class ApplicationTreeNodeMixin: return nodes @classmethod - def create_types_tree_nodes(cls, root_node, counts, show_empty=True, show_count=True): + def create_types_tree_nodes(cls, pid, counts, show_empty=True, show_count=True): nodes = [] + temp_pid = pid type_category_mapper = const.AppType.type_category_mapper() - for tp in const.AppType.type_category_mapper().keys(): + types = const.AppType.type_category_mapper().keys() + for tp in types: category = type_category_mapper.get(tp) - pid = root_node.id + '_' + category.value - i = root_node.id + '_' + tp.value + pid = cls.create_tree_id(pid, 'category', category.value) + i = cls.create_tree_id(pid, 'type', tp.value) node = cls.create_choice_node( tp, i, pid, tp='type', counts=counts, opened=False, show_empty=show_empty, show_count=show_count ) + pid = temp_pid if not node: continue nodes.append(node) @@ -109,40 +122,63 @@ class ApplicationTreeNodeMixin: return counts @classmethod - def create_tree_nodes(cls, queryset, root_node=None, show_empty=True, show_count=True): + def create_category_type_tree_nodes(cls, queryset, pid, show_empty=True, show_count=True): counts = cls.get_tree_node_counts(queryset) tree_nodes = [] + # 类别的节点 + tree_nodes += cls.create_category_tree_nodes( + pid, counts, show_empty=show_empty, + show_count=show_count + ) + + # 类型的节点 + tree_nodes += cls.create_types_tree_nodes( + pid, counts, show_empty=show_empty, + show_count=show_count + ) + + return tree_nodes + + @classmethod + def create_tree_nodes(cls, queryset, root_node=None, show_empty=True, show_count=True): + tree_nodes = [] + # 根节点有可能是组织名称 if root_node is None: root_node = cls.create_root_tree_node(queryset, show_count=show_count) tree_nodes.append(root_node) - # 类别的节点 - tree_nodes += cls.create_category_tree_nodes( - root_node, counts, show_empty=show_empty, - show_count=show_count - ) - - # 类型的节点 - tree_nodes += cls.create_types_tree_nodes( - root_node, counts, show_empty=show_empty, - show_count=show_count + tree_nodes += cls.create_category_type_tree_nodes( + queryset, root_node.id, show_empty=show_empty, show_count=show_count ) # 应用的节点 for app in queryset: - pid = root_node.id + '_' + app.type - tree_nodes.append(app.as_tree_node(pid)) + node = app.as_tree_node(root_node.id) + tree_nodes.append(node) return tree_nodes - def as_tree_node(self, pid): + def create_app_tree_pid(self, root_id): + pid = self.create_tree_id(root_id, 'category', self.category) + pid = self.create_tree_id(pid, 'type', self.type) + return pid + + def as_tree_node(self, pid, is_luna=False): + if is_luna and self.type == const.AppType.k8s: + node = KubernetesTree(pid).as_tree_node(self) + else: + node = self._as_tree_node(pid) + return node + + def _as_tree_node(self, pid): icon_skin_category_mapper = { 'remote_app': 'chrome', 'db': 'database', 'cloud': 'cloud' } icon_skin = icon_skin_category_mapper.get(self.category, 'file') + pid = self.create_app_tree_pid(pid) node = TreeNode(**{ 'id': str(self.id), 'name': self.name, diff --git a/apps/applications/utils/__init__.py b/apps/applications/utils/__init__.py new file mode 100644 index 000000000..5efec40b2 --- /dev/null +++ b/apps/applications/utils/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# + +from .kubernetes_util import * diff --git a/apps/applications/utils/kubernetes_util.py b/apps/applications/utils/kubernetes_util.py new file mode 100644 index 000000000..e95922d48 --- /dev/null +++ b/apps/applications/utils/kubernetes_util.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +from urllib3.exceptions import MaxRetryError +from urllib.parse import urlencode + +from kubernetes.client import api_client +from kubernetes.client.api import core_v1_api +from kubernetes import client +from kubernetes.client.exceptions import ApiException + +from rest_framework.generics import get_object_or_404 + +from common.utils import get_logger +from common.tree import TreeNode +from assets.models import SystemUser + +from .. import const + +logger = get_logger(__file__) + + +class KubernetesClient: + def __init__(self, url, token): + self.url = url + self.token = token + + def get_api(self): + configuration = client.Configuration() + configuration.host = self.url + configuration.verify_ssl = False + configuration.api_key = {"authorization": "Bearer " + self.token} + c = api_client.ApiClient(configuration=configuration) + api = core_v1_api.CoreV1Api(c) + return api + + def get_namespace_list(self): + api = self.get_api() + namespace_list = [] + for ns in api.list_namespace().items: + namespace_list.append(ns.metadata.name) + return namespace_list + + def get_services(self): + api = self.get_api() + ret = api.list_service_for_all_namespaces(watch=False) + for i in ret.items: + print("%s \t%s \t%s \t%s \t%s \n" % ( + i.kind, i.metadata.namespace, i.metadata.name, i.spec.cluster_ip, i.spec.ports)) + + def get_pod_info(self, namespace, pod): + api = self.get_api() + resp = api.read_namespaced_pod(namespace=namespace, name=pod) + return resp + + def get_pod_logs(self, namespace, pod): + api = self.get_api() + log_content = api.read_namespaced_pod_log(pod, namespace, pretty=True, tail_lines=200) + return log_content + + def get_pods(self): + api = self.get_api() + try: + ret = api.list_pod_for_all_namespaces(watch=False, _request_timeout=(3, 3)) + except MaxRetryError: + logger.warning('Kubernetes connection timed out') + return + except ApiException as e: + if e.status == 401: + logger.warning('Kubernetes User not authenticated') + else: + logger.warning(e) + return + data = {} + for i in ret.items: + namespace = i.metadata.namespace + pod_info = { + 'pod_name': i.metadata.name, + 'containers': [j.name for j in i.spec.containers] + } + if namespace in data: + data[namespace].append(pod_info) + else: + data[namespace] = [pod_info, ] + return data + + @staticmethod + def get_kubernetes_data(app_id, system_user_id): + from ..models import Application + app = get_object_or_404(Application, id=app_id) + system_user = get_object_or_404(SystemUser, id=system_user_id) + k8s = KubernetesClient(app.attrs['cluster'], system_user.token) + return k8s.get_pods() + + +class KubernetesTree: + def __init__(self, tree_id): + self.tree_id = tree_id + + def as_tree_node(self, app): + pid = app.create_app_tree_pid(self.tree_id) + app_id = str(app.id) + parent_info = {'app_id': app_id} + node = self.create_tree_node( + app_id, pid, app.name, 'k8s', parent_info + ) + return node + + def as_system_user_tree_node(self, system_user, parent_info): + from ..models import ApplicationTreeNodeMixin + system_user_id = str(system_user.id) + username = system_user.username + username = username if username else '*' + name = f'{system_user.name}({username})' + pid = urlencode({'app_id': self.tree_id}) + i = ApplicationTreeNodeMixin.create_tree_id(pid, 'system_user_id', system_user_id) + parent_info.update({'system_user_id': system_user_id}) + node = self.create_tree_node( + i, pid, name, 'system_user', parent_info, icon='user-tie' + ) + return node + + def as_namespace_pod_tree_node(self, name, meta, type, counts=0, is_container=False): + from ..models import ApplicationTreeNodeMixin + i = ApplicationTreeNodeMixin.create_tree_id(self.tree_id, type, name) + meta.update({type: name}) + name = name if is_container else f'{name}({counts})' + node = self.create_tree_node( + i, self.tree_id, name, type, meta, icon='cloud', is_container=is_container + ) + return node + + @staticmethod + def create_tree_node(id_, pid, name, identity, parent_info, icon='', is_container=False): + node = TreeNode(**{ + 'id': id_, + 'name': name, + 'title': name, + 'pId': pid, + 'isParent': not is_container, + 'open': False, + 'iconSkin': icon, + 'parentInfo': urlencode(parent_info), + 'meta': { + 'type': 'application', + 'data': { + 'category': const.AppCategory.cloud, + 'type': const.AppType.k8s, + 'identity': identity + } + } + }) + return node + + def async_tree_node(self, parent_info): + pod_name = parent_info.get('pod') + app_id = parent_info.get('app_id') + namespace = parent_info.get('namespace') + system_user_id = parent_info.get('system_user_id') + + tree_nodes = [] + data = KubernetesClient.get_kubernetes_data(app_id, system_user_id) + if not data: + return tree_nodes + + if pod_name: + for container in next( + filter( + lambda x: x['pod_name'] == pod_name, data[namespace] + ) + )['containers']: + container_node = self.as_namespace_pod_tree_node( + container, parent_info, 'container', is_container=True + ) + tree_nodes.append(container_node) + elif namespace: + for pod in data[namespace]: + pod_nodes = self.as_namespace_pod_tree_node( + pod['pod_name'], parent_info, 'pod', len(pod['containers']) + ) + tree_nodes.append(pod_nodes) + elif system_user_id: + for namespace, pods in data.items(): + namespace_node = self.as_namespace_pod_tree_node( + namespace, parent_info, 'namespace', len(pods) + ) + tree_nodes.append(namespace_node) + return tree_nodes diff --git a/apps/common/tree.py b/apps/common/tree.py index fbade582e..8d5e40e5b 100644 --- a/apps/common/tree.py +++ b/apps/common/tree.py @@ -13,6 +13,7 @@ class TreeNode: pId = "" open = False iconSkin = "" + parentInfo = '' meta = {} _tree = None @@ -95,6 +96,7 @@ class TreeNodeSerializer(serializers.Serializer): name = serializers.CharField(max_length=128) title = serializers.CharField(max_length=128) pId = serializers.CharField(max_length=128) + parentInfo = serializers.CharField(max_length=4096, allow_blank=True) isParent = serializers.BooleanField(default=False) open = serializers.BooleanField(default=False) iconSkin = serializers.CharField(max_length=128, allow_blank=True) diff --git a/apps/orgs/models.py b/apps/orgs/models.py index efbee20b2..c8baec8fb 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -234,9 +234,9 @@ class Organization(models.Model): with tmp_to_org(self): return resource_model.objects.all().count() - def as_tree_node(self, pid, opened=True): + def as_tree_node(self, oid, pid, opened=True): node = TreeNode(**{ - 'id': str(self.id), + 'id': oid, 'name': self.name, 'title': self.name, 'pId': pid, diff --git a/apps/perms/api/application/user_permission/common.py b/apps/perms/api/application/user_permission/common.py index 560fcb519..707e2ca72 100644 --- a/apps/perms/api/application/user_permission/common.py +++ b/apps/perms/api/application/user_permission/common.py @@ -13,7 +13,6 @@ from rest_framework.generics import ( from orgs.utils import tmp_to_root_org from applications.models import Application from perms.utils.application.permission import ( - has_application_system_permission, get_application_system_user_ids, validate_permission, ) diff --git a/apps/perms/api/application/user_permission/user_permission_applications.py b/apps/perms/api/application/user_permission/user_permission_applications.py index be5fc6745..b9ddf1850 100644 --- a/apps/perms/api/application/user_permission/user_permission_applications.py +++ b/apps/perms/api/application/user_permission/user_permission_applications.py @@ -54,10 +54,15 @@ class ApplicationsAsTreeMixin(SerializeApplicationToTreeNodeMixin): 将应用序列化成树的结构返回 """ serializer_class = TreeNodeSerializer + user: None def list(self, request, *args, **kwargs): + tree_id = request.query_params.get('tree_id', None) + parent_info = request.query_params.get('parentInfo', None) queryset = self.filter_queryset(self.get_queryset()) - tree_nodes = self.serialize_applications_with_org(queryset) + tree_nodes = self.serialize_applications_with_org( + queryset, tree_id, parent_info, self.user + ) serializer = self.get_serializer(tree_nodes, many=True) return Response(data=serializer.data) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index b75ece14f..31123ad82 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -59,7 +59,7 @@ PyNaCl==1.2.1 python-dateutil==2.6.1 #python-gssapi==0.6.4 pytz==2018.3 -PyYAML==5.4 +PyYAML==6.0 redis==3.5.3 requests==2.25.1 jms-storage==0.0.40 @@ -125,3 +125,5 @@ pyzipper==0.3.5 python3-saml==1.12.0 python-keystoneclient==4.3.0 pymssql==2.1.5 +kubernetes==21.7.0 +websocket-client==1.2.3