diff --git a/cli/resources/cleanResources.go b/cli/resources/cleanResources.go index 8fc3b9936..e92e1471d 100644 --- a/cli/resources/cleanResources.go +++ b/cli/resources/cleanResources.go @@ -42,14 +42,29 @@ func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, ku defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider, mizuResourcesNamespace) } - if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil { - resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName) + if resources, err := kubernetesProvider.ListManagedClusterRoles(ctx); err != nil { + resourceDesc := "ClusterRoles" handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveClusterRole(ctx, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("ClusterRole %s", resource.Name) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } - if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName) + + if resources, err := kubernetesProvider.ListManagedClusterRoleBindings(ctx); err != nil { + resourceDesc := "ClusterRoleBindings" handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", resource.Name) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } return leftoverResources @@ -91,14 +106,40 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P handleDeletionError(err, resourceDesc, &leftoverResources) } - if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil { - resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, mizuResourcesNamespace) + if resources, err := kubernetesProvider.ListManagedServiceAccounts(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("ServiceAccounts in namespace %s", mizuResourcesNamespace) handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("ServiceAccount %s in namespace %s", resource.Name, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } - if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.RoleName); err != nil { - resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, mizuResourcesNamespace) + if resources, err := kubernetesProvider.ListManagedRoles(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("Roles in namespace %s", mizuResourcesNamespace) handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("Role %s in namespace %s", resource.Name, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } + } + + if resources, err := kubernetesProvider.ListManagedRoleBindings(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("RoleBindings in namespace %s", mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", resource.Name, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } if err := kubernetesProvider.RemovePod(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { @@ -107,10 +148,6 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P } //install mode resources - if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.RoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, mizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } if err := kubernetesProvider.RemoveDeployment(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace) @@ -122,16 +159,6 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P handleDeletionError(err, resourceDesc, &leftoverResources) } - if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil { - resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, mizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, mizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - return leftoverResources } diff --git a/shared/kubernetes/consts.go b/shared/kubernetes/consts.go index 68acdd1ec..de84d5714 100644 --- a/shared/kubernetes/consts.go +++ b/shared/kubernetes/consts.go @@ -17,3 +17,12 @@ const ( PersistentVolumeClaimName = MizuResourcesPrefix + "volume-claim" MinKubernetesServerVersion = "1.16.0" ) + +const ( + LabelPrefixApp = "app.kubernetes.io/" + LabelManagedBy = LabelPrefixApp + "managed-by" + LabelCreatedBy = LabelPrefixApp + "created-by" + LabelValueMizu = "mizu" + LabelValueMizuCLI = "mizu-cli" + LabelValueMizuAgent = "mizu-agent" +) diff --git a/shared/kubernetes/provider.go b/shared/kubernetes/provider.go index a1a0195d1..d0a28cc48 100644 --- a/shared/kubernetes/provider.go +++ b/shared/kubernetes/provider.go @@ -43,6 +43,8 @@ type Provider struct { kubernetesConfig clientcmd.ClientConfig clientConfig restclient.Config Namespace string + managedBy string + createdBy string } const ( @@ -86,6 +88,8 @@ func NewProvider(kubeConfigPath string) (*Provider, error) { clientSet: clientSet, kubernetesConfig: kubernetesConfig, clientConfig: *restClientConfig, + managedBy: LabelValueMizu, + createdBy: LabelValueMizuCLI, }, nil } @@ -103,6 +107,8 @@ func NewProviderInCluster() (*Provider, error) { clientSet: clientSet, kubernetesConfig: nil, // not relevant in cluster clientConfig: *restClientConfig, + managedBy: LabelValueMizu, + createdBy: LabelValueMizuAgent, }, nil } @@ -158,6 +164,10 @@ func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*co namespaceSpec := &core.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, } return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{}) @@ -243,7 +253,11 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: opts.PodName, - Labels: map[string]string{"app": opts.PodName}, + Labels: map[string]string{ + "app": opts.PodName, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: core.PodSpec{ Containers: []core.Container{ @@ -303,6 +317,10 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string deployment := &v1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: v1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -319,6 +337,10 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s service := core.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: core.ServiceSpec{ Ports: []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}}, @@ -351,13 +373,21 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: serviceAccountName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, } clusterRole := &rbac.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: clusterRoleName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Rules: []rbac.PolicyRule{ { @@ -370,7 +400,11 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, clusterRoleBinding := &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: clusterRoleBindingName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, RoleRef: rbac.RoleRef{ Name: clusterRoleName, @@ -404,13 +438,21 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: serviceAccountName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, } role := &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ Name: roleName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Rules: []rbac.PolicyRule{ { @@ -423,7 +465,11 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, roleBinding := &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: roleBindingName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, RoleRef: rbac.RoleRef{ Name: roleName, @@ -457,7 +503,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str role := &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ Name: roleName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Rules: []rbac.PolicyRule{ { @@ -475,7 +525,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str roleBinding := &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: roleBindingName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, RoleRef: rbac.RoleRef{ Name: roleName, @@ -588,6 +642,10 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, }, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Data: configMapData, } @@ -746,14 +804,23 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac podSpec.WithVolumes(&configMapVolume, procfsVolume) podTemplate := applyconfcore.PodTemplateSpec() - podTemplate.WithLabels(map[string]string{"app": tapperPodName}) + podTemplate.WithLabels(map[string]string{ + "app": tapperPodName, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }) podTemplate.WithSpec(podSpec) labelSelector := applyconfmeta.LabelSelector() labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName}) daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace) - daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) + daemonSet. + WithLabels(map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }). + WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) _, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) return err @@ -827,6 +894,41 @@ func (provider *Provider) GetNamespaceEvents(ctx context.Context, namespace stri return eventList.String(), nil } +func (provider *Provider) ListManagedServiceAccounts(ctx context.Context, namespace string) (*core.ServiceAccountList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.CoreV1().ServiceAccounts(namespace).List(ctx, listOptions) +} + +func (provider *Provider) ListManagedClusterRoles(ctx context.Context) (*rbac.ClusterRoleList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions) +} + +func (provider *Provider) ListManagedClusterRoleBindings(ctx context.Context) (*rbac.ClusterRoleBindingList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions) +} + +func (provider *Provider) ListManagedRoles(ctx context.Context, namespace string) (*rbac.RoleList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().Roles(namespace).List(ctx, listOptions) +} + +func (provider *Provider) ListManagedRoleBindings(ctx context.Context, namespace string) (*rbac.RoleBindingList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().RoleBindings(namespace).List(ctx, listOptions) +} + func (provider *Provider) IsDefaultStorageProviderAvailable(ctx context.Context) (bool, error) { storageClassList, err := provider.clientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) if err != nil { @@ -845,6 +947,10 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names volumeClaim := &core.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: volumeClaimName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: core.PersistentVolumeClaimSpec{ AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce},