mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Add the feature gate OrderedNamespaceDeletion
for apiserver.
This commit is contained in:
parent
a9964aec6b
commit
f12dbc31a9
@ -32,9 +32,11 @@ import (
|
|||||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||||
"k8s.io/apimachinery/pkg/util/sets"
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||||
"k8s.io/client-go/discovery"
|
"k8s.io/client-go/discovery"
|
||||||
v1clientset "k8s.io/client-go/kubernetes/typed/core/v1"
|
v1clientset "k8s.io/client-go/kubernetes/typed/core/v1"
|
||||||
"k8s.io/client-go/metadata"
|
"k8s.io/client-go/metadata"
|
||||||
|
"k8s.io/kubernetes/pkg/features"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NamespacedResourcesDeleterInterface is the interface to delete a namespace with all resources in it.
|
// NamespacedResourcesDeleterInterface is the interface to delete a namespace with all resources in it.
|
||||||
@ -526,7 +528,41 @@ func (d *namespacedResourcesDeleter) deleteAllContent(ctx context.Context, ns *v
|
|||||||
gvrToNumRemaining: map[schema.GroupVersionResource]int{},
|
gvrToNumRemaining: map[schema.GroupVersionResource]int{},
|
||||||
finalizersToNumRemaining: map[string]int{},
|
finalizersToNumRemaining: map[string]int{},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
podsGVR := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}
|
||||||
|
if _, hasPods := groupVersionResources[podsGVR]; hasPods && utilfeature.DefaultFeatureGate.Enabled(features.OrderedNamespaceDeletion) {
|
||||||
|
// Ensure all pods in the namespace are deleted first
|
||||||
|
gvrDeletionMetadata, err := d.deleteAllContentForGroupVersionResource(ctx, podsGVR, namespace, namespaceDeletedAt)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("failed to delete pods for namespace: %s, err: %w", namespace, err))
|
||||||
|
conditionUpdater.ProcessDeleteContentErr(err)
|
||||||
|
}
|
||||||
|
if gvrDeletionMetadata.finalizerEstimateSeconds > estimate {
|
||||||
|
estimate = gvrDeletionMetadata.finalizerEstimateSeconds
|
||||||
|
}
|
||||||
|
if gvrDeletionMetadata.numRemaining > 0 {
|
||||||
|
numRemainingTotals.gvrToNumRemaining[podsGVR] = gvrDeletionMetadata.numRemaining
|
||||||
|
for finalizer, numRemaining := range gvrDeletionMetadata.finalizersToNumRemaining {
|
||||||
|
if numRemaining == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
numRemainingTotals.finalizersToNumRemaining[finalizer] += numRemaining
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if any pods remain before proceeding to delete other resources
|
||||||
|
if numRemainingTotals.gvrToNumRemaining[podsGVR] > 0 {
|
||||||
|
logger.V(5).Info("Namespace controller - pods still remain, delaying deletion of other resources", "namespace", namespace)
|
||||||
|
return estimate, utilerrors.NewAggregate(errs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proceed with deleting other resources in the namespace
|
||||||
for gvr := range groupVersionResources {
|
for gvr := range groupVersionResources {
|
||||||
|
if utilfeature.DefaultFeatureGate.Enabled(features.OrderedNamespaceDeletion) && gvr.Group == podsGVR.Group &&
|
||||||
|
gvr.Version == podsGVR.Version && gvr.Resource == podsGVR.Resource {
|
||||||
|
continue
|
||||||
|
}
|
||||||
gvrDeletionMetadata, err := d.deleteAllContentForGroupVersionResource(ctx, gvr, namespace, namespaceDeletedAt)
|
gvrDeletionMetadata, err := d.deleteAllContentForGroupVersionResource(ctx, gvr, namespace, namespaceDeletedAt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// If there is an error, hold on to it but proceed with all the remaining
|
// If there is an error, hold on to it but proceed with all the remaining
|
||||||
|
@ -546,6 +546,12 @@ const (
|
|||||||
// Permits kubelet to run with swap enabled.
|
// Permits kubelet to run with swap enabled.
|
||||||
NodeSwap featuregate.Feature = "NodeSwap"
|
NodeSwap featuregate.Feature = "NodeSwap"
|
||||||
|
|
||||||
|
// owner: @cici37
|
||||||
|
// kep: https://kep.k8s.io/5080
|
||||||
|
//
|
||||||
|
// Enables ordered namespace deletion.
|
||||||
|
OrderedNamespaceDeletion featuregate.Feature = "OrderedNamespaceDeletion"
|
||||||
|
|
||||||
// owner: @mortent, @atiratree, @ravig
|
// owner: @mortent, @atiratree, @ravig
|
||||||
// kep: http://kep.k8s.io/3018
|
// kep: http://kep.k8s.io/3018
|
||||||
// alpha: v1.26
|
// alpha: v1.26
|
||||||
@ -1137,6 +1143,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
|
|||||||
|
|
||||||
NodeSwap: {Default: true, PreRelease: featuregate.Beta},
|
NodeSwap: {Default: true, PreRelease: featuregate.Beta},
|
||||||
|
|
||||||
|
OrderedNamespaceDeletion: {Default: false, PreRelease: featuregate.Beta},
|
||||||
|
|
||||||
PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.33
|
PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.33
|
||||||
|
|
||||||
PersistentVolumeLastPhaseTransitionTime: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.33
|
PersistentVolumeLastPhaseTransitionTime: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.33
|
||||||
|
@ -20,6 +20,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"k8s.io/kubernetes/pkg/features"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@ -475,3 +476,106 @@ func unstructuredToNamespace(obj *unstructured.Unstructured) (*v1.Namespace, err
|
|||||||
|
|
||||||
return ns, err
|
return ns, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var _ = SIGDescribe("OrderedNamespaceDeletion", func() {
|
||||||
|
f := framework.NewDefaultFramework("namespacedeletion")
|
||||||
|
f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
|
||||||
|
|
||||||
|
f.It("namespace deletion should delete pod first", feature.OrderedNamespaceDeletion, framework.WithFeatureGate(features.OrderedNamespaceDeletion), func(ctx context.Context) {
|
||||||
|
ensurePodsAreRemovedFirstInOrderedNamespaceDeletion(ctx, f)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
func ensurePodsAreRemovedFirstInOrderedNamespaceDeletion(ctx context.Context, f *framework.Framework) {
|
||||||
|
ginkgo.By("Creating a test namespace")
|
||||||
|
namespaceName := "nsdeletetest"
|
||||||
|
namespace, err := f.CreateNamespace(ctx, namespaceName, nil)
|
||||||
|
framework.ExpectNoError(err, "failed to create namespace: %s", namespaceName)
|
||||||
|
nsName := namespace.Name
|
||||||
|
|
||||||
|
ginkgo.By("Waiting for a default service account to be provisioned in namespace")
|
||||||
|
err = framework.WaitForDefaultServiceAccountInNamespace(ctx, f.ClientSet, nsName)
|
||||||
|
framework.ExpectNoError(err, "failure while waiting for a default service account to be provisioned in namespace: %s", nsName)
|
||||||
|
|
||||||
|
ginkgo.By("Creating a pod with finalizer in the namespace")
|
||||||
|
podName := "test-pod"
|
||||||
|
pod := &v1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: podName,
|
||||||
|
Finalizers: []string{
|
||||||
|
"e2e.example.com/finalizer",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Spec: v1.PodSpec{
|
||||||
|
Containers: []v1.Container{
|
||||||
|
{
|
||||||
|
Name: "nginx",
|
||||||
|
Image: imageutils.GetPauseImageName(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
pod, err = f.ClientSet.CoreV1().Pods(nsName).Create(ctx, pod, metav1.CreateOptions{})
|
||||||
|
framework.ExpectNoError(err, "failed to create pod %s in namespace: %s", podName, nsName)
|
||||||
|
|
||||||
|
ginkgo.By("Waiting for the pod to have running status")
|
||||||
|
framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
|
||||||
|
|
||||||
|
configMapName := "test-configmap"
|
||||||
|
ginkgo.By(fmt.Sprintf("Creating a configmap %q in namespace %q", configMapName, nsName))
|
||||||
|
configMap := &v1.ConfigMap{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: configMapName,
|
||||||
|
Namespace: nsName,
|
||||||
|
},
|
||||||
|
Data: map[string]string{
|
||||||
|
"key": "value",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err = f.ClientSet.CoreV1().ConfigMaps(nsName).Create(ctx, configMap, metav1.CreateOptions{})
|
||||||
|
framework.ExpectNoError(err, "failed to create configmap %q in namespace %q", configMapName, nsName)
|
||||||
|
|
||||||
|
ginkgo.By("Deleting the namespace")
|
||||||
|
err = f.ClientSet.CoreV1().Namespaces().Delete(ctx, nsName, metav1.DeleteOptions{})
|
||||||
|
framework.ExpectNoError(err, "failed to delete namespace: %s", nsName)
|
||||||
|
// wait 10 seconds to allow the namespace controller to process
|
||||||
|
time.Sleep(10 * time.Second)
|
||||||
|
ginkgo.By("the pod should be deleted before processing deletion for other resources")
|
||||||
|
framework.ExpectNoError(wait.PollUntilContextTimeout(ctx, 2*time.Second, 60*time.Second, true,
|
||||||
|
func(ctx context.Context) (bool, error) {
|
||||||
|
_, err = f.ClientSet.CoreV1().ConfigMaps(nsName).Get(ctx, configMapName, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err, "configmap %q should still exist in namespace %q", configMapName, nsName)
|
||||||
|
// the pod should exist and has a deletionTimestamp set
|
||||||
|
pod, err = f.ClientSet.CoreV1().Pods(nsName).Get(ctx, pod.Name, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err, "failed to get pod %q in namespace %q", pod.Name, nsName)
|
||||||
|
if pod.DeletionTimestamp == nil {
|
||||||
|
framework.Failf("Pod %q in namespace %q does not have a metadata.deletionTimestamp set", pod.Name, nsName)
|
||||||
|
}
|
||||||
|
_, err = f.ClientSet.CoreV1().Namespaces().Get(ctx, nsName, metav1.GetOptions{})
|
||||||
|
if err != nil && apierrors.IsNotFound(err) {
|
||||||
|
return false, fmt.Errorf("namespace %s was deleted unexpectedly", nsName)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}))
|
||||||
|
|
||||||
|
ginkgo.By(fmt.Sprintf("Removing finalizer from pod %q in namespace %q", podName, nsName))
|
||||||
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||||
|
pod, err = f.ClientSet.CoreV1().Pods(nsName).Get(ctx, podName, metav1.GetOptions{})
|
||||||
|
framework.ExpectNoError(err, "failed to get pod %q in namespace %q", pod.Name, nsName)
|
||||||
|
pod.Finalizers = []string{}
|
||||||
|
_, err = f.ClientSet.CoreV1().Pods(nsName).Update(ctx, pod, metav1.UpdateOptions{})
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
framework.ExpectNoError(err, "failed to update pod %q and remove finalizer in namespace %q", podName, nsName)
|
||||||
|
|
||||||
|
ginkgo.By("Waiting for the namespace to be removed.")
|
||||||
|
maxWaitSeconds := int64(60) + *pod.Spec.TerminationGracePeriodSeconds
|
||||||
|
framework.ExpectNoError(wait.PollUntilContextTimeout(ctx, 1*time.Second, time.Duration(maxWaitSeconds)*time.Second, true,
|
||||||
|
func(ctx context.Context) (bool, error) {
|
||||||
|
_, err = f.ClientSet.CoreV1().Namespaces().Get(ctx, namespace.Name, metav1.GetOptions{})
|
||||||
|
if err != nil && apierrors.IsNotFound(err) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
@ -251,6 +251,10 @@ var (
|
|||||||
// TODO: document the feature (owning SIG, when to use this feature for a test)
|
// TODO: document the feature (owning SIG, when to use this feature for a test)
|
||||||
NodeOutOfServiceVolumeDetach = framework.WithFeature(framework.ValidFeatures.Add("NodeOutOfServiceVolumeDetach"))
|
NodeOutOfServiceVolumeDetach = framework.WithFeature(framework.ValidFeatures.Add("NodeOutOfServiceVolumeDetach"))
|
||||||
|
|
||||||
|
// Owner: sig-api-machinery
|
||||||
|
// Marks tests that enforce ordered namespace deletion.
|
||||||
|
OrderedNamespaceDeletion = framework.WithFeature(framework.ValidFeatures.Add("OrderedNamespaceDeletion"))
|
||||||
|
|
||||||
// Owner: sig-network
|
// Owner: sig-network
|
||||||
// Marks a single test that tests cluster DNS performance with many services.
|
// Marks a single test that tests cluster DNS performance with many services.
|
||||||
PerformanceDNS = framework.WithFeature(framework.ValidFeatures.Add("PerformanceDNS"))
|
PerformanceDNS = framework.WithFeature(framework.ValidFeatures.Add("PerformanceDNS"))
|
||||||
|
Loading…
Reference in New Issue
Block a user