Merge pull request #116554 from atiratree/eviction-resource-version-fix

API-initiated eviction: handle deleteOptions correctly

test/integration/evictions/evictions_test.go

@@ -38,12 +38,14 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/feature"
 	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
+	policyv1client "k8s.io/client-go/kubernetes/typed/policy/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/scale"
@@ -345,22 +347,34 @@ func TestEvictionWithFinalizers(t *testing.T) {
 	cases := map[string]struct {
 		enablePodDisruptionConditions bool
 		phase                         v1.PodPhase
+		dryRun                        bool
+		wantDisruptionTargetCond      bool
 	}{
 		"terminal pod with PodDisruptionConditions enabled": {
 			enablePodDisruptionConditions: true,
 			phase:                         v1.PodSucceeded,
+			wantDisruptionTargetCond:      true,
 		},
 		"terminal pod with PodDisruptionConditions disabled": {
 			enablePodDisruptionConditions: false,
 			phase:                         v1.PodSucceeded,
+			wantDisruptionTargetCond:      false,
 		},
 		"running pod with PodDisruptionConditions enabled": {
 			enablePodDisruptionConditions: true,
 			phase:                         v1.PodRunning,
+			wantDisruptionTargetCond:      true,
 		},
 		"running pod with PodDisruptionConditions disabled": {
 			enablePodDisruptionConditions: false,
 			phase:                         v1.PodRunning,
+			wantDisruptionTargetCond:      false,
+		},
+		"running pod with PodDisruptionConditions enabled should not update conditions in dry-run mode": {
+			enablePodDisruptionConditions: true,
+			phase:                         v1.PodRunning,
+			dryRun:                        true,
+			wantDisruptionTargetCond:      false,
 		},
 	}
 	for name, tc := range cases {
@@ -391,6 +405,9 @@ func TestEvictionWithFinalizers(t *testing.T) {
 
 			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, tc.phase)
 			deleteOption := metav1.DeleteOptions{}
+			if tc.dryRun {
+				deleteOption.DryRun = []string{metav1.DryRunAll}
+			}
 
 			eviction := newV1Eviction(ns.Name, pod.Name, deleteOption)
 
@@ -404,9 +421,9 @@ func TestEvictionWithFinalizers(t *testing.T) {
 				t.Fatalf("Failed to get the pod %q with error: %q", klog.KObj(pod), e)
 			}
 			_, cond := podutil.GetPodCondition(&updatedPod.Status, v1.PodConditionType(v1.DisruptionTarget))
-			if tc.enablePodDisruptionConditions == true && cond == nil {
+			if tc.wantDisruptionTargetCond == true && cond == nil {
 				t.Errorf("Pod %q does not have the expected condition: %q", klog.KObj(updatedPod), v1.DisruptionTarget)
-			} else if tc.enablePodDisruptionConditions == false && cond != nil {
+			} else if tc.wantDisruptionTargetCond == false && cond != nil {
 				t.Errorf("Pod %q has an unexpected condition: %q", klog.KObj(updatedPod), v1.DisruptionTarget)
 			}
 		})
@@ -489,12 +506,20 @@ func TestEvictionWithUnhealthyPodEvictionPolicy(t *testing.T) {
 					return pdb.Status.ExpectedPods == 1
 				})
 			}
+			// Eviction API can potentially return http.StatusTooManyRequests (429) or http.StatusGatewayTimeout (504) with retryAfterSeconds == 10s
+			// Do not retry - we want to test that the first request succeeds and make sure it doesn't unnecessarily block the test for 10s
+			policyV1NoRetriesRESTClient := &noRetriesRESTClient{Interface: clientSet.PolicyV1().RESTClient()}
+			policyV1NoRetriesClient := policyv1client.New(policyV1NoRetriesRESTClient)
+
 			deleteOption := metav1.DeleteOptions{}
 			eviction := newV1Eviction(ns.Name, pod.Name, deleteOption)
-			err := clientSet.PolicyV1().Evictions(ns.Name).Evict(ctx, eviction)
+			err := policyV1NoRetriesClient.Evictions(ns.Name).Evict(ctx, eviction)
 			if err != nil {
 				t.Fatalf("Eviction of pod failed %v", err)
 			}
+			if policyV1NoRetriesRESTClient.postCalls != 1 {
+				t.Fatalf("expected a single POST call, got %d", policyV1NoRetriesRESTClient.postCalls)
+			}
 
 			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 0, v1.PodRunning)
 			waitPDBStable(t, clientSet, ns.Name, pdb.Name, 0)
@@ -502,6 +527,109 @@ func TestEvictionWithUnhealthyPodEvictionPolicy(t *testing.T) {
 	}
 }
 
+// TestEvictionWithPrecondition tests eviction with delete preconditions
+func TestEvictionWithPrecondition(t *testing.T) {
+	cases := map[string]struct {
+		enforceResourceVersion     bool
+		injectWrongResourceVersion bool
+		enforceUID                 bool
+		injectWrongUID             bool
+		shouldErr                  bool
+	}{
+		"eviction enforcing resource version": {
+			enforceResourceVersion: true,
+		},
+		"eviction enforcing UID": {
+			enforceUID: true,
+		},
+		"eviction enforcing resource version and UID": {
+			enforceUID:             true,
+			enforceResourceVersion: true,
+		},
+		"eviction enforcing wrong resource version should fail": {
+			enforceResourceVersion:     true,
+			injectWrongResourceVersion: true,
+			shouldErr:                  true,
+		},
+		"eviction enforcing wrong UID should fail": {
+			enforceUID:     true,
+			injectWrongUID: true,
+			shouldErr:      true,
+		},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			closeFn, rm, informers, _, clientSet := rmSetup(t)
+			defer closeFn()
+
+			ns := framework.CreateNamespaceOrDie(clientSet, "eviction-with-preconditions", t)
+			defer framework.DeleteNamespaceOrDie(clientSet, ns, t)
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			informers.Start(ctx.Done())
+			go rm.Run(ctx)
+
+			pod := newPod("pod")
+			pod, err := clientSet.CoreV1().Pods(ns.Name).Create(context.TODO(), pod, metav1.CreateOptions{})
+			if err != nil {
+				t.Errorf("Failed to create pod: %q", err)
+			}
+
+			pod.Status.Phase = v1.PodRunning
+			addPodConditionReady(pod)
+
+			// generate a new resource version
+			updatedPod, err := clientSet.CoreV1().Pods(ns.Name).UpdateStatus(ctx, pod, metav1.UpdateOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			waitToObservePods(t, informers.Core().V1().Pods().Informer(), 1, v1.PodRunning)
+
+			deleteOption := metav1.DeleteOptions{}
+
+			if tc.enforceResourceVersion || tc.enforceUID {
+				deleteOption.Preconditions = &metav1.Preconditions{}
+			}
+
+			if tc.enforceResourceVersion {
+				if tc.injectWrongResourceVersion {
+					deleteOption.Preconditions.ResourceVersion = &pod.ResourceVersion
+				} else {
+					deleteOption.Preconditions.ResourceVersion = &updatedPod.ResourceVersion
+				}
+
+			}
+			if tc.enforceUID {
+				if tc.injectWrongUID {
+					newUID := uuid.NewUUID()
+					deleteOption.Preconditions.UID = &newUID
+				} else {
+					deleteOption.Preconditions.UID = &updatedPod.UID
+				}
+			}
+
+			// Eviction API can potentially return http.StatusTooManyRequests (429) or http.StatusGatewayTimeout (504) with retryAfterSeconds == 10s
+			// Do not retry - we want to test that the first request succeeds and make sure it doesn't unnecessarily block the test for 10s
+			policyV1NoRetriesRESTClient := &noRetriesRESTClient{Interface: clientSet.PolicyV1().RESTClient()}
+			policyV1NoRetriesClient := policyv1client.New(policyV1NoRetriesRESTClient)
+
+			eviction := newV1Eviction(ns.Name, updatedPod.Name, deleteOption)
+			err = policyV1NoRetriesClient.Evictions(ns.Name).Evict(ctx, eviction)
+			if err != nil && !tc.shouldErr {
+				t.Fatalf("Eviction of pod failed %q", err)
+			}
+			if err == nil && tc.shouldErr {
+				t.Fatal("Eviction of pod should fail")
+			}
+			if policyV1NoRetriesRESTClient.postCalls != 1 {
+				t.Fatalf("expected a single POST call, got %d", policyV1NoRetriesRESTClient.postCalls)
+			}
+		})
+	}
+}
+
 func newPod(podName string) *v1.Pod {
 	return &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -637,3 +765,16 @@ func waitPDB(t *testing.T, clientSet clientset.Interface, ns, pdbName string, co
 func unhealthyPolicyPtr(unhealthyPodEvictionPolicy policyv1.UnhealthyPodEvictionPolicyType) *policyv1.UnhealthyPodEvictionPolicyType {
 	return &unhealthyPodEvictionPolicy
 }
+
+type noRetriesRESTClient struct {
+	mu        sync.Mutex
+	postCalls int
+	restclient.Interface
+}
+
+func (n *noRetriesRESTClient) Post() *restclient.Request {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.postCalls++
+	return n.Interface.Post().MaxRetries(0)
+}