Merge pull request #129058 from dom4ha/nominated_cluster_event

Trigger all pods rescheduling on nominated node changes

test/integration/scheduler/eventhandler/eventhandler_test.go

@@ -18,13 +18,19 @@ package eventhandler
 
 import (
 	"context"
+	"fmt"
 	"testing"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/component-helpers/scheduling/corev1"
 	configv1 "k8s.io/kube-scheduler/config/v1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/scheduler"
 	configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
 	"k8s.io/kubernetes/pkg/scheduler/framework"
@@ -32,9 +38,12 @@ import (
 	st "k8s.io/kubernetes/pkg/scheduler/testing"
 	schedulerutils "k8s.io/kubernetes/test/integration/scheduler"
 	testutils "k8s.io/kubernetes/test/integration/util"
+	testingclock "k8s.io/utils/clock/testing"
 	"k8s.io/utils/ptr"
 )
 
+var lowPriority, mediumPriority, highPriority int32 = 100, 200, 300
+
 var _ framework.FilterPlugin = &fooPlugin{}
 
 type fooPlugin struct {
@@ -135,3 +144,136 @@ func TestUpdateNodeEvent(t *testing.T) {
 		t.Errorf("Pod %v was not scheduled: %v", pod.Name, err)
 	}
 }
+
+func TestUpdateNominatedNodeName(t *testing.T) {
+	fakeClock := testingclock.NewFakeClock(time.Now())
+	testBackoff := time.Minute
+	testContext := testutils.InitTestAPIServer(t, "test-event", nil)
+	capacity := map[v1.ResourceName]string{
+		v1.ResourceMemory: "32",
+	}
+	var cleanupPods []*v1.Pod
+
+	testNode := st.MakeNode().Name("node-0").Label("kubernetes.io/hostname", "node-0").Capacity(capacity).Obj()
+	// Note that the low priority pod that cannot fit with the mid priority, but can fit with the high priority one.
+	podLow := testutils.InitPausePod(&testutils.PausePodConfig{
+		Name:      "test-lp-pod",
+		Namespace: testContext.NS.Name,
+		Priority:  &lowPriority,
+		Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{
+			v1.ResourceMemory: *resource.NewQuantity(20, resource.DecimalSI)},
+		}})
+	cleanupPods = append(cleanupPods, podLow)
+	podMidNominated := testutils.InitPausePod(&testutils.PausePodConfig{
+		Name:      "test-nominated-pod",
+		Namespace: testContext.NS.Name,
+		Priority:  &mediumPriority,
+		Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{
+			v1.ResourceMemory: *resource.NewQuantity(25, resource.DecimalSI)},
+		}})
+	cleanupPods = append(cleanupPods, podMidNominated)
+	podHigh := testutils.InitPausePod(&testutils.PausePodConfig{
+		Name:      "test-hp-pod",
+		Namespace: testContext.NS.Name,
+		Priority:  &highPriority,
+		Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{
+			v1.ResourceMemory: *resource.NewQuantity(10, resource.DecimalSI)},
+		}})
+	cleanupPods = append(cleanupPods, podHigh)
+
+	tests := []struct {
+		name       string
+		updateFunc func(testCtx *testutils.TestContext)
+	}{
+		{
+			name: "Preempt nominated pod",
+			updateFunc: func(testCtx *testutils.TestContext) {
+				// Create high-priority pod and wait until it's scheduled (unnominate mid-priority pod)
+				pod, err := testutils.CreatePausePod(testCtx.ClientSet, podHigh)
+				if err != nil {
+					t.Fatalf("Creating pod error: %v", err)
+				}
+				if err = testutils.WaitForPodToSchedule(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
+					t.Fatalf("Pod %v was not scheduled: %v", pod.Name, err)
+				}
+			},
+		},
+		{
+			name: "Remove nominated pod",
+			updateFunc: func(testCtx *testutils.TestContext) {
+				if err := testutils.DeletePod(testCtx.ClientSet, podMidNominated.Name, podMidNominated.Namespace); err != nil {
+					t.Fatalf("Deleting pod error: %v", err)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		for _, qHintEnabled := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s, with queuehint(%v)", tt.name, qHintEnabled), func(t *testing.T) {
+				featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerQueueingHints, qHintEnabled)
+
+				testCtx, teardown := schedulerutils.InitTestSchedulerForFrameworkTest(t, testContext, 0, true,
+					scheduler.WithClock(fakeClock),
+					// UpdateFunc needs to be called when the nominated pod is still in the backoff queue, thus small, but non 0 value.
+					scheduler.WithPodInitialBackoffSeconds(int64(testBackoff.Seconds())),
+					scheduler.WithPodMaxBackoffSeconds(int64(testBackoff.Seconds())),
+				)
+				defer teardown()
+
+				_, err := testutils.CreateNode(testCtx.ClientSet, testNode)
+				if err != nil {
+					t.Fatalf("Creating node error: %v", err)
+				}
+
+				// Create initial low-priority pod and wait until it's scheduled.
+				pod, err := testutils.CreatePausePod(testCtx.ClientSet, podLow)
+				if err != nil {
+					t.Fatalf("Creating pod error: %v", err)
+				}
+				if err := testutils.WaitForPodToSchedule(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
+					t.Fatalf("Pod %v was not scheduled: %v", pod.Name, err)
+				}
+
+				// Create mid-priority pod and wait until it becomes nominated (preempt low-priority pod) and remain uschedulable.
+				pod, err = testutils.CreatePausePod(testCtx.ClientSet, podMidNominated)
+				if err != nil {
+					t.Fatalf("Creating pod error: %v", err)
+				}
+				if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
+					t.Errorf("NominatedNodeName field was not set for pod %v: %v", pod.Name, err)
+				}
+				if err := testutils.WaitForPodUnschedulable(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
+					t.Errorf("Pod %v haven't become unschedulabe: %v", pod.Name, err)
+				}
+
+				// Remove the initial low-priority pod, which will move the nominated unschedulable pod back to the backoff queue.
+				if err := testutils.DeletePod(testCtx.ClientSet, podLow.Name, podLow.Namespace); err != nil {
+					t.Fatalf("Deleting pod error: %v", err)
+				}
+
+				// Create another low-priority pods which cannot be scheduled because the mid-priority pod is nominated on the node and the node doesn't have enough resource to have two pods both.
+				pod, err = testutils.CreatePausePod(testCtx.ClientSet, podLow)
+				if err != nil {
+					t.Fatalf("Creating pod error: %v", err)
+				}
+				if err := testutils.WaitForPodUnschedulable(testCtx.Ctx, testCtx.ClientSet, pod); err != nil {
+					t.Fatalf("Pod %v was not scheduled: %v", pod.Name, err)
+				}
+
+				// Update causing the nominated pod to be removed or to get its nominated node name removed, which should trigger scheduling of the low priority pod.
+				// Note that the update has to happen since the nominated pod is still in the backoffQ to actually test updates of nominated, but not bound yet pods.
+				tt.updateFunc(testCtx)
+
+				// Advance time by the maxPodBackoffSeconds to move low priority pod out of the backoff queue.
+				fakeClock.Step(testBackoff)
+
+				// Expect the low-priority pod is notified about unnominated mid-pririty pod and gets scheduled, as it should fit this time.
+				if err := testutils.WaitForPodToSchedule(testCtx.Ctx, testCtx.ClientSet, podLow); err != nil {
+					t.Fatalf("Pod %v was not scheduled: %v", podLow.Name, err)
+				}
+				testutils.CleanupPods(testCtx.Ctx, testCtx.ClientSet, t, cleanupPods)
+			})
+		}
+	}
+}

test/integration/scheduler/preemption/preemption_test.go

@@ -84,26 +84,6 @@ const filterPluginName = "filter-plugin"
 
 var lowPriority, mediumPriority, highPriority = int32(100), int32(200), int32(300)
 
-func waitForNominatedNodeNameWithTimeout(ctx context.Context, cs clientset.Interface, pod *v1.Pod, timeout time.Duration) error {
-	if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, timeout, false, func(ctx context.Context) (bool, error) {
-		pod, err := cs.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		if len(pod.Status.NominatedNodeName) > 0 {
-			return true, nil
-		}
-		return false, err
-	}); err != nil {
-		return fmt.Errorf(".status.nominatedNodeName of Pod %v/%v did not get set: %v", pod.Namespace, pod.Name, err)
-	}
-	return nil
-}
-
-func waitForNominatedNodeName(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error {
-	return waitForNominatedNodeNameWithTimeout(ctx, cs, pod, wait.ForeverTestTimeout)
-}
-
 const tokenFilterName = "token-filter"
 
 // tokenFilter is a fake plugin that implements PreFilter and Filter.
@@ -504,7 +484,7 @@ func TestPreemption(t *testing.T) {
 				}
 				// Also check that the preemptor pod gets the NominatedNodeName field set.
 				if len(test.preemptedPodIndexes) > 0 {
-					if err := waitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
+					if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
 						t.Errorf("NominatedNodeName field was not set for pod %v: %v", preemptor.Name, err)
 					}
 				}
@@ -1086,7 +1066,7 @@ func TestNonPreemption(t *testing.T) {
 					t.Fatalf("Error while creating preemptor: %v", err)
 				}
 
-				err = waitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptorPod, 5*time.Second)
+				err = testutils.WaitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptorPod, 5*time.Second)
 				// test.PreemptionPolicy == nil means we expect the preemptor to be nominated.
 				expect := test.PreemptionPolicy == nil
 				// err == nil indicates the preemptor is indeed nominated.
@@ -1168,7 +1148,7 @@ func TestDisablePreemption(t *testing.T) {
 				}
 
 				// Ensure preemptor should not be nominated.
-				if err := waitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptor, 5*time.Second); err == nil {
+				if err := testutils.WaitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptor, 5*time.Second); err == nil {
 					t.Errorf("Preemptor %v should not be nominated", preemptor.Name)
 				}
 
@@ -1381,7 +1361,7 @@ func TestPreemptionStarvation(t *testing.T) {
 					t.Errorf("Error while creating the preempting pod: %v", err)
 				}
 				// Check if .status.nominatedNodeName of the preemptor pod gets set.
-				if err := waitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
+				if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
 					t.Errorf(".status.nominatedNodeName was not set for pod %v/%v: %v", preemptor.Namespace, preemptor.Name, err)
 				}
 				// Make sure that preemptor is scheduled after preemptions.
@@ -1481,7 +1461,7 @@ func TestPreemptionRaces(t *testing.T) {
 						}
 					}
 					// Check that the preemptor pod gets nominated node name.
-					if err := waitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
+					if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
 						t.Errorf(".status.nominatedNodeName was not set for pod %v/%v: %v", preemptor.Namespace, preemptor.Name, err)
 					}
 					// Make sure that preemptor is scheduled after preemptions.
@@ -1577,8 +1557,8 @@ func TestNominatedNodeCleanUp(t *testing.T) {
 			},
 			postChecks: []func(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error{
 				testutils.WaitForPodToSchedule,
-				waitForNominatedNodeName,
-				waitForNominatedNodeName,
+				testutils.WaitForNominatedNodeName,
+				testutils.WaitForNominatedNodeName,
 			},
 		},
 		{
@@ -1597,7 +1577,7 @@ func TestNominatedNodeCleanUp(t *testing.T) {
 			},
 			postChecks: []func(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error{
 				testutils.WaitForPodToSchedule,
-				waitForNominatedNodeName,
+				testutils.WaitForNominatedNodeName,
 				testutils.WaitForPodToSchedule,
 			},
 			podNamesToDelete: []string{"low"},
@@ -1615,7 +1595,7 @@ func TestNominatedNodeCleanUp(t *testing.T) {
 			},
 			postChecks: []func(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error{
 				testutils.WaitForPodToSchedule,
-				waitForNominatedNodeName,
+				testutils.WaitForNominatedNodeName,
 			},
 			// Delete the node to simulate an ErrNoNodesAvailable error.
 			deleteNode:       true,
@@ -1634,7 +1614,7 @@ func TestNominatedNodeCleanUp(t *testing.T) {
 			},
 			postChecks: []func(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error{
 				testutils.WaitForPodToSchedule,
-				waitForNominatedNodeName,
+				testutils.WaitForNominatedNodeName,
 			},
 			podNamesToDelete: []string{fmt.Sprintf("low-%v", doNotFailMe)},
 			customPlugins: &configv1.Plugins{
@@ -1990,7 +1970,7 @@ func TestPDBInPreemption(t *testing.T) {
 				}
 				// Also check if .status.nominatedNodeName of the preemptor pod gets set.
 				if len(test.preemptedPodIndexes) > 0 {
-					if err := waitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
+					if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
 						t.Errorf("Test [%v]: .status.nominatedNodeName was not set for pod %v/%v: %v", test.name, preemptor.Namespace, preemptor.Name, err)
 					}
 				}
@@ -2483,7 +2463,7 @@ func TestReadWriteOncePodPreemption(t *testing.T) {
 				}
 				// Also check that the preemptor pod gets the NominatedNodeName field set.
 				if len(test.preemptedPodIndexes) > 0 {
-					if err := waitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
+					if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil {
 						t.Errorf("NominatedNodeName field was not set for pod %v: %v", preemptor.Name, err)
 					}
 				}

test/integration/util/util.go

@@ -1160,3 +1160,23 @@ func NextPodOrDie(t *testing.T, testCtx *TestContext) *schedulerframework.Queued
 	}
 	return podInfo
 }
+
+func WaitForNominatedNodeNameWithTimeout(ctx context.Context, cs clientset.Interface, pod *v1.Pod, timeout time.Duration) error {
+	if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, timeout, false, func(ctx context.Context) (bool, error) {
+		pod, err := cs.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if len(pod.Status.NominatedNodeName) > 0 {
+			return true, nil
+		}
+		return false, err
+	}); err != nil {
+		return fmt.Errorf(".status.nominatedNodeName of Pod %v/%v did not get set: %w", pod.Namespace, pod.Name, err)
+	}
+	return nil
+}
+
+func WaitForNominatedNodeName(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error {
+	return WaitForNominatedNodeNameWithTimeout(ctx, cs, pod, wait.ForeverTestTimeout)
+}