Add e2e for GracefulNodeShutdownBasedOnPodPriority

test/e2e_node/node_shutdown_linux_test.go

@@ -35,13 +35,15 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	v1 "k8s.io/api/core/v1"
+	schedulingv1 "k8s.io/api/scheduling/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/features"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	testutils "k8s.io/kubernetes/test/utils"
 )
 
-var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShutdown]", func() {
+var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShutdown] [NodeFeature:GracefulNodeShutdownBasedOnPodPriority]", func() {
 	f := framework.NewDefaultFramework("graceful-node-shutdown")
 	ginkgo.Context("when gracefully shutting down", func() {
 
@@ -54,6 +56,10 @@ var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShut
 		)
 
 		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
+			initialConfig.FeatureGates = map[string]bool{
+				string(features.GracefulNodeShutdown):                   true,
+				string(features.GracefulNodeShutdownBasedOnPodPriority): false,
+			}
 			initialConfig.ShutdownGracePeriod = metav1.Duration{Duration: nodeShutdownGracePeriod}
 			initialConfig.ShutdownGracePeriodCriticalPods = metav1.Duration{Duration: nodeShutdownGracePeriodCriticalPods}
 		})
@@ -77,10 +83,10 @@ var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShut
 
 			// Define test pods
 			pods := []*v1.Pod{
-				getGracePeriodOverrideTestPod("period-120", nodeName, 120, false),
+				getGracePeriodOverrideTestPod("period-120", nodeName, 120, ""),
-				getGracePeriodOverrideTestPod("period-5", nodeName, 5, false),
+				getGracePeriodOverrideTestPod("period-5", nodeName, 5, ""),
-				getGracePeriodOverrideTestPod("period-critical-120", nodeName, 120, true),
+				getGracePeriodOverrideTestPod("period-critical-120", nodeName, 120, scheduling.SystemNodeCritical),
-				getGracePeriodOverrideTestPod("period-critical-5", nodeName, 5, true),
+				getGracePeriodOverrideTestPod("period-critical-5", nodeName, 5, scheduling.SystemNodeCritical),
 			}
 
 			ginkgo.By("Creating batch pods")
@@ -117,12 +123,12 @@ var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShut
 				for _, pod := range list.Items {
 					if kubelettypes.IsCriticalPod(&pod) {
 						if isPodShutdown(&pod) {
-							framework.Logf("Expecting critcal pod to be running, but it's not currently. Pod: %q, Pod Status %+v", pod.Name, pod.Status)
+							framework.Logf("Expecting critical pod to be running, but it's not currently. Pod: %q, Pod Status %+v", pod.Name, pod.Status)
 							return fmt.Errorf("critical pod should not be shutdown, phase: %s", pod.Status.Phase)
 						}
 					} else {
 						if !isPodShutdown(&pod) {
-							framework.Logf("Expecting non-critcal pod to be shutdown, but it's not currently. Pod: %q, Pod Status %+v", pod.Name, pod.Status)
+							framework.Logf("Expecting non-critical pod to be shutdown, but it's not currently. Pod: %q, Pod Status %+v", pod.Name, pod.Status)
 							return fmt.Errorf("pod should be shutdown, phase: %s", pod.Status.Phase)
 						}
 					}
@@ -207,9 +213,190 @@ var _ = SIGDescribe("GracefulNodeShutdown [Serial] [NodeFeature:GracefulNodeShut
 			}, nodeStatusUpdateTimeout, pollInterval).Should(gomega.BeNil())
 		})
 	})
+
+	ginkgo.Context("when gracefully shutting down with Pod priority", func() {
+
+		const (
+			pollInterval           = 1 * time.Second
+			podStatusUpdateTimeout = 10 * time.Second
+		)
+
+		var (
+			customClassA = getPriorityClass("custom-class-a", 100000)
+			customClassB = getPriorityClass("custom-class-b", 10000)
+			customClassC = getPriorityClass("custom-class-c", 1000)
+		)
+
+		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
+			initialConfig.FeatureGates = map[string]bool{
+				string(features.GracefulNodeShutdown):                   true,
+				string(features.GracefulNodeShutdownBasedOnPodPriority): true,
+			}
+			initialConfig.ShutdownGracePeriodByPodPriority = []kubeletconfig.ShutdownGracePeriodByPodPriority{
+				{
+					Priority:                   scheduling.SystemCriticalPriority,
+					ShutdownGracePeriodSeconds: int64(podStatusUpdateTimeout / time.Second),
+				},
+				{
+					Priority:                   customClassA.Value,
+					ShutdownGracePeriodSeconds: int64(podStatusUpdateTimeout / time.Second),
+				},
+				{
+					Priority:                   customClassB.Value,
+					ShutdownGracePeriodSeconds: int64(podStatusUpdateTimeout / time.Second),
+				},
+				{
+					Priority:                   customClassC.Value,
+					ShutdownGracePeriodSeconds: int64(podStatusUpdateTimeout / time.Second),
+				},
+				{
+					Priority:                   scheduling.DefaultPriorityWhenNoDefaultClassExists,
+					ShutdownGracePeriodSeconds: int64(podStatusUpdateTimeout / time.Second),
+				},
+			}
+
+		})
+
+		ginkgo.BeforeEach(func() {
+			ginkgo.By("Wait for the node to be ready")
+			waitForNodeReady()
+
+			for _, customClass := range []*schedulingv1.PriorityClass{customClassA, customClassB, customClassC} {
+				_, err := f.ClientSet.SchedulingV1().PriorityClasses().Create(context.Background(), customClass, metav1.CreateOptions{})
+				framework.ExpectNoError(err)
+			}
+		})
+
+		ginkgo.AfterEach(func() {
+			ginkgo.By("Emitting Shutdown false signal; cancelling the shutdown")
+			err := emitSignalPrepareForShutdown(false)
+			framework.ExpectNoError(err)
+		})
+
+		ginkgo.It("should be able to gracefully shutdown pods with various grace periods", func() {
+			nodeName := getNodeName(f)
+			nodeSelector := fields.Set{
+				"spec.nodeName": nodeName,
+			}.AsSelector().String()
+
+			// Define test pods
+			pods := []*v1.Pod{
+				getGracePeriodOverrideTestPod("period-120", nodeName, 120, ""),
+				getGracePeriodOverrideTestPod("period-5", nodeName, 5, ""),
+				getGracePeriodOverrideTestPod("period-c-120", nodeName, 120, customClassC.Name),
+				getGracePeriodOverrideTestPod("period-c-5", nodeName, 5, customClassC.Name),
+				getGracePeriodOverrideTestPod("period-b-120", nodeName, 120, customClassB.Name),
+				getGracePeriodOverrideTestPod("period-b-5", nodeName, 5, customClassB.Name),
+				getGracePeriodOverrideTestPod("period-a-120", nodeName, 120, customClassA.Name),
+				getGracePeriodOverrideTestPod("period-a-5", nodeName, 5, customClassA.Name),
+				getGracePeriodOverrideTestPod("period-critical-120", nodeName, 120, scheduling.SystemNodeCritical),
+				getGracePeriodOverrideTestPod("period-critical-5", nodeName, 5, scheduling.SystemNodeCritical),
+			}
+
+			// Expected down steps
+			downSteps := [][]string{
+				{
+					"period-5", "period-120",
+				},
+				{
+					"period-5", "period-120",
+					"period-c-5", "period-c-120",
+				},
+				{
+					"period-5", "period-120",
+					"period-c-5", "period-c-120",
+					"period-b-5", "period-b-120",
+				},
+				{
+					"period-5", "period-120",
+					"period-c-5", "period-c-120",
+					"period-b-5", "period-b-120",
+					"period-a-5", "period-a-120",
+				},
+				{
+					"period-5", "period-120",
+					"period-c-5", "period-c-120",
+					"period-b-5", "period-b-120",
+					"period-a-5", "period-a-120",
+					"period-critical-5", "period-critical-120",
+				},
+			}
+
+			ginkgo.By("Creating batch pods")
+			f.PodClient().CreateBatch(pods)
+
+			list, err := f.PodClient().List(context.TODO(), metav1.ListOptions{
+				FieldSelector: nodeSelector,
+			})
+			framework.ExpectNoError(err)
+			framework.ExpectEqual(len(list.Items), len(pods), "the number of pods is not as expected")
+
+			ginkgo.By("Verifying batch pods are running")
+			for _, pod := range list.Items {
+				if podReady, err := testutils.PodRunningReady(&pod); err != nil || !podReady {
+					framework.Failf("Failed to start batch pod: %v", pod.Name)
+				}
+			}
+
+			ginkgo.By("Emitting shutdown signal")
+			err = emitSignalPrepareForShutdown(true)
+			framework.ExpectNoError(err)
+
+			ginkgo.By("Verifying that pods are shutdown")
+
+			for _, step := range downSteps {
+				gomega.Eventually(func() error {
+					list, err = f.PodClient().List(context.TODO(), metav1.ListOptions{
+						FieldSelector: nodeSelector,
+					})
+					if err != nil {
+						return err
+					}
+					framework.ExpectEqual(len(list.Items), len(pods), "the number of pods is not as expected")
+					for _, pod := range list.Items {
+						shouldShutdown := false
+						for _, podName := range step {
+							if podName == pod.Name {
+								shouldShutdown = true
+								break
+							}
+						}
+						if !shouldShutdown {
+							if pod.Status.Phase != v1.PodRunning {
+								framework.Logf("Expecting pod to be running, but it's not currently. Pod: %q, Pod Status Phase: %q, Pod Status Reason: %q", pod.Name, pod.Status.Phase, pod.Status.Reason)
+								return fmt.Errorf("pod should not be shutdown, phase: %s, reason: %s", pod.Status.Phase, pod.Status.Reason)
+							}
+						} else {
+							if pod.Status.Reason != podShutdownReason {
+								framework.Logf("Expecting pod to be shutdown, but it's not currently. Pod: %q, Pod Status Phase: %q, Pod Status Reason: %q", pod.Name, pod.Status.Phase, pod.Status.Reason)
+								for _, item := range list.Items {
+									framework.Logf("DEBUG %s, %s, %s", item.Name, item.Status.Phase, pod.Status.Reason)
+								}
+								return fmt.Errorf("pod should be shutdown, reason: %s", pod.Status.Reason)
+							}
+						}
+					}
+					return nil
+				}, podStatusUpdateTimeout, pollInterval).Should(gomega.BeNil())
+			}
+		})
+	})
 })
 
-func getGracePeriodOverrideTestPod(name string, node string, gracePeriod int64, critical bool) *v1.Pod {
+func getPriorityClass(name string, value int32) *schedulingv1.PriorityClass {
+	priority := &schedulingv1.PriorityClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "PriorityClass",
+			APIVersion: "scheduling.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Value: value,
+	}
+	return priority
+}
+func getGracePeriodOverrideTestPod(name string, node string, gracePeriod int64, priorityClassName string) *v1.Pod {
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -238,14 +425,14 @@ while true; do sleep 5; done
 			NodeName:                      node,
 		},
 	}
-	if critical {
+	if priorityClassName == scheduling.SystemNodeCritical {
 		pod.ObjectMeta.Annotations = map[string]string{
 			kubelettypes.ConfigSourceAnnotationKey: kubelettypes.FileSource,
 		}
-		pod.Spec.PriorityClassName = scheduling.SystemNodeCritical
+		pod.Spec.PriorityClassName = priorityClassName
-
 		framework.ExpectEqual(kubelettypes.IsCriticalPod(pod), true, "pod should be a critical pod")
 	} else {
+		pod.Spec.PriorityClassName = priorityClassName
 		framework.ExpectEqual(kubelettypes.IsCriticalPod(pod), false, "pod should not be a critical pod")
 	}
 	return pod