Merge pull request #58835 from ravisantoshgudimetla/critical-pod-with-priority

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Critical pod priorityClass addition

**What this PR does / why we need it**:
@bsalamat - Apologies for the delay. This PR is to ensure that all pods with priorityClassName `system-node-critical` and `system-cluster-critical` will be critical pods while preserving backwards compatibility.

**Special notes for your reviewer**:

- Moved some constants and other data structures to scheduler/api/types.go where other constants are present.
- An automatic assignment of critical priorities to pods based on critical pod annotation for backwards compatibility including some unit tests. 
xref:  https://github.com/kubernetes/kubernetes/issues/57471 

**Release note**:

```release-note
Critical pods to use priorityClasses.
```

plugin/pkg/admission/priority/BUILD

@@ -16,6 +16,7 @@ go_test(
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/scheduler/api:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
@@ -35,6 +36,8 @@ go_library(
         "//pkg/client/listers/scheduling/internalversion:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubeapiserver/admission:go_default_library",
+        "//pkg/kubelet/types:go_default_library",
+        "//pkg/scheduler/api:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",

plugin/pkg/admission/priority/admission.go

@@ -31,26 +31,15 @@ import (
 	schedulinglisters "k8s.io/kubernetes/pkg/client/listers/scheduling/internalversion"
 	"k8s.io/kubernetes/pkg/features"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
+	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 )
 
 const (
 	// PluginName indicates name of admission plugin.
 	PluginName = "Priority"
-
-	// HighestUserDefinablePriority is the highest priority for user defined priority classes. Priority values larger than 1 billion are reserved for Kubernetes system use.
-	HighestUserDefinablePriority = 1000000000
-	// SystemCriticalPriority is the beginning of the range of priority values for critical system components.
-	SystemCriticalPriority = 2 * HighestUserDefinablePriority
 )
 
-// SystemPriorityClasses defines special priority classes which are used by system critical pods that should not be preempted by workload pods.
-// NOTE: In order to avoid conflict of names with user-defined priority classes, all the names must
-// start with scheduling.SystemPriorityClassPrefix which is by default "system-".
-var SystemPriorityClasses = map[string]int32{
-	"system-cluster-critical": SystemCriticalPriority,
-	"system-node-critical":    SystemCriticalPriority + 1000,
-}
-
 // Register registers a plugin
 func Register(plugins *admission.Plugins) {
 	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
@@ -166,6 +155,13 @@ func (p *PriorityPlugin) admitPod(a admission.Attributes) error {
 	}
 	if utilfeature.DefaultFeatureGate.Enabled(features.PodPriority) {
 		var priority int32
+		// TODO: @ravig - This is for backwards compatibility to ensure that critical pods with annotations just work fine.
+		// Remove when no longer needed.
+		if len(pod.Spec.PriorityClassName) == 0 &&
+			utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) &&
+			kubelettypes.IsCritical(a.GetNamespace(), pod.Annotations) {
+			pod.Spec.PriorityClassName = schedulerapi.SystemClusterCritical
+		}
 		if len(pod.Spec.PriorityClassName) == 0 {
 			var err error
 			priority, err = p.getDefaultPriority()
@@ -174,7 +170,7 @@ func (p *PriorityPlugin) admitPod(a admission.Attributes) error {
 			}
 		} else {
 			// First try to resolve by system priority classes.
-			priority, ok = SystemPriorityClasses[pod.Spec.PriorityClassName]
+			priority, ok = schedulerapi.SystemPriorityClasses[pod.Spec.PriorityClassName]
 			if !ok {
 				// Now that we didn't find any system priority, try resolving by user defined priority classes.
 				pc, err := p.lister.Get(pod.Spec.PriorityClassName)
@@ -202,10 +198,10 @@ func (p *PriorityPlugin) validatePriorityClass(a admission.Attributes) error {
 	if !ok {
 		return errors.NewBadRequest("resource was marked with kind PriorityClass but was unable to be converted")
 	}
-	if pc.Value > HighestUserDefinablePriority {
-		return admission.NewForbidden(a, fmt.Errorf("maximum allowed value of a user defined priority is %v", HighestUserDefinablePriority))
+	if pc.Value > schedulerapi.HighestUserDefinablePriority {
+		return admission.NewForbidden(a, fmt.Errorf("maximum allowed value of a user defined priority is %v", schedulerapi.HighestUserDefinablePriority))
 	}
-	if _, ok := SystemPriorityClasses[pc.Name]; ok {
+	if _, ok := schedulerapi.SystemPriorityClasses[pc.Name]; ok {
 		return admission.NewForbidden(a, fmt.Errorf("the name of the priority class is a reserved name for system use only: %v", pc.Name))
 	}
 	// If the new PriorityClass tries to be the default priority, make sure that no other priority class is marked as default.

plugin/pkg/admission/priority/admission_test.go

@@ -30,6 +30,7 @@ import (
 	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/features"
+	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 )
 
 func addPriorityClasses(ctrl *PriorityPlugin, priorityClasses []*scheduling.PriorityClass) {
@@ -82,7 +83,7 @@ func TestPriorityClassAdmission(t *testing.T) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "toohighclass",
 		},
-		Value:       HighestUserDefinablePriority + 1,
+		Value:       schedulerapi.HighestUserDefinablePriority + 1,
 		Description: "Just a test priority class",
 	}
 
@@ -91,9 +92,9 @@ func TestPriorityClassAdmission(t *testing.T) {
 			Kind: "PriorityClass",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "system-cluster-critical",
+			Name: schedulerapi.SystemClusterCritical,
 		},
-		Value:       HighestUserDefinablePriority + 1,
+		Value:       schedulerapi.HighestUserDefinablePriority + 1,
 		Description: "Name conflicts with system priority class names",
 	}
 
@@ -321,7 +322,7 @@ func TestPodAdmission(t *testing.T) {
 						Name: containerName,
 					},
 				},
-				PriorityClassName: "system-cluster-critical",
+				PriorityClassName: schedulerapi.SystemClusterCritical,
 			},
 		},
 		// pod[5]: mirror Pod with a system priority class name
@@ -357,9 +358,27 @@ func TestPodAdmission(t *testing.T) {
 				Priority:          &intPriority,
 			},
 		},
+		// pod[7]: Pod with a critical priority annotation. This needs to be automatically assigned
+		// system-cluster-critical
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        "pod-w-system-priority",
+				Namespace:   "kube-system",
+				Annotations: map[string]string{"scheduler.alpha.kubernetes.io/critical-pod": ""},
+			},
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name: containerName,
+					},
+				},
+			},
+		},
 	}
 	// Enable PodPriority feature gate.
 	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.PodPriority))
+	// Enable ExperimentalCriticalPodAnnotation feature gate.
+	utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("%s=true", features.ExperimentalCriticalPodAnnotation))
 	tests := []struct {
 		name            string
 		existingClasses []*scheduling.PriorityClass
@@ -402,7 +421,7 @@ func TestPodAdmission(t *testing.T) {
 			"pod with a system priority class",
 			[]*scheduling.PriorityClass{},
 			*pods[4],
-			SystemCriticalPriority,
+			schedulerapi.SystemCriticalPriority,
 			false,
 		},
 		{
@@ -423,7 +442,7 @@ func TestPodAdmission(t *testing.T) {
 			"mirror pod with system priority class",
 			[]*scheduling.PriorityClass{},
 			*pods[5],
-			SystemCriticalPriority,
+			schedulerapi.SystemCriticalPriority,
 			false,
 		},
 		{
@@ -433,6 +452,13 @@ func TestPodAdmission(t *testing.T) {
 			0,
 			true,
 		},
+		{
+			"pod with critical pod annotation",
+			[]*scheduling.PriorityClass{},
+			*pods[7],
+			schedulerapi.SystemCriticalPriority,
+			false,
+		},
 	}
 
 	for _, test := range tests {