Merge pull request #47356 from dashpole/master_critical_pods

Automatic merge from submit-queue (batch tested with PRs 47669, 40284, 47356, 47458, 47701)

Mark Static pods on the Master as critical

fixes #47277.

A known issue with static pods is that they do not interact well with evictions.  If a static pod is evicted or oom killed, then it will never be recreated.  To mitigate this, we do not evict static pods that are critical.  In addition, non-critical pods are candidates for preemption if a critical pod is scheduled to the node.  If there are not enough allocatable resources on the node, this causes the static pod to be preempted.

This PR marks all static pods in the kube-system namspace as critical.

cc @vishh @dchen1107

cluster/addons/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: etcd-empty-dir-cleanup
   namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
   labels:
     k8s-app: etcd-empty-dir-cleanup
 spec:

cluster/saltbase/salt/etcd/etcd.manifest

@@ -29,7 +29,10 @@
 "kind": "Pod",
 "metadata": {
   "name":"etcd-server{{ suffix }}",
-  "namespace": "kube-system"
+  "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  }
 },
 "spec":{
 "hostNetwork": true,

cluster/saltbase/salt/kube-addons/kube-addon-manager.yaml

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: kube-addon-manager
   namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
   labels:
     component: kube-addon-manager
 spec:

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -212,6 +212,9 @@
 "metadata": {
   "name":"kube-apiserver",
   "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  },
   "labels": {
     "tier": "control-plane",
     "component": "kube-apiserver"

cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest

@@ -94,6 +94,9 @@
 "metadata": {
   "name":"kube-controller-manager",
   "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  },
   "labels": {
     "tier": "control-plane",
     "component": "kube-controller-manager"

cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest

@@ -29,6 +29,9 @@
 "metadata": {
   "name":"kube-scheduler",
   "namespace": "kube-system",
+  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": ""
+  },
   "labels": {
     "tier": "control-plane",
     "component": "kube-scheduler"

cluster/saltbase/salt/l7-gcp/glbc.manifest

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: l7-lb-controller-v0.9.5
   namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
   labels:
     k8s-app: glbc
     version: v0.9.5

cluster/saltbase/salt/rescheduler/rescheduler.manifest

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: rescheduler-v0.3.0
   namespace: kube-system
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
   labels:
     k8s-app: rescheduler
     version: v0.3.0

cmd/kubeadm/app/master/BUILD

@@ -26,6 +26,7 @@ go_library(
         "//pkg/bootstrap/api:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
+        "//pkg/kubelet/types:go_default_library",
         "//pkg/util/version:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",

cmd/kubeadm/app/master/manifests.go

@@ -38,6 +38,7 @@ import (
 	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
@@ -303,6 +304,7 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        container.Name,
 			Namespace:   "kube-system",
+			Annotations: map[string]string{kubetypes.CriticalPodAnnotationKey: ""},
 			Labels:      map[string]string{"component": container.Name, "tier": "control-plane"},
 		},
 		Spec: api.PodSpec{