Merge pull request #80277 from draveness/feature/revert-cleanup-critical-pod

Revert "feat: cleanup pod critical pod annotations feature"
2026-01-06 07:57:35 +00:00 · 2019-07-18 19:31:37 -07:00
parent 1593fad116 d83526d253
commit 49f6510d9a
56 changed files with 309 additions and 104 deletions
--- a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml
+++ b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml
@@ -17,6 +17,8 @@ spec:
    metadata:
      labels:
        k8s-app: calico-node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      nodeSelector:
--- a/cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml
+++ b/cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml
@@ -16,6 +16,8 @@ spec:
    metadata:
      labels:
        k8s-app: calico-node-autoscaler
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      containers:
--- a/cluster/addons/calico-policy-controller/typha-deployment.yaml
+++ b/cluster/addons/calico-policy-controller/typha-deployment.yaml
@@ -16,6 +16,8 @@ spec:
    metadata:
      labels:
        k8s-app: calico-typha
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      tolerations:
--- a/cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml
+++ b/cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml
@@ -16,6 +16,8 @@ spec:
    metadata:
      labels:
        k8s-app: calico-typha-autoscaler
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      securityContext:
--- a/cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml
+++ b/cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml
@@ -16,6 +16,8 @@ spec:
    metadata:
      labels:
        k8s-app: calico-typha-autoscaler
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      containers:
--- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml
+++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml
@@ -51,6 +51,7 @@ spec:
        k8s-app: heapster
        version: v1.6.0-beta.1
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml
+++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml
@@ -51,6 +51,7 @@ spec:
        k8s-app: heapster
        version: v1.6.0-beta.1
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml
+++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml
@@ -51,6 +51,7 @@ spec:
        k8s-app: heapster
        version: v1.6.0-beta.1
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml
+++ b/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml
@@ -19,6 +19,7 @@ spec:
        k8s-app: influxGrafana
        version: v4
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml
+++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml
@@ -39,6 +39,7 @@ spec:
        k8s-app: heapster
        version: v1.6.0-beta.1
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml
+++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml
@@ -39,6 +39,7 @@ spec:
        k8s-app: heapster
        version: v1.6.0-beta.1
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/dashboard/dashboard-controller.yaml
+++ b/cluster/addons/dashboard/dashboard-controller.yaml
@@ -24,6 +24,7 @@ spec:
      labels:
        k8s-app: kubernetes-dashboard
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml
+++ b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml
@@ -14,6 +14,8 @@ spec:
    metadata:
      labels:
        k8s-app: nvidia-gpu-device-plugin
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      affinity:
--- a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml
+++ b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml
@@ -76,6 +76,7 @@ spec:
      labels:
        k8s-app: kube-dns-autoscaler
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/dns/kube-dns/kube-dns.yaml.base
+++ b/cluster/addons/dns/kube-dns/kube-dns.yaml.base
@@ -82,6 +82,7 @@ spec:
      labels:
        k8s-app: kube-dns
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
        prometheus.io/port: "10054"
        prometheus.io/scrape: "true"
--- a/cluster/addons/dns/kube-dns/kube-dns.yaml.in
+++ b/cluster/addons/dns/kube-dns/kube-dns.yaml.in
@@ -82,6 +82,7 @@ spec:
      labels:
        k8s-app: kube-dns
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
        prometheus.io/port: "10054"
        prometheus.io/scrape: "true"
--- a/cluster/addons/dns/kube-dns/kube-dns.yaml.sed
+++ b/cluster/addons/dns/kube-dns/kube-dns.yaml.sed
@@ -82,6 +82,7 @@ spec:
      labels:
        k8s-app: kube-dns
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
        prometheus.io/port: "10054"
        prometheus.io/scrape: "true"
--- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml
+++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml
@@ -65,6 +65,7 @@ spec:
      # supports critical pod annotation based priority scheme.
      # Note that this does not guarantee admission on the nodes (#40573).
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-node-critical
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml
@@ -21,6 +21,11 @@ spec:
        k8s-app: fluentd-gcp
        kubernetes.io/cluster-service: "true"
        version: {{ fluentd_gcp_yaml_version }}
+      # This annotation ensures that fluentd does not get evicted if the node
+      # supports critical pod annotation based priority scheme.
+      # Note that this does not guarantee admission on the nodes (#40573).
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      serviceAccountName: fluentd-gcp
--- a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml
+++ b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml
@@ -24,6 +24,8 @@ spec:
    metadata:
      labels:
        k8s-app: ip-masq-agent
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      serviceAccountName: ip-masq-agent
--- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml
+++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml
@@ -21,6 +21,8 @@ spec:
    metadata:
      labels:
        k8s-app: kube-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      hostNetwork: true
--- a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml
+++ b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml
@@ -31,6 +31,11 @@ spec:
        k8s-app: metadata-proxy
        kubernetes.io/cluster-service: "true"
        version: v0.1
+      # This annotation ensures that the proxy does not get evicted if the node
+      # supports critical pod annotation based priority scheme.
+      # Note that this does not guarantee admission on the nodes (#40573).
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      serviceAccountName: metadata-proxy
--- a/cluster/addons/metrics-server/metrics-server-deployment.yaml
+++ b/cluster/addons/metrics-server/metrics-server-deployment.yaml
@@ -42,6 +42,7 @@ spec:
        k8s-app: metrics-server
        version: v0.3.3
      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
--- a/cluster/addons/prometheus/alertmanager-deployment.yaml
+++ b/cluster/addons/prometheus/alertmanager-deployment.yaml
@@ -19,6 +19,8 @@ spec:
      labels:
        k8s-app: alertmanager
        version: v0.14.0
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      containers:
--- a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml
+++ b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml
@@ -19,6 +19,8 @@ spec:
      labels:
        k8s-app: kube-state-metrics
        version: v1.3.0
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: kube-state-metrics
--- a/cluster/addons/prometheus/node-exporter-ds.yml
+++ b/cluster/addons/prometheus/node-exporter-ds.yml
@@ -20,6 +20,8 @@ spec:
      labels:
        k8s-app: node-exporter
        version: v0.15.2
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      containers:
--- a/cluster/addons/prometheus/prometheus-statefulset.yaml
+++ b/cluster/addons/prometheus/prometheus-statefulset.yaml
@@ -21,6 +21,8 @@ spec:
    metadata:
      labels:
        k8s-app: prometheus
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: prometheus
--- a/cluster/gce/addons/node-termination-handler/daemonset.yaml
+++ b/cluster/gce/addons/node-termination-handler/daemonset.yaml
@@ -17,6 +17,8 @@ spec:
    metadata:
      labels:
        k8s-app: node-termination-handler
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      priorityClassName: system-node-critical
      # Necessary to reboot node
--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@@ -250,14 +250,10 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
 fi

 # Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-}"
+FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"

 if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
-    if [[ -z "${FEATURE_GATES:-}" ]]; then
-        FEATURE_GATES="DevicePlugins=true"
-    else
-        FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
-    fi
+    FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
    if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then
        NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}"
    fi
--- a/cluster/gce/config-test.sh
+++ b/cluster/gce/config-test.sh
@@ -139,7 +139,7 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
 fi

 # Optional: set feature gates
-FEATURE_GATES="${KUBE_FEATURE_GATES:-}"
+FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"

 TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}

@@ -283,11 +283,7 @@ if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
 fi

 if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
-    if [[ -z "${FEATURE_GATES:-}" ]]; then
-        FEATURE_GATES="DevicePlugins=true"
-    else
-        FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
-    fi
+    FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true"
    if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then
        NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}"
    fi
--- a/cluster/gce/manifests/etcd-empty-dir-cleanup.yaml
+++ b/cluster/gce/manifests/etcd-empty-dir-cleanup.yaml
@@ -4,6 +4,7 @@ metadata:
  name: etcd-empty-dir-cleanup
  namespace: kube-system
  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
  labels:
    k8s-app: etcd-empty-dir-cleanup
--- a/cluster/gce/manifests/etcd.manifest
+++ b/cluster/gce/manifests/etcd.manifest
@@ -5,11 +5,11 @@
  "name":"etcd-server{{ suffix }}",
  "namespace": "kube-system",
  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": "",
    "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
  }
 },
 "spec":{
-"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
    {
--- a/cluster/gce/manifests/glbc.manifest
+++ b/cluster/gce/manifests/glbc.manifest
@@ -4,13 +4,13 @@ metadata:
  name: l7-lb-controller-v1.2.3
  namespace: kube-system
  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
  labels:
    k8s-app: gcp-lb-controller
    version: v1.2.3
    kubernetes.io/name: "GLBC"
 spec:
-  priorityClassName: system-node-critical
  terminationGracePeriodSeconds: 600
  hostNetwork: true
  containers:
--- a/cluster/gce/manifests/kube-addon-manager.yaml
+++ b/cluster/gce/manifests/kube-addon-manager.yaml
@@ -4,11 +4,11 @@ metadata:
  name: kube-addon-manager
  namespace: kube-system
  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
  labels:
    component: kube-addon-manager
 spec:
-  priorityClassName: system-node-critical
  hostNetwork: true
  containers:
  - name: kube-addon-manager
--- a/cluster/gce/manifests/kube-apiserver.manifest
+++ b/cluster/gce/manifests/kube-apiserver.manifest
@@ -5,6 +5,7 @@
  "name":"kube-apiserver",
  "namespace": "kube-system",
  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": "",
    "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
  },
  "labels": {
@@ -13,7 +14,6 @@
  }
 },
 "spec":{
-"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
    {
--- a/cluster/gce/manifests/kube-controller-manager.manifest
+++ b/cluster/gce/manifests/kube-controller-manager.manifest
@@ -5,6 +5,7 @@
  "name":"kube-controller-manager",
  "namespace": "kube-system",
  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": "",
    "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
  },
  "labels": {
@@ -13,7 +14,6 @@
  }
 },
 "spec":{
-"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
    {
--- a/cluster/gce/manifests/kube-proxy.manifest
+++ b/cluster/gce/manifests/kube-proxy.manifest
@@ -3,6 +3,12 @@ kind: Pod
 metadata:
  name: kube-proxy
  namespace: kube-system
+  # This annotation ensures that kube-proxy does not get evicted if the node
+  # supports critical pod annotation based priority scheme.
+  # Note that kube-proxy runs as a static pod so this annotation does NOT have
+  # any effect on default scheduler which scheduling kube-proxy.
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
  labels:
    tier: node
    component: kube-proxy
--- a/cluster/gce/manifests/kube-scheduler.manifest
+++ b/cluster/gce/manifests/kube-scheduler.manifest
@@ -5,6 +5,7 @@
  "name":"kube-scheduler",
  "namespace": "kube-system",
  "annotations": {
+    "scheduler.alpha.kubernetes.io/critical-pod": "",
    "seccomp.security.alpha.kubernetes.io/pod": "docker/default"
  },
  "labels": {
@@ -13,7 +14,6 @@
  }
 },
 "spec":{
-"priorityClass": "system-node-critical",
 "hostNetwork": true,
 "containers":[
    {
--- a/cluster/gce/windows/k8s-node-setup.psm1
+++ b/cluster/gce/windows/k8s-node-setup.psm1
@@ -973,6 +973,7 @@ function Start-WorkerServices {
  #   kube-proxy --master=https://35.239.84.171
  #   --kubeconfig=/var/lib/kube-proxy/kubeconfig --cluster-cidr=10.64.0.0/14
  #   --oom-score-adj=-998 --v=2
+  #   --feature-gates=ExperimentalCriticalPodAnnotation=true
  #   --iptables-sync-period=1m --iptables-min-sync-period=10s
  #   --ipvs-sync-period=1m --ipvs-min-sync-period=10s
  # And also with various volumeMounts and "securityContext: privileged: true".