From b6d41ee5cc176d60ba71992ef82fa9b1c8be9af7 Mon Sep 17 00:00:00 2001 From: draveness Date: Sat, 29 Jun 2019 09:58:39 +0800 Subject: [PATCH] feat: cleanup pod critical pod annotations feature --- .../calico-node-daemonset.yaml | 2 - ...o-node-vertical-autoscaler-deployment.yaml | 2 - .../typha-deployment.yaml | 2 - ...ypha-horizontal-autoscaler-deployment.yaml | 2 - .../typha-vertical-autoscaler-deployment.yaml | 2 - .../google/heapster-controller.yaml | 1 - .../heapster-controller-combined.yaml | 1 - .../influxdb/heapster-controller.yaml | 1 - .../influxdb/influxdb-grafana-controller.yaml | 1 - .../stackdriver/heapster-controller.yaml | 1 - .../standalone/heapster-controller.yaml | 1 - .../dashboard/dashboard-controller.yaml | 1 - .../device-plugins/nvidia-gpu/daemonset.yaml | 2 - .../dns-horizontal-autoscaler.yaml | 1 - .../addons/dns/kube-dns/kube-dns.yaml.base | 1 - cluster/addons/dns/kube-dns/kube-dns.yaml.in | 1 - cluster/addons/dns/kube-dns/kube-dns.yaml.sed | 1 - .../fluentd-elasticsearch/fluentd-es-ds.yaml | 1 - .../addons/fluentd-gcp/fluentd-gcp-ds.yaml | 5 - .../addons/ip-masq-agent/ip-masq-agent.yaml | 2 - cluster/addons/kube-proxy/kube-proxy-ds.yaml | 2 - .../metadata-proxy/gce/metadata-proxy.yaml | 5 - .../metrics-server-deployment.yaml | 1 - .../prometheus/alertmanager-deployment.yaml | 2 - .../kube-state-metrics-deployment.yaml | 2 - .../addons/prometheus/node-exporter-ds.yml | 2 - .../prometheus/prometheus-statefulset.yaml | 2 - .../node-termination-handler/daemonset.yaml | 2 - cluster/gce/config-default.sh | 8 +- cluster/gce/config-test.sh | 8 +- .../gce/manifests/etcd-empty-dir-cleanup.yaml | 1 - cluster/gce/manifests/etcd.manifest | 2 +- cluster/gce/manifests/glbc.manifest | 2 +- cluster/gce/manifests/kube-addon-manager.yaml | 2 +- cluster/gce/manifests/kube-apiserver.manifest | 2 +- .../kube-controller-manager.manifest | 2 +- cluster/gce/manifests/kube-proxy.manifest | 6 - cluster/gce/manifests/kube-scheduler.manifest | 2 +- cluster/gce/windows/k8s-node-setup.psm1 | 1 - pkg/controller/daemon/BUILD | 2 +- .../daemon/daemon_controller_test.go | 50 ++++--- pkg/features/kube_features.go | 126 ++++++++---------- pkg/kubelet/eviction/BUILD | 1 + pkg/kubelet/eviction/eviction_manager_test.go | 22 +-- pkg/kubelet/preemption/BUILD | 4 - pkg/kubelet/preemption/preemption_test.go | 47 +++---- pkg/kubelet/types/BUILD | 6 - pkg/kubelet/types/pod_update.go | 30 +---- pkg/kubelet/types/pod_update_test.go | 67 ---------- plugin/pkg/admission/priority/BUILD | 1 - plugin/pkg/admission/priority/admission.go | 8 -- .../pkg/admission/priority/admission_test.go | 54 ++------ .../sample-device-plugin.yaml | 1 - test/e2e_node/BUILD | 1 + test/e2e_node/critical_pod_test.go | 19 +-- .../kubemark/resources/kube_dns_template.yaml | 3 +- 56 files changed, 161 insertions(+), 366 deletions(-) diff --git a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml index 491b2172e6e..5d6ab990aa9 100644 --- a/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml +++ b/cluster/addons/calico-policy-controller/calico-node-daemonset.yaml @@ -17,8 +17,6 @@ spec: metadata: labels: k8s-app: calico-node - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical nodeSelector: diff --git a/cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml b/cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml index fc6fab3dd0f..96d5aeb9e79 100644 --- a/cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml +++ b/cluster/addons/calico-policy-controller/calico-node-vertical-autoscaler-deployment.yaml @@ -16,8 +16,6 @@ spec: metadata: labels: k8s-app: calico-node-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: diff --git a/cluster/addons/calico-policy-controller/typha-deployment.yaml b/cluster/addons/calico-policy-controller/typha-deployment.yaml index f8f143f57ed..a521df42121 100644 --- a/cluster/addons/calico-policy-controller/typha-deployment.yaml +++ b/cluster/addons/calico-policy-controller/typha-deployment.yaml @@ -16,8 +16,6 @@ spec: metadata: labels: k8s-app: calico-typha - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical tolerations: diff --git a/cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml b/cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml index 82c5a935db9..b9ae5bf1bbf 100644 --- a/cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml +++ b/cluster/addons/calico-policy-controller/typha-horizontal-autoscaler-deployment.yaml @@ -16,8 +16,6 @@ spec: metadata: labels: k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical securityContext: diff --git a/cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml b/cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml index 0e4f22355a1..3e66cfe565a 100644 --- a/cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml +++ b/cluster/addons/calico-policy-controller/typha-vertical-autoscaler-deployment.yaml @@ -16,8 +16,6 @@ spec: metadata: labels: k8s-app: calico-typha-autoscaler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index 4ff89886741..533e7e2a806 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -51,7 +51,6 @@ spec: k8s-app: heapster version: v1.6.0-beta.1 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index 9f359f41418..1d1cd9a31d4 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -51,7 +51,6 @@ spec: k8s-app: heapster version: v1.6.0-beta.1 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index c5b78d12680..26820387bef 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -51,7 +51,6 @@ spec: k8s-app: heapster version: v1.6.0-beta.1 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml index a4f95cb6607..769683b9c30 100644 --- a/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml @@ -19,7 +19,6 @@ spec: k8s-app: influxGrafana version: v4 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index 6bb898baf91..fb3a82499ca 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -39,7 +39,6 @@ spec: k8s-app: heapster version: v1.6.0-beta.1 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index 414bb1af00c..b69083c07fe 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -39,7 +39,6 @@ spec: k8s-app: heapster version: v1.6.0-beta.1 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/dashboard/dashboard-controller.yaml b/cluster/addons/dashboard/dashboard-controller.yaml index c4dee396bba..830fa7696e6 100644 --- a/cluster/addons/dashboard/dashboard-controller.yaml +++ b/cluster/addons/dashboard/dashboard-controller.yaml @@ -24,7 +24,6 @@ spec: labels: k8s-app: kubernetes-dashboard annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml index 75d0ea1df67..708184acd5a 100644 --- a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml +++ b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml @@ -14,8 +14,6 @@ spec: metadata: labels: k8s-app: nvidia-gpu-device-plugin - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical affinity: diff --git a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml index 49fd35e76a7..3e8ef1da643 100644 --- a/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml +++ b/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml @@ -76,7 +76,6 @@ spec: labels: k8s-app: kube-dns-autoscaler annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/dns/kube-dns/kube-dns.yaml.base b/cluster/addons/dns/kube-dns/kube-dns.yaml.base index cd8abb1a82f..6a827ce89a0 100644 --- a/cluster/addons/dns/kube-dns/kube-dns.yaml.base +++ b/cluster/addons/dns/kube-dns/kube-dns.yaml.base @@ -82,7 +82,6 @@ spec: labels: k8s-app: kube-dns annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' prometheus.io/port: "10054" prometheus.io/scrape: "true" diff --git a/cluster/addons/dns/kube-dns/kube-dns.yaml.in b/cluster/addons/dns/kube-dns/kube-dns.yaml.in index f4160658aaa..b677a232d3c 100644 --- a/cluster/addons/dns/kube-dns/kube-dns.yaml.in +++ b/cluster/addons/dns/kube-dns/kube-dns.yaml.in @@ -82,7 +82,6 @@ spec: labels: k8s-app: kube-dns annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' prometheus.io/port: "10054" prometheus.io/scrape: "true" diff --git a/cluster/addons/dns/kube-dns/kube-dns.yaml.sed b/cluster/addons/dns/kube-dns/kube-dns.yaml.sed index 2e397d29175..ea5e6bae54a 100644 --- a/cluster/addons/dns/kube-dns/kube-dns.yaml.sed +++ b/cluster/addons/dns/kube-dns/kube-dns.yaml.sed @@ -82,7 +82,6 @@ spec: labels: k8s-app: kube-dns annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' prometheus.io/port: "10054" prometheus.io/scrape: "true" diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index 047133466e7..5a379ac20d5 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -65,7 +65,6 @@ spec: # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-node-critical diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 49c97ad35a0..5921f9ab969 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -21,11 +21,6 @@ spec: k8s-app: fluentd-gcp kubernetes.io/cluster-service: "true" version: {{ fluentd_gcp_yaml_version }} - # This annotation ensures that fluentd does not get evicted if the node - # supports critical pod annotation based priority scheme. - # Note that this does not guarantee admission on the nodes (#40573). - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical serviceAccountName: fluentd-gcp diff --git a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml index 18dc76ad284..0436c6ba1d9 100644 --- a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml +++ b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml @@ -24,8 +24,6 @@ spec: metadata: labels: k8s-app: ip-masq-agent - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical serviceAccountName: ip-masq-agent diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml index aaa9641d13f..ea81f7c5dc5 100644 --- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -21,8 +21,6 @@ spec: metadata: labels: k8s-app: kube-proxy - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical hostNetwork: true diff --git a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml index f5b9ba720c7..7267a393dc5 100644 --- a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml +++ b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml @@ -31,11 +31,6 @@ spec: k8s-app: metadata-proxy kubernetes.io/cluster-service: "true" version: v0.1 - # This annotation ensures that the proxy does not get evicted if the node - # supports critical pod annotation based priority scheme. - # Note that this does not guarantee admission on the nodes (#40573). - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical serviceAccountName: metadata-proxy diff --git a/cluster/addons/metrics-server/metrics-server-deployment.yaml b/cluster/addons/metrics-server/metrics-server-deployment.yaml index 2306dc98f9f..6da5ca70459 100644 --- a/cluster/addons/metrics-server/metrics-server-deployment.yaml +++ b/cluster/addons/metrics-server/metrics-server-deployment.yaml @@ -42,7 +42,6 @@ spec: k8s-app: metrics-server version: v0.3.3 annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: priorityClassName: system-cluster-critical diff --git a/cluster/addons/prometheus/alertmanager-deployment.yaml b/cluster/addons/prometheus/alertmanager-deployment.yaml index 85cecd3dd21..38ec99d3055 100644 --- a/cluster/addons/prometheus/alertmanager-deployment.yaml +++ b/cluster/addons/prometheus/alertmanager-deployment.yaml @@ -19,8 +19,6 @@ spec: labels: k8s-app: alertmanager version: v0.14.0 - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical containers: diff --git a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml index 5d81e8e002d..14ea9802b40 100644 --- a/cluster/addons/prometheus/kube-state-metrics-deployment.yaml +++ b/cluster/addons/prometheus/kube-state-metrics-deployment.yaml @@ -19,8 +19,6 @@ spec: labels: k8s-app: kube-state-metrics version: v1.3.0 - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical serviceAccountName: kube-state-metrics diff --git a/cluster/addons/prometheus/node-exporter-ds.yml b/cluster/addons/prometheus/node-exporter-ds.yml index f5f88f76d41..bc1766a1d38 100644 --- a/cluster/addons/prometheus/node-exporter-ds.yml +++ b/cluster/addons/prometheus/node-exporter-ds.yml @@ -20,8 +20,6 @@ spec: labels: k8s-app: node-exporter version: v0.15.2 - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical containers: diff --git a/cluster/addons/prometheus/prometheus-statefulset.yaml b/cluster/addons/prometheus/prometheus-statefulset.yaml index 3fee8e4d6a1..01a12d0a969 100644 --- a/cluster/addons/prometheus/prometheus-statefulset.yaml +++ b/cluster/addons/prometheus/prometheus-statefulset.yaml @@ -21,8 +21,6 @@ spec: metadata: labels: k8s-app: prometheus - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-cluster-critical serviceAccountName: prometheus diff --git a/cluster/gce/addons/node-termination-handler/daemonset.yaml b/cluster/gce/addons/node-termination-handler/daemonset.yaml index 789a1ba41ae..c3b42149c87 100644 --- a/cluster/gce/addons/node-termination-handler/daemonset.yaml +++ b/cluster/gce/addons/node-termination-handler/daemonset.yaml @@ -17,8 +17,6 @@ spec: metadata: labels: k8s-app: node-termination-handler - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical # Necessary to reboot node diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index fe68c3816ab..e9e19297f08 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -250,10 +250,14 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then fi # Optional: set feature gates -FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" +FEATURE_GATES="${KUBE_FEATURE_GATES:-}" if [[ ! -z "${NODE_ACCELERATORS}" ]]; then - FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" + if [[ -z "${FEATURE_GATES:-}" ]]; then + FEATURE_GATES="DevicePlugins=true" + else + FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" + fi if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}" fi diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 7950767c069..9bcb381d0d1 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -139,7 +139,7 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then fi # Optional: set feature gates -FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" +FEATURE_GATES="${KUBE_FEATURE_GATES:-}" TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100} @@ -283,7 +283,11 @@ if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then fi if [[ ! -z "${NODE_ACCELERATORS}" ]]; then - FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" + if [[ -z "${FEATURE_GATES:-}" ]]; then + FEATURE_GATES="DevicePlugins=true" + else + FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" + fi if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}" fi diff --git a/cluster/gce/manifests/etcd-empty-dir-cleanup.yaml b/cluster/gce/manifests/etcd-empty-dir-cleanup.yaml index 51750a06304..34ae25baf05 100644 --- a/cluster/gce/manifests/etcd-empty-dir-cleanup.yaml +++ b/cluster/gce/manifests/etcd-empty-dir-cleanup.yaml @@ -4,7 +4,6 @@ metadata: name: etcd-empty-dir-cleanup namespace: kube-system annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' labels: k8s-app: etcd-empty-dir-cleanup diff --git a/cluster/gce/manifests/etcd.manifest b/cluster/gce/manifests/etcd.manifest index fa54fbc0725..377cc60d5f5 100644 --- a/cluster/gce/manifests/etcd.manifest +++ b/cluster/gce/manifests/etcd.manifest @@ -5,11 +5,11 @@ "name":"etcd-server{{ suffix }}", "namespace": "kube-system", "annotations": { - "scheduler.alpha.kubernetes.io/critical-pod": "", "seccomp.security.alpha.kubernetes.io/pod": "docker/default" } }, "spec":{ +"priorityClass": "system-node-critical", "hostNetwork": true, "containers":[ { diff --git a/cluster/gce/manifests/glbc.manifest b/cluster/gce/manifests/glbc.manifest index 319037d0ef4..13e1a0fc936 100644 --- a/cluster/gce/manifests/glbc.manifest +++ b/cluster/gce/manifests/glbc.manifest @@ -4,13 +4,13 @@ metadata: name: l7-lb-controller-v1.2.3 namespace: kube-system annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' labels: k8s-app: gcp-lb-controller version: v1.2.3 kubernetes.io/name: "GLBC" spec: + priorityClassName: system-node-critical terminationGracePeriodSeconds: 600 hostNetwork: true containers: diff --git a/cluster/gce/manifests/kube-addon-manager.yaml b/cluster/gce/manifests/kube-addon-manager.yaml index f5ac42e940e..f3a4f8b2c3c 100644 --- a/cluster/gce/manifests/kube-addon-manager.yaml +++ b/cluster/gce/manifests/kube-addon-manager.yaml @@ -4,11 +4,11 @@ metadata: name: kube-addon-manager namespace: kube-system annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' labels: component: kube-addon-manager spec: + priorityClassName: system-node-critical hostNetwork: true containers: - name: kube-addon-manager diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 2ae818a0fcd..fc418c47488 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -5,7 +5,6 @@ "name":"kube-apiserver", "namespace": "kube-system", "annotations": { - "scheduler.alpha.kubernetes.io/critical-pod": "", "seccomp.security.alpha.kubernetes.io/pod": "docker/default" }, "labels": { @@ -14,6 +13,7 @@ } }, "spec":{ +"priorityClass": "system-node-critical", "hostNetwork": true, "containers":[ { diff --git a/cluster/gce/manifests/kube-controller-manager.manifest b/cluster/gce/manifests/kube-controller-manager.manifest index be19cab1ce1..8d8839c4266 100644 --- a/cluster/gce/manifests/kube-controller-manager.manifest +++ b/cluster/gce/manifests/kube-controller-manager.manifest @@ -5,7 +5,6 @@ "name":"kube-controller-manager", "namespace": "kube-system", "annotations": { - "scheduler.alpha.kubernetes.io/critical-pod": "", "seccomp.security.alpha.kubernetes.io/pod": "docker/default" }, "labels": { @@ -14,6 +13,7 @@ } }, "spec":{ +"priorityClass": "system-node-critical", "hostNetwork": true, "containers":[ { diff --git a/cluster/gce/manifests/kube-proxy.manifest b/cluster/gce/manifests/kube-proxy.manifest index 0dcc3c52ec5..7644461ad7f 100644 --- a/cluster/gce/manifests/kube-proxy.manifest +++ b/cluster/gce/manifests/kube-proxy.manifest @@ -3,12 +3,6 @@ kind: Pod metadata: name: kube-proxy namespace: kube-system - # This annotation ensures that kube-proxy does not get evicted if the node - # supports critical pod annotation based priority scheme. - # Note that kube-proxy runs as a static pod so this annotation does NOT have - # any effect on default scheduler which scheduling kube-proxy. - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' labels: tier: node component: kube-proxy diff --git a/cluster/gce/manifests/kube-scheduler.manifest b/cluster/gce/manifests/kube-scheduler.manifest index a0648f9c189..ab907f11daf 100644 --- a/cluster/gce/manifests/kube-scheduler.manifest +++ b/cluster/gce/manifests/kube-scheduler.manifest @@ -5,7 +5,6 @@ "name":"kube-scheduler", "namespace": "kube-system", "annotations": { - "scheduler.alpha.kubernetes.io/critical-pod": "", "seccomp.security.alpha.kubernetes.io/pod": "docker/default" }, "labels": { @@ -14,6 +13,7 @@ } }, "spec":{ +"priorityClass": "system-node-critical", "hostNetwork": true, "containers":[ { diff --git a/cluster/gce/windows/k8s-node-setup.psm1 b/cluster/gce/windows/k8s-node-setup.psm1 index f8c5ef9d553..483cca85d9b 100644 --- a/cluster/gce/windows/k8s-node-setup.psm1 +++ b/cluster/gce/windows/k8s-node-setup.psm1 @@ -973,7 +973,6 @@ function Start-WorkerServices { # kube-proxy --master=https://35.239.84.171 # --kubeconfig=/var/lib/kube-proxy/kubeconfig --cluster-cidr=10.64.0.0/14 # --oom-score-adj=-998 --v=2 - # --feature-gates=ExperimentalCriticalPodAnnotation=true # --iptables-sync-period=1m --iptables-min-sync-period=10s # --ipvs-sync-period=1m --ipvs-min-sync-period=10s # And also with various volumeMounts and "securityContext: privileged: true". diff --git a/pkg/controller/daemon/BUILD b/pkg/controller/daemon/BUILD index 8e3e2d55ee1..0b32d734c9e 100644 --- a/pkg/controller/daemon/BUILD +++ b/pkg/controller/daemon/BUILD @@ -66,9 +66,9 @@ go_test( "//pkg/api/legacyscheme:go_default_library", "//pkg/api/v1/pod:go_default_library", "//pkg/apis/core:go_default_library", + "//pkg/apis/scheduling:go_default_library", "//pkg/controller:go_default_library", "//pkg/features:go_default_library", - "//pkg/kubelet/types:go_default_library", "//pkg/scheduler/api:go_default_library", "//pkg/securitycontext:go_default_library", "//pkg/util/labels:go_default_library", diff --git a/pkg/controller/daemon/daemon_controller_test.go b/pkg/controller/daemon/daemon_controller_test.go index 95a77f086eb..09f5d1bbda9 100644 --- a/pkg/controller/daemon/daemon_controller_test.go +++ b/pkg/controller/daemon/daemon_controller_test.go @@ -46,9 +46,9 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" podutil "k8s.io/kubernetes/pkg/api/v1/pod" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/features" - kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" schedulerapi "k8s.io/kubernetes/pkg/scheduler/api" "k8s.io/kubernetes/pkg/securitycontext" labelsutil "k8s.io/kubernetes/pkg/util/labels" @@ -1815,6 +1815,34 @@ func TestTaintPressureNodeDaemonLaunchesPod(t *testing.T) { // When ScheduleDaemonSetPods is disabled, DaemonSet should launch a critical pod even when the node has insufficient free resource. func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ScheduleDaemonSetPods, false)() + for _, strategy := range updateStrategies() { + podSpec := resourcePodSpec("too-much-mem", "75M", "75m") + ds := newDaemonSet("critical") + ds.Spec.UpdateStrategy = *strategy + ds.Spec.Template.Spec = podSpec + + manager, podControl, _, err := newTestController(ds) + if err != nil { + t.Fatalf("error creating DaemonSets controller: %v", err) + } + node := newNode("too-much-mem", nil) + node.Status.Allocatable = allocatableResources("100M", "200m") + manager.nodeStore.Add(node) + manager.podStore.Add(&v1.Pod{ + Spec: podSpec, + }) + + manager.dsStore.Add(ds) + switch strategy.Type { + case apps.OnDeleteDaemonSetStrategyType: + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 2) + case apps.RollingUpdateDaemonSetStrategyType: + syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 3) + default: + t.Fatalf("unexpected UpdateStrategy %+v", strategy) + } + } + for _, strategy := range updateStrategies() { podSpec := resourcePodSpec("too-much-mem", "75M", "75m") ds := newDaemonSet("critical") @@ -1833,25 +1861,13 @@ func TestInsufficientCapacityNodeDaemonLaunchesCriticalPod(t *testing.T) { Spec: podSpec, }) - // Without enabling critical pod annotation feature gate, we shouldn't create critical pod - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)() manager.dsStore.Add(ds) - switch strategy.Type { - case apps.OnDeleteDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 2) - case apps.RollingUpdateDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0, 3) - default: - t.Fatalf("unexpected UpdateStrategy %+v", strategy) - } - // Enabling critical pod annotation feature gate should create critical pod - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() switch strategy.Type { case apps.OnDeleteDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 2) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) case apps.RollingUpdateDaemonSetStrategyType: - syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 3) + syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0, 0) default: t.Fatalf("unexpected UpdateStrategy %+v", strategy) } @@ -1880,7 +1896,6 @@ func TestPortConflictNodeDaemonDoesNotLaunchCriticalPod(t *testing.T) { Spec: podSpec, }) - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() ds := newDaemonSet("critical") ds.Spec.UpdateStrategy = *strategy ds.Spec.Template.Spec = podSpec @@ -1895,7 +1910,8 @@ func setDaemonSetCritical(ds *apps.DaemonSet) { if ds.Spec.Template.ObjectMeta.Annotations == nil { ds.Spec.Template.ObjectMeta.Annotations = make(map[string]string) } - ds.Spec.Template.ObjectMeta.Annotations[kubelettypes.CriticalPodAnnotationKey] = "" + podPriority := scheduling.SystemCriticalPriority + ds.Spec.Template.Spec.Priority = &podPriority } func TestNodeShouldRunDaemonPod(t *testing.T) { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 98d9d3dfb73..258d8445b76 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -48,15 +48,6 @@ const ( // SYS_TIME). This should only be enabled if user namespace remapping is enabled in the docker daemon. ExperimentalHostUserNamespaceDefaultingGate featuregate.Feature = "ExperimentalHostUserNamespaceDefaulting" - // owner: @vishh - // alpha: v1.5 - // - // DEPRECATED - This feature is deprecated by Pod Priority and Preemption as of Kubernetes 1.13. - // Ensures guaranteed scheduling of pods marked with a special pod annotation `scheduler.alpha.kubernetes.io/critical-pod` - // and also prevents them from being evicted from a node. - // Note: This feature is not supported for `BestEffort` pods. - ExperimentalCriticalPodAnnotation featuregate.Feature = "ExperimentalCriticalPodAnnotation" - // owner: @jiayingz // beta: v1.10 // @@ -472,65 +463,64 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS AppArmor: {Default: true, PreRelease: featuregate.Beta}, DynamicKubeletConfig: {Default: true, PreRelease: featuregate.Beta}, ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: featuregate.Beta}, - ExperimentalCriticalPodAnnotation: {Default: false, PreRelease: featuregate.Alpha}, - DevicePlugins: {Default: true, PreRelease: featuregate.Beta}, - TaintBasedEvictions: {Default: true, PreRelease: featuregate.Beta}, - RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta}, - RotateKubeletClientCertificate: {Default: true, PreRelease: featuregate.Beta}, - PersistentLocalVolumes: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17 - LocalStorageCapacityIsolation: {Default: true, PreRelease: featuregate.Beta}, - Sysctls: {Default: true, PreRelease: featuregate.Beta}, - DebugContainers: {Default: false, PreRelease: featuregate.Alpha}, - PodShareProcessNamespace: {Default: true, PreRelease: featuregate.Beta}, - PodPriority: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18 - TaintNodesByCondition: {Default: true, PreRelease: featuregate.Beta}, - QOSReserved: {Default: false, PreRelease: featuregate.Alpha}, - ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, - ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, - ExpandCSIVolumes: {Default: false, PreRelease: featuregate.Alpha}, - AttachVolumeLimit: {Default: true, PreRelease: featuregate.Beta}, - CPUManager: {Default: true, PreRelease: featuregate.Beta}, - CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, - TopologyManager: {Default: false, PreRelease: featuregate.Alpha}, - ServiceNodeExclusion: {Default: false, PreRelease: featuregate.Alpha}, - MountContainers: {Default: false, PreRelease: featuregate.Alpha}, - CSIDriverRegistry: {Default: true, PreRelease: featuregate.Beta}, - CSINodeInfo: {Default: true, PreRelease: featuregate.Beta}, - BlockVolume: {Default: true, PreRelease: featuregate.Beta}, - StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, - ResourceLimitsPriorityFunction: {Default: false, PreRelease: featuregate.Alpha}, - SupportIPVSProxyMode: {Default: true, PreRelease: featuregate.GA}, - SupportPodPidsLimit: {Default: true, PreRelease: featuregate.Beta}, - SupportNodePidsLimit: {Default: true, PreRelease: featuregate.Beta}, - HyperVContainer: {Default: false, PreRelease: featuregate.Alpha}, - ScheduleDaemonSetPods: {Default: true, PreRelease: featuregate.Beta}, - TokenRequest: {Default: true, PreRelease: featuregate.Beta}, - TokenRequestProjection: {Default: true, PreRelease: featuregate.Beta}, - BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, - CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, - deprecatedGCERegionalPersistentDisk: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17 - CSIMigration: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationGCE: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAWS: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Alpha}, - CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, - RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, - CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Alpha}, - VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, - BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, - VolumeSubpathEnvExpansion: {Default: true, PreRelease: featuregate.Beta}, - ResourceQuotaScopeSelectors: {Default: true, PreRelease: featuregate.Beta}, - CSIBlockVolume: {Default: true, PreRelease: featuregate.Beta}, - CSIInlineVolume: {Default: false, PreRelease: featuregate.Alpha}, - RuntimeClass: {Default: true, PreRelease: featuregate.Beta}, - NodeLease: {Default: true, PreRelease: featuregate.Beta}, - SCTPSupport: {Default: false, PreRelease: featuregate.Alpha}, - VolumeSnapshotDataSource: {Default: false, PreRelease: featuregate.Alpha}, - ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, - TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, - KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, - WindowsGMSA: {Default: false, PreRelease: featuregate.Alpha}, - ServiceLoadBalancerFinalizer: {Default: false, PreRelease: featuregate.Alpha}, + DevicePlugins: {Default: true, PreRelease: featuregate.Beta}, + TaintBasedEvictions: {Default: true, PreRelease: featuregate.Beta}, + RotateKubeletServerCertificate: {Default: true, PreRelease: featuregate.Beta}, + RotateKubeletClientCertificate: {Default: true, PreRelease: featuregate.Beta}, + PersistentLocalVolumes: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17 + LocalStorageCapacityIsolation: {Default: true, PreRelease: featuregate.Beta}, + Sysctls: {Default: true, PreRelease: featuregate.Beta}, + DebugContainers: {Default: false, PreRelease: featuregate.Alpha}, + PodShareProcessNamespace: {Default: true, PreRelease: featuregate.Beta}, + PodPriority: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.18 + TaintNodesByCondition: {Default: true, PreRelease: featuregate.Beta}, + QOSReserved: {Default: false, PreRelease: featuregate.Alpha}, + ExpandPersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, + ExpandInUsePersistentVolumes: {Default: true, PreRelease: featuregate.Beta}, + ExpandCSIVolumes: {Default: false, PreRelease: featuregate.Alpha}, + AttachVolumeLimit: {Default: true, PreRelease: featuregate.Beta}, + CPUManager: {Default: true, PreRelease: featuregate.Beta}, + CPUCFSQuotaPeriod: {Default: false, PreRelease: featuregate.Alpha}, + TopologyManager: {Default: false, PreRelease: featuregate.Alpha}, + ServiceNodeExclusion: {Default: false, PreRelease: featuregate.Alpha}, + MountContainers: {Default: false, PreRelease: featuregate.Alpha}, + CSIDriverRegistry: {Default: true, PreRelease: featuregate.Beta}, + CSINodeInfo: {Default: true, PreRelease: featuregate.Beta}, + BlockVolume: {Default: true, PreRelease: featuregate.Beta}, + StorageObjectInUseProtection: {Default: true, PreRelease: featuregate.GA}, + ResourceLimitsPriorityFunction: {Default: false, PreRelease: featuregate.Alpha}, + SupportIPVSProxyMode: {Default: true, PreRelease: featuregate.GA}, + SupportPodPidsLimit: {Default: true, PreRelease: featuregate.Beta}, + SupportNodePidsLimit: {Default: true, PreRelease: featuregate.Beta}, + HyperVContainer: {Default: false, PreRelease: featuregate.Alpha}, + ScheduleDaemonSetPods: {Default: true, PreRelease: featuregate.Beta}, + TokenRequest: {Default: true, PreRelease: featuregate.Beta}, + TokenRequestProjection: {Default: true, PreRelease: featuregate.Beta}, + BoundServiceAccountTokenVolume: {Default: false, PreRelease: featuregate.Alpha}, + CRIContainerLogRotation: {Default: true, PreRelease: featuregate.Beta}, + deprecatedGCERegionalPersistentDisk: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.17 + CSIMigration: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationGCE: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAWS: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureDisk: {Default: false, PreRelease: featuregate.Alpha}, + CSIMigrationAzureFile: {Default: false, PreRelease: featuregate.Alpha}, + RunAsGroup: {Default: true, PreRelease: featuregate.Beta}, + CSIMigrationOpenStack: {Default: false, PreRelease: featuregate.Alpha}, + VolumeSubpath: {Default: true, PreRelease: featuregate.GA}, + BalanceAttachedNodeVolumes: {Default: false, PreRelease: featuregate.Alpha}, + VolumeSubpathEnvExpansion: {Default: true, PreRelease: featuregate.Beta}, + ResourceQuotaScopeSelectors: {Default: true, PreRelease: featuregate.Beta}, + CSIBlockVolume: {Default: true, PreRelease: featuregate.Beta}, + CSIInlineVolume: {Default: false, PreRelease: featuregate.Alpha}, + RuntimeClass: {Default: true, PreRelease: featuregate.Beta}, + NodeLease: {Default: true, PreRelease: featuregate.Beta}, + SCTPSupport: {Default: false, PreRelease: featuregate.Alpha}, + VolumeSnapshotDataSource: {Default: false, PreRelease: featuregate.Alpha}, + ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, + TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, + KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, + WindowsGMSA: {Default: false, PreRelease: featuregate.Alpha}, + ServiceLoadBalancerFinalizer: {Default: false, PreRelease: featuregate.Alpha}, LocalStorageCapacityIsolationFSQuotaMonitoring: {Default: false, PreRelease: featuregate.Alpha}, NonPreemptingPriority: {Default: false, PreRelease: featuregate.Alpha}, VolumePVCDataSource: {Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/kubelet/eviction/BUILD b/pkg/kubelet/eviction/BUILD index 2e6a49bd6f0..1c81a0003cf 100644 --- a/pkg/kubelet/eviction/BUILD +++ b/pkg/kubelet/eviction/BUILD @@ -17,6 +17,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", + "//pkg/apis/scheduling:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/eviction/api:go_default_library", diff --git a/pkg/kubelet/eviction/eviction_manager_test.go b/pkg/kubelet/eviction/eviction_manager_test.go index c74faf854e8..38c2cba983d 100644 --- a/pkg/kubelet/eviction/eviction_manager_test.go +++ b/pkg/kubelet/eviction/eviction_manager_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/record" featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapi "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/features" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" @@ -1132,12 +1133,12 @@ func TestInodePressureNodeFsInodes(t *testing.T) { } } -// TestCriticalPodsAreNotEvicted -func TestCriticalPodsAreNotEvicted(t *testing.T) { +// TestStaticCriticalPodsAreNotEvicted +func TestStaticCriticalPodsAreNotEvicted(t *testing.T) { podMaker := makePodWithMemoryStats summaryStatsMaker := makeMemoryStats podsToMake := []podToMake{ - {name: "critical", priority: defaultPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "800Mi"}, + {name: "critical", priority: scheduling.SystemCriticalPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "800Mi"}, } pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} @@ -1147,11 +1148,12 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) { podStats[pod] = podStat } - // Mark the pod as critical pods[0].Annotations = map[string]string{ - kubelettypes.CriticalPodAnnotationKey: "", kubelettypes.ConfigSourceAnnotationKey: kubelettypes.FileSource, } + // Mark the pod as critical + podPriority := scheduling.SystemCriticalPriority + pods[0].Spec.Priority = &podPriority pods[0].Namespace = kubeapi.NamespaceSystem podToEvict := pods[0] @@ -1208,9 +1210,6 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) { thresholdsFirstObservedAt: thresholdsObservedAt{}, } - // Enable critical pod annotation feature gate - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() - // induce soft threshold fakeClock.Step(1 * time.Minute) summaryProvider.result = summaryStatsMaker("1500Mi", podStats) manager.synchronize(diskInfoProvider, activePodsFunc) @@ -1253,8 +1252,11 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) { t.Errorf("Manager should not report memory pressure") } - // Disable critical pod annotation feature gate - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, false)() + pods[0].Annotations = map[string]string{ + kubelettypes.ConfigSourceAnnotationKey: kubelettypes.FileSource, + } + pods[0].Spec.Priority = nil + pods[0].Namespace = kubeapi.NamespaceSystem // induce memory pressure! fakeClock.Step(1 * time.Minute) diff --git a/pkg/kubelet/preemption/BUILD b/pkg/kubelet/preemption/BUILD index f3b80386bf1..7869ab83683 100644 --- a/pkg/kubelet/preemption/BUILD +++ b/pkg/kubelet/preemption/BUILD @@ -45,13 +45,9 @@ go_test( deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/scheduling:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", ], ) diff --git a/pkg/kubelet/preemption/preemption_test.go b/pkg/kubelet/preemption/preemption_test.go index 2110db33585..00a0c2e5475 100644 --- a/pkg/kubelet/preemption/preemption_test.go +++ b/pkg/kubelet/preemption/preemption_test.go @@ -23,17 +23,12 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/record" - featuregatetesting "k8s.io/component-base/featuregate/testing" kubeapi "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling" - "k8s.io/kubernetes/pkg/features" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) const ( - critical = "critical" clusterCritical = "cluster-critical" nodeCritical = "node-critical" bestEffort = "bestEffort" @@ -96,7 +91,6 @@ func getTestCriticalPodAdmissionHandler(podProvider *fakePodProvider, podKiller } func TestEvictPodsToFreeRequestsWithError(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() type testRun struct { testName string inputPods []*v1.Pod @@ -112,7 +106,7 @@ func TestEvictPodsToFreeRequestsWithError(t *testing.T) { { testName: "multiple pods eviction error", inputPods: []*v1.Pod{ - allPods[critical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], + allPods[clusterCritical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], allPods[guaranteed], allPods[highRequestGuaranteed]}, insufficientResources: getAdmissionRequirementList(0, 550, 0), expectErr: false, @@ -121,7 +115,7 @@ func TestEvictPodsToFreeRequestsWithError(t *testing.T) { } for _, r := range runs { podProvider.setPods(r.inputPods) - outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[critical], r.insufficientResources) + outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[clusterCritical], r.insufficientResources) outputPods := podKiller.getKilledPods() if !r.expectErr && outErr != nil { t.Errorf("evictPodsToFreeRequests returned an unexpected error during the %s test. Err: %v", r.testName, outErr) @@ -135,7 +129,6 @@ func TestEvictPodsToFreeRequestsWithError(t *testing.T) { } func TestEvictPodsToFreeRequests(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() type testRun struct { testName string inputPods []*v1.Pod @@ -150,7 +143,7 @@ func TestEvictPodsToFreeRequests(t *testing.T) { runs := []testRun{ { testName: "critical pods cannot be preempted", - inputPods: []*v1.Pod{allPods[critical]}, + inputPods: []*v1.Pod{allPods[clusterCritical]}, insufficientResources: getAdmissionRequirementList(0, 0, 1), expectErr: true, expectedOutput: nil, @@ -165,7 +158,7 @@ func TestEvictPodsToFreeRequests(t *testing.T) { { testName: "multiple pods evicted", inputPods: []*v1.Pod{ - allPods[critical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], + allPods[clusterCritical], allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], allPods[guaranteed], allPods[highRequestGuaranteed]}, insufficientResources: getAdmissionRequirementList(0, 550, 0), expectErr: false, @@ -174,7 +167,7 @@ func TestEvictPodsToFreeRequests(t *testing.T) { } for _, r := range runs { podProvider.setPods(r.inputPods) - outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[critical], r.insufficientResources) + outErr := criticalPodAdmissionHandler.evictPodsToFreeRequests(allPods[clusterCritical], r.insufficientResources) outputPods := podKiller.getKilledPods() if !r.expectErr && outErr != nil { t.Errorf("evictPodsToFreeRequests returned an unexpected error during the %s test. Err: %v", r.testName, outErr) @@ -203,7 +196,6 @@ func BenchmarkGetPodsToPreempt(t *testing.B) { } func TestGetPodsToPreempt(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() type testRun struct { testName string preemptor *v1.Pod @@ -216,7 +208,7 @@ func TestGetPodsToPreempt(t *testing.T) { runs := []testRun{ { testName: "no requirements", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{}, insufficientResources: getAdmissionRequirementList(0, 0, 0), expectErr: false, @@ -224,7 +216,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "no pods", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{}, insufficientResources: getAdmissionRequirementList(0, 0, 1), expectErr: true, @@ -232,7 +224,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "equal pods and resources requirements", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[burstable]}, insufficientResources: getAdmissionRequirementList(100, 100, 1), expectErr: false, @@ -240,7 +232,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "higher requirements than pod requests", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[burstable]}, insufficientResources: getAdmissionRequirementList(200, 200, 2), expectErr: true, @@ -248,7 +240,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "choose between bestEffort and burstable", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[burstable], allPods[bestEffort]}, insufficientResources: getAdmissionRequirementList(0, 0, 1), expectErr: false, @@ -256,7 +248,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "choose between burstable and guaranteed", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[burstable], allPods[guaranteed]}, insufficientResources: getAdmissionRequirementList(0, 0, 1), expectErr: false, @@ -264,7 +256,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "choose lower request burstable if it meets requirements", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[bestEffort], allPods[highRequestBurstable], allPods[burstable]}, insufficientResources: getAdmissionRequirementList(100, 100, 0), expectErr: false, @@ -272,7 +264,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "choose higher request burstable if lower does not meet requirements", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable]}, insufficientResources: getAdmissionRequirementList(150, 150, 0), expectErr: false, @@ -280,7 +272,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "multiple pods required", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], allPods[guaranteed], allPods[highRequestGuaranteed]}, insufficientResources: getAdmissionRequirementList(350, 350, 0), expectErr: false, @@ -288,7 +280,7 @@ func TestGetPodsToPreempt(t *testing.T) { }, { testName: "evict guaranteed when we have to, and dont evict the extra burstable", - preemptor: allPods[critical], + preemptor: allPods[clusterCritical], inputPods: []*v1.Pod{allPods[bestEffort], allPods[burstable], allPods[highRequestBurstable], allPods[guaranteed], allPods[highRequestGuaranteed]}, insufficientResources: getAdmissionRequirementList(0, 550, 0), expectErr: false, @@ -423,12 +415,6 @@ func getTestPods() map[string]*v1.Pod { }, }), bestEffort: getPodWithResources(bestEffort, v1.ResourceRequirements{}), - critical: getPodWithResources(critical, v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, - }), clusterCritical: getPodWithResources(clusterCritical, v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("100m"), @@ -474,9 +460,6 @@ func getTestPods() map[string]*v1.Pod { }, }), } - allPods[critical].Namespace = kubeapi.NamespaceSystem - allPods[critical].Annotations[kubetypes.CriticalPodAnnotationKey] = "" - allPods[clusterCritical].Namespace = kubeapi.NamespaceSystem allPods[clusterCritical].Spec.PriorityClassName = scheduling.SystemClusterCritical clusterPriority := scheduling.SystemCriticalPriority diff --git a/pkg/kubelet/types/BUILD b/pkg/kubelet/types/BUILD index 61a4583b905..437153301de 100644 --- a/pkg/kubelet/types/BUILD +++ b/pkg/kubelet/types/BUILD @@ -18,13 +18,10 @@ go_library( ], importpath = "k8s.io/kubernetes/pkg/kubelet/types", deps = [ - "//pkg/apis/core:go_default_library", "//pkg/apis/scheduling:go_default_library", - "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) @@ -38,11 +35,8 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", ], diff --git a/pkg/kubelet/types/pod_update.go b/pkg/kubelet/types/pod_update.go index 82a5f9a4020..c854b958e50 100644 --- a/pkg/kubelet/types/pod_update.go +++ b/pkg/kubelet/types/pod_update.go @@ -19,12 +19,9 @@ package types import ( "fmt" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - kubeapi "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling" - "k8s.io/kubernetes/pkg/features" ) const ( @@ -32,7 +29,6 @@ const ( ConfigMirrorAnnotationKey = v1.MirrorPodAnnotationKey ConfigFirstSeenAnnotationKey = "kubernetes.io/config.seen" ConfigHashAnnotationKey = "kubernetes.io/config.hash" - CriticalPodAnnotationKey = "scheduler.alpha.kubernetes.io/critical-pod" ) // PodOperation defines what changes will be made on a pod configuration. @@ -142,18 +138,11 @@ func (sp SyncPodType) String() string { } } -// IsCriticalPod returns true if the pod bears the critical pod annotation key or if pod's priority is greater than -// or equal to SystemCriticalPriority. Both the default scheduler and the kubelet use this function -// to make admission and scheduling decisions. +// IsCriticalPod returns true if pod's priority is greater than or equal to SystemCriticalPriority. func IsCriticalPod(pod *v1.Pod) bool { if pod.Spec.Priority != nil && IsCriticalPodBasedOnPriority(*pod.Spec.Priority) { return true } - if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) { - if IsCritical(pod.Namespace, pod.Annotations) { - return true - } - } return false } @@ -171,21 +160,6 @@ func Preemptable(preemptor, preemptee *v1.Pod) bool { return false } -// IsCritical returns true if parameters bear the critical pod annotation -// key. The DaemonSetController use this key directly to make scheduling decisions. -// TODO: @ravig - Deprecated. Remove this when we move to resolving critical pods based on priorityClassName. -func IsCritical(ns string, annotations map[string]string) bool { - // Critical pods are restricted to "kube-system" namespace as of now. - if ns != kubeapi.NamespaceSystem { - return false - } - val, ok := annotations[CriticalPodAnnotationKey] - if ok && val == "" { - return true - } - return false -} - // IsCriticalPodBasedOnPriority checks if the given pod is a critical pod based on priority resolved from pod Spec. func IsCriticalPodBasedOnPriority(priority int32) bool { if priority >= scheduling.SystemCriticalPriority { diff --git a/pkg/kubelet/types/pod_update_test.go b/pkg/kubelet/types/pod_update_test.go index 42cc2fae97c..6b45d1fdb9d 100644 --- a/pkg/kubelet/types/pod_update_test.go +++ b/pkg/kubelet/types/pod_update_test.go @@ -23,9 +23,6 @@ import ( "github.com/stretchr/testify/require" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" ) func TestGetValidatedSources(t *testing.T) { @@ -117,70 +114,6 @@ func TestString(t *testing.T) { } } -func TestIsCriticalPod(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() - cases := []struct { - pod v1.Pod - expected bool - }{ - { - pod: v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Namespace: "ns", - Annotations: map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "", - }, - }, - }, - expected: false, - }, - { - pod: v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod2", - Namespace: "ns", - Annotations: map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "abc", - }, - }, - }, - expected: false, - }, - { - pod: v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod3", - Namespace: "kube-system", - Annotations: map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "abc", - }, - }, - }, - expected: false, - }, - { - pod: v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod4", - Namespace: "kube-system", - Annotations: map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "", - }, - }, - }, - expected: true, - }, - } - for i, data := range cases { - actual := IsCriticalPod(&data.pod) - if actual != data.expected { - t.Errorf("IsCriticalPod result wrong:\nexpected: %v\nactual: %v for test[%d] with Annotations: %v", - data.expected, actual, i, data.pod.Annotations) - } - } -} - func TestIsCriticalPodBasedOnPriority(t *testing.T) { tests := []struct { priority int32 diff --git a/plugin/pkg/admission/priority/BUILD b/plugin/pkg/admission/priority/BUILD index 2b34a81b71d..50f3e71ebf7 100644 --- a/plugin/pkg/admission/priority/BUILD +++ b/plugin/pkg/admission/priority/BUILD @@ -36,7 +36,6 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/apis/scheduling:go_default_library", "//pkg/features:go_default_library", - "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/scheduling/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/plugin/pkg/admission/priority/admission.go b/plugin/pkg/admission/priority/admission.go index 75eae31d3ca..481f4f893ee 100644 --- a/plugin/pkg/admission/priority/admission.go +++ b/plugin/pkg/admission/priority/admission.go @@ -35,7 +35,6 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/features" - kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" ) const ( @@ -178,13 +177,6 @@ func (p *priorityPlugin) admitPod(a admission.Attributes) error { if operation == admission.Create { var priority int32 var preemptionPolicy *apiv1.PreemptionPolicy - // TODO: @ravig - This is for backwards compatibility to ensure that critical pods with annotations just work fine. - // Remove when no longer needed. - if len(pod.Spec.PriorityClassName) == 0 && - utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) && - kubelettypes.IsCritical(a.GetNamespace(), pod.Annotations) { - pod.Spec.PriorityClassName = scheduling.SystemClusterCritical - } if len(pod.Spec.PriorityClassName) == 0 { var err error var pcName string diff --git a/plugin/pkg/admission/priority/admission_test.go b/plugin/pkg/admission/priority/admission_test.go index b84fa4bcf26..1feff4318b5 100644 --- a/plugin/pkg/admission/priority/admission_test.go +++ b/plugin/pkg/admission/priority/admission_test.go @@ -422,23 +422,7 @@ func TestPodAdmission(t *testing.T) { Priority: &intPriority, }, }, - // pod[7]: Pod with a critical priority annotation. This needs to be automatically assigned - // system-cluster-critical - { - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-w-system-priority", - Namespace: "kube-system", - Annotations: map[string]string{"scheduler.alpha.kubernetes.io/critical-pod": ""}, - }, - Spec: api.PodSpec{ - Containers: []api.Container{ - { - Name: containerName, - }, - }, - }, - }, - // pod[8]: Pod with a system priority class name in non-system namespace + // pod[7]: Pod with a system priority class name in non-system namespace { ObjectMeta: metav1.ObjectMeta{ Name: "pod-w-system-priority-in-nonsystem-namespace", @@ -453,7 +437,7 @@ func TestPodAdmission(t *testing.T) { PriorityClassName: scheduling.SystemClusterCritical, }, }, - // pod[9]: Pod with a priority value that matches the resolved priority + // pod[8]: Pod with a priority value that matches the resolved priority { ObjectMeta: metav1.ObjectMeta{ Name: "pod-w-zero-priority-in-nonsystem-namespace", @@ -468,7 +452,7 @@ func TestPodAdmission(t *testing.T) { Priority: &zeroPriority, }, }, - // pod[10]: Pod with a priority value that matches the resolved default priority + // pod[9]: Pod with a priority value that matches the resolved default priority { ObjectMeta: metav1.ObjectMeta{ Name: "pod-w-priority-matching-default-priority", @@ -483,7 +467,7 @@ func TestPodAdmission(t *testing.T) { Priority: &defaultClass2.Value, }, }, - // pod[11]: Pod with a priority value that matches the resolved priority + // pod[10]: Pod with a priority value that matches the resolved priority { ObjectMeta: metav1.ObjectMeta{ Name: "pod-w-priority-matching-resolved-default-priority", @@ -499,7 +483,7 @@ func TestPodAdmission(t *testing.T) { Priority: &systemClusterCritical.Value, }, }, - // pod[12]: Pod without a preemption policy that matches the resolved preemption policy + // pod[11]: Pod without a preemption policy that matches the resolved preemption policy { ObjectMeta: metav1.ObjectMeta{ Name: "pod-never-preemption-policy-matching-resolved-preemption-policy", @@ -516,7 +500,7 @@ func TestPodAdmission(t *testing.T) { PreemptionPolicy: nil, }, }, - // pod[13]: Pod with a preemption policy that matches the resolved preemption policy + // pod[12]: Pod with a preemption policy that matches the resolved preemption policy { ObjectMeta: metav1.ObjectMeta{ Name: "pod-preemption-policy-matching-resolved-preemption-policy", @@ -533,7 +517,7 @@ func TestPodAdmission(t *testing.T) { PreemptionPolicy: &preemptLowerPriority, }, }, - // pod[14]: Pod with a preemption policy that does't match the resolved preemption policy + // pod[13]: Pod with a preemption policy that does't match the resolved preemption policy { ObjectMeta: metav1.ObjectMeta{ Name: "pod-preemption-policy-not-matching-resolved-preemption-policy", @@ -551,8 +535,6 @@ func TestPodAdmission(t *testing.T) { }, }, } - // Enable ExperimentalCriticalPodAnnotation feature gate. - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ExperimentalCriticalPodAnnotation, true)() // Enable NonPreemptingPriority feature gate. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NonPreemptingPriority, true)() tests := []struct { @@ -638,18 +620,10 @@ func TestPodAdmission(t *testing.T) { true, nil, }, - { - "pod with critical pod annotation", - []*scheduling.PriorityClass{systemClusterCritical}, - *pods[7], - scheduling.SystemCriticalPriority, - false, - nil, - }, { "pod with system critical priority in non-system namespace", []*scheduling.PriorityClass{systemClusterCritical}, - *pods[8], + *pods[7], scheduling.SystemCriticalPriority, true, nil, @@ -657,7 +631,7 @@ func TestPodAdmission(t *testing.T) { { "pod with priority that matches computed priority", []*scheduling.PriorityClass{nondefaultClass1}, - *pods[9], + *pods[8], 0, false, nil, @@ -665,7 +639,7 @@ func TestPodAdmission(t *testing.T) { { "pod with priority that matches default priority", []*scheduling.PriorityClass{defaultClass2}, - *pods[10], + *pods[9], defaultClass2.Value, false, nil, @@ -673,7 +647,7 @@ func TestPodAdmission(t *testing.T) { { "pod with priority that matches resolved priority", []*scheduling.PriorityClass{systemClusterCritical}, - *pods[11], + *pods[10], systemClusterCritical.Value, false, nil, @@ -681,7 +655,7 @@ func TestPodAdmission(t *testing.T) { { "pod with nil preemtpion policy", []*scheduling.PriorityClass{preemptionPolicyClass}, - *pods[12], + *pods[11], preemptionPolicyClass.Value, false, nil, @@ -689,7 +663,7 @@ func TestPodAdmission(t *testing.T) { { "pod with preemtpion policy that matches resolved preemtpion policy", []*scheduling.PriorityClass{preemptionPolicyClass}, - *pods[13], + *pods[12], preemptionPolicyClass.Value, false, &preemptLowerPriority, @@ -697,7 +671,7 @@ func TestPodAdmission(t *testing.T) { { "pod with preemtpion policy that does't matches resolved preemtpion policy", []*scheduling.PriorityClass{preemptionPolicyClass}, - *pods[14], + *pods[13], preemptionPolicyClass.Value, true, &preemptLowerPriority, diff --git a/test/e2e/testing-manifests/sample-device-plugin.yaml b/test/e2e/testing-manifests/sample-device-plugin.yaml index 1c7baff5eb0..c2512737881 100644 --- a/test/e2e/testing-manifests/sample-device-plugin.yaml +++ b/test/e2e/testing-manifests/sample-device-plugin.yaml @@ -14,7 +14,6 @@ spec: labels: k8s-app: sample-device-plugin annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical tolerations: diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 21490fd0d1a..6d5e3db81f2 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -126,6 +126,7 @@ go_test( tags = ["e2e"], deps = [ "//pkg/apis/core:go_default_library", + "//pkg/apis/scheduling:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet:go_default_library", "//pkg/kubelet/apis/config:go_default_library", diff --git a/test/e2e_node/critical_pod_test.go b/test/e2e_node/critical_pod_test.go index 297be277070..4c71738e5c4 100644 --- a/test/e2e_node/critical_pod_test.go +++ b/test/e2e_node/critical_pod_test.go @@ -23,8 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeapi "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/features" - kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/apis/scheduling" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/test/e2e/framework" imageutils "k8s.io/kubernetes/test/utils/image" @@ -44,13 +43,6 @@ var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive] [NodeFeature:C f := framework.NewDefaultFramework("critical-pod-test") Context("when we need to admit a critical pod", func() { - tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { - if initialConfig.FeatureGates == nil { - initialConfig.FeatureGates = make(map[string]bool) - } - initialConfig.FeatureGates[string(features.ExperimentalCriticalPodAnnotation)] = true - }) - It("should be able to create and delete a critical pod", func() { configEnabled, err := isKubeletConfigEnabled(f) framework.ExpectNoError(err) @@ -142,12 +134,11 @@ func getTestPod(critical bool, name string, resources v1.ResourceRequirements) * } if critical { pod.ObjectMeta.Namespace = kubeapi.NamespaceSystem - pod.ObjectMeta.Annotations = map[string]string{ - kubelettypes.CriticalPodAnnotationKey: "", - } - Expect(kubelettypes.IsCritical(pod.Namespace, pod.Annotations)).To(BeTrue(), "pod should be a critical pod") + pod.ObjectMeta.Annotations = map[string]string{} + pod.Spec.PriorityClassName = scheduling.SystemClusterCritical + Expect(kubelettypes.IsCriticalPod(pod)).To(BeTrue(), "pod should be a critical pod") } else { - Expect(kubelettypes.IsCritical(pod.Namespace, pod.Annotations)).To(BeFalse(), "pod should not be a critical pod") + Expect(kubelettypes.IsCriticalPod(pod)).To(BeFalse(), "pod should not be a critical pod") } return pod } diff --git a/test/kubemark/resources/kube_dns_template.yaml b/test/kubemark/resources/kube_dns_template.yaml index 02c2a68a2e5..5857f43271e 100644 --- a/test/kubemark/resources/kube_dns_template.yaml +++ b/test/kubemark/resources/kube_dns_template.yaml @@ -57,9 +57,8 @@ spec: metadata: labels: k8s-app: kube-dns - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: + priorityClassName: system-node-critical tolerations: - key: "CriticalAddonsOnly" operator: "Exists"