From 5e4946b21eff6bf83468ffaef23a8d70e714f79d Mon Sep 17 00:00:00 2001 From: Zachary Spar Date: Tue, 30 Jun 2026 16:21:28 -0400 Subject: [PATCH 1/5] kata-deploy: add podLabels, podAnnotations and affinity to DaemonSet Expose podLabels, podAnnotations and affinity in values.yaml so operators can apply more granular scheduling controls on the kata-deploy DaemonSet alongside the existing nodeSelector and tolerations knobs. Defaults are empty, so existing deployments render the same manifest. When NFD is enabled, the chart merges the hard-coded virtualization nodeAffinity with any user provided affinity. Signed-off-by: Zachary Spar --- docs/helm-configuration.md | 67 ++++++++- .../kata-deploy/kata-deploy-scheduling.bats | 129 ++++++++++++++++++ .../kata-deploy/templates/_helpers.tpl | 108 +++++++++++++++ .../kata-deploy/templates/kata-deploy.yaml | 75 ++-------- .../helm-chart/kata-deploy/values.yaml | 39 ++++++ 5 files changed, 351 insertions(+), 67 deletions(-) create mode 100644 tests/functional/kata-deploy/kata-deploy-scheduling.bats diff --git a/docs/helm-configuration.md b/docs/helm-configuration.md index e2843c6a41..59f2d73df7 100644 --- a/docs/helm-configuration.md +++ b/docs/helm-configuration.md @@ -16,7 +16,7 @@ helm show values --version X.Y.Z oci://ghcr.io/kata-containers/kata-deploy-chart Kata ships with a number of pre-built artifacts and runtimes. You may selectively enable or disable specific shims. For example: -```yaml title="values.yaml" +```yaml shims: disableAll: true qemu: @@ -332,7 +332,7 @@ $ helm install kata-deploy \ You can also use a values file: -```yaml title="values.yaml" +```yaml nodeSelector: kata-containers: "enabled" node-type: "worker" @@ -342,6 +342,69 @@ nodeSelector: $ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}" ``` +### `podLabels` + +You can add extra labels to the kata-deploy DaemonSet pods. These are applied +in addition to the `name: kata-deploy` label that the chart uses internally. + +```sh +$ helm install kata-deploy \ + --set podLabels.team=platform \ + "${CHART}" --version "${VERSION}" +``` + +Or via a values file: + +```yaml +podLabels: + team: platform +``` + +### `affinity` + +Use `affinity` when you need more granular scheduling controls than +`nodeSelector` alone. `nodeSelector` only matches exact key/value pairs on a +node; affinity gives you `matchExpressions` (e.g. `In`, `NotIn`) and rules +about other pods on the same node. For example, you might want kata-deploy on +nodes reserved for your platform team but *not* on nodes that run the GPU +operator. + +```sh +# First, label the nodes where kata-deploy should run +$ kubectl label nodes worker-node-1 node.cloud/reserved=platform-team +$ kubectl label nodes worker-node-2 node.cloud/reserved=platform-team + +# Then install the chart with affinity +$ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}" +``` + +```yaml +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.cloud/reserved + operator: In + values: + - platform-team + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - gpu-operator + topologyKey: kubernetes.io/hostname +``` + +When `node-feature-discovery.enabled=true`, the chart also merges in a +`nodeAffinity` rule that requires hardware virtualization support. If you set +`affinity.nodeAffinity` yourself, your required `nodeSelectorTerms` are +combined with the built-in virtualization terms. Other affinity fields +(`podAffinity`, `podAntiAffinity`, and so on) are passed through as-is. + ### Multiple Kata installations on the Same Node For debugging, testing and other use-case it is possible to deploy multiple diff --git a/tests/functional/kata-deploy/kata-deploy-scheduling.bats b/tests/functional/kata-deploy/kata-deploy-scheduling.bats new file mode 100644 index 0000000000..df10da09bd --- /dev/null +++ b/tests/functional/kata-deploy/kata-deploy-scheduling.bats @@ -0,0 +1,129 @@ +#!/usr/bin/env bats +# Copyright (c) 2026 The Kata Containers Authors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Helm template tests for kata-deploy DaemonSet scheduling options +# (podLabels, podAnnotations, affinity). No cluster required. + +load "${BATS_TEST_DIRNAME}/../../common.bash" + +source "${BATS_TEST_DIRNAME}/lib/helm-deploy.bash" + +CHART_PATH="$(get_chart_path)" +RENDERED="/tmp/kata-deploy-scheduling-rendered.yaml" + +render_chart() { + helm template kata-deploy "${CHART_PATH}" \ + --set image.reference=quay.io/kata-containers/kata-deploy \ + --set image.tag=latest \ + "$@" > "${RENDERED}" +} + +# Extract the kata-deploy DaemonSet manifest (not kata-monitor or NFD subchart). +extract_kata_deploy_ds() { + awk ' + /^kind: DaemonSet$/ { buf = $0 "\n"; in_ds = 1; has_name = 0; next } + in_ds { + buf = buf $0 "\n" + if ($0 ~ /^ name: kata-deploy$/) { has_name = 1 } + if ($0 ~ /^---$/) { + if (has_name) { printf "%s", buf; exit } + in_ds = 0; buf = ""; has_name = 0 + next + } + } + END { if (has_name && in_ds) { printf "%s", buf } } + ' "${RENDERED}" +} + +# ============================================================================= +# Template Rendering Tests (no cluster required) +# ============================================================================= + +@test "Helm template: default values keep single name pod label and no affinity" { + render_chart + + local ds + ds=$(extract_kata_deploy_ds) + + [[ -n "${ds}" ]] + echo "${ds}" | grep -q "name: kata-deploy" + echo "${ds}" | grep -A5 "template:" | grep -A3 "labels:" | grep -q "name: kata-deploy" + ! echo "${ds}" | grep -A10 "template:" | grep -A5 "metadata:" | grep -q "annotations:" + ! echo "${ds}" | grep -q "affinity:" +} + +@test "Helm template: podLabels are applied to pod template" { + render_chart --set podLabels.team=platform + + local ds + ds=$(extract_kata_deploy_ds) + + echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "name: kata-deploy" + echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "team: platform" +} + +@test "Helm template: user affinity is applied to pod spec" { + local values_file + values_file=$(mktemp) + cat > "${values_file}" < "${values_file}" < Date: Fri, 12 Jun 2026 17:07:57 -0400 Subject: [PATCH 2/5] kata-deploy: harden affinity merge Merge NFD and user nodeSelectorTerms with merge/concat. Ignore the name key in podLabels, document podAnnotations and extend bats coverage for annotations. Signed-off-by: Zachary Spar --- docs/helm-configuration.md | 24 ++++++++ .../kata-deploy/kata-deploy-scheduling.bats | 58 +++++++++++++++++++ .../kata-deploy/templates/_helpers.tpl | 7 ++- .../kata-deploy/templates/kata-deploy.yaml | 9 ++- .../helm-chart/kata-deploy/values.yaml | 3 +- 5 files changed, 94 insertions(+), 7 deletions(-) diff --git a/docs/helm-configuration.md b/docs/helm-configuration.md index 59f2d73df7..3e8d5770f7 100644 --- a/docs/helm-configuration.md +++ b/docs/helm-configuration.md @@ -346,6 +346,8 @@ $ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}" You can add extra labels to the kata-deploy DaemonSet pods. These are applied in addition to the `name: kata-deploy` label that the chart uses internally. +The chart ignores a `name` key in `podLabels` and always sets the required +selector label itself. ```sh $ helm install kata-deploy \ @@ -360,6 +362,28 @@ podLabels: team: platform ``` +### `podAnnotations` + +You can add annotations to the kata-deploy DaemonSet pods for whatever your +environment needs: Prometheus scrape hints, policy markers, revision metadata, +and so on. The chart does not set any annotations on the kata-deploy DaemonSet +pod template by default, so nothing is reserved or overwritten unless you set +`podAnnotations` yourself. + +```sh +$ helm install kata-deploy \ + --set-string podAnnotations.prometheus\.io/scrape=false \ + "${CHART}" --version "${VERSION}" +``` + +Or via a values file: + +```yaml +podAnnotations: + prometheus.io/scrape: "false" + example.com/owner: platform-team +``` + ### `affinity` Use `affinity` when you need more granular scheduling controls than diff --git a/tests/functional/kata-deploy/kata-deploy-scheduling.bats b/tests/functional/kata-deploy/kata-deploy-scheduling.bats index df10da09bd..53c24e1b05 100644 --- a/tests/functional/kata-deploy/kata-deploy-scheduling.bats +++ b/tests/functional/kata-deploy/kata-deploy-scheduling.bats @@ -64,6 +64,37 @@ extract_kata_deploy_ds() { echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "team: platform" } +@test "Helm template: podLabels cannot override required name selector label" { + render_chart --set podLabels.name=wrong + + local ds + ds=$(extract_kata_deploy_ds) + + ! echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep -q "name: wrong" + echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep -q "name: kata-deploy" + ! echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep "name:" | grep -qv "name: kata-deploy" +} + +@test "Helm template: podAnnotations are applied to pod template" { + local values_file + values_file=$(mktemp) + cat > "${values_file}" < "${values_file}" < Date: Tue, 30 Jun 2026 16:23:52 -0400 Subject: [PATCH 3/5] kata-deploy: include bats test Add kata-deploy-scheduling.bats to KATA_DEPLOY_TEST_UNION so the new Helm scheduling template tests run in CI. Signed-off-by: Zachary Spar --- tests/functional/kata-deploy/run-kata-deploy-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/kata-deploy/run-kata-deploy-tests.sh b/tests/functional/kata-deploy/run-kata-deploy-tests.sh index 611b891d0c..b801977c60 100644 --- a/tests/functional/kata-deploy/run-kata-deploy-tests.sh +++ b/tests/functional/kata-deploy/run-kata-deploy-tests.sh @@ -22,6 +22,7 @@ else "kata-deploy.bats" \ "kata-deploy-custom-runtimes.bats" \ "kata-deploy-lifecycle.bats" \ + "kata-deploy-scheduling.bats" \ ) fi From bf131b2448cd2b6c0283a06559f0b16d9db675c7 Mon Sep 17 00:00:00 2001 From: Zachary Spar Date: Thu, 2 Jul 2026 11:04:51 -0400 Subject: [PATCH 4/5] kata-deploy: tweak teardown options Increase timeout for node wait condition to 300s, and add timeout for pod deletion. Signed-off-by: Zachary Spar --- tests/functional/kata-deploy/kata-deploy-lifecycle.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/kata-deploy/kata-deploy-lifecycle.bats b/tests/functional/kata-deploy/kata-deploy-lifecycle.bats index 4b88a254a5..0436afaf5e 100644 --- a/tests/functional/kata-deploy/kata-deploy-lifecycle.bats +++ b/tests/functional/kata-deploy/kata-deploy-lifecycle.bats @@ -200,8 +200,8 @@ EOF echo "# Uninstall complete, verifying cleanup..." >&3 # Wait for node to recover — containerd restart during cleanup may - # cause brief unavailability (especially on k3s/rke2). - kubectl wait nodes --timeout=120s --all --for condition=Ready=True + # cause brief unavailability (especially on k3s/rke2/microk8s). + kubectl wait nodes --timeout=300s --all --for condition=Ready=True # RuntimeClasses must be gone (filter out AKS-managed ones) local rc_count @@ -242,7 +242,7 @@ EOF teardown() { if [[ "${BATS_TEST_NAME}" == *"restart"* ]]; then - kubectl delete pod "${LIFECYCLE_POD_NAME}" --ignore-not-found=true --wait=false 2>/dev/null || true + kubectl delete pod "${LIFECYCLE_POD_NAME}" --ignore-not-found=true --timeout=120s 2>/dev/null || true fi } From 83497cf994e3d5de0fd20cdab994093588fc18b6 Mon Sep 17 00:00:00 2001 From: Zachary Spar Date: Mon, 6 Jul 2026 12:00:47 -0400 Subject: [PATCH 5/5] kata-deploy: harden NFD nodeAffinity merge and document semantics Replace fragile per-term Helm merge with explicit NodeSelectorTerm construction, document (NFD OR-group) AND (user OR-group) behavior, and add bats coverage to prevent merge regressions. Signed-off-by: Zachary Spar --- docs/helm-configuration.md | 22 +- .../kata-deploy/kata-deploy-scheduling.bats | 189 +++++++++++++++++- .../kata-deploy/templates/_helpers.tpl | 19 +- .../helm-chart/kata-deploy/values.yaml | 5 +- 4 files changed, 225 insertions(+), 10 deletions(-) diff --git a/docs/helm-configuration.md b/docs/helm-configuration.md index 3e8d5770f7..69f69b7e22 100644 --- a/docs/helm-configuration.md +++ b/docs/helm-configuration.md @@ -424,10 +424,24 @@ affinity: ``` When `node-feature-discovery.enabled=true`, the chart also merges in a -`nodeAffinity` rule that requires hardware virtualization support. If you set -`affinity.nodeAffinity` yourself, your required `nodeSelectorTerms` are -combined with the built-in virtualization terms. Other affinity fields -(`podAffinity`, `podAntiAffinity`, and so on) are passed through as-is. +`nodeAffinity` rule that requires hardware virtualization support. + +!!! note "How NFD and user nodeAffinity are combined" + Within a single `nodeSelectorTerm`, `matchExpressions` and `matchFields` are + **AND**-ed (all must match). Across `nodeSelectorTerms`, terms are **OR**-ed + (any one term may match). + + If you set `affinity.nodeAffinity` yourself, only your **required** + `nodeSelectorTerms` participate in the merge. They are combined with the + built-in virtualization terms as **(NFD OR-group) AND (user OR-group)**: + each built-in term is AND-ed with each of your required terms. Multiple + user required terms remain OR among themselves. NFD virtualization + requirements cannot be bypassed by user affinity. + + If you set `nodeAffinity` without `requiredDuringSchedulingIgnoredDuringExecution`, + the built-in NFD required terms are still applied. Other affinity fields + (`podAffinity`, `podAntiAffinity`, and `preferredDuringSchedulingIgnoredDuringExecution`) + are passed through unchanged. ### Multiple Kata installations on the Same Node diff --git a/tests/functional/kata-deploy/kata-deploy-scheduling.bats b/tests/functional/kata-deploy/kata-deploy-scheduling.bats index 53c24e1b05..b4af9f6929 100644 --- a/tests/functional/kata-deploy/kata-deploy-scheduling.bats +++ b/tests/functional/kata-deploy/kata-deploy-scheduling.bats @@ -37,6 +37,18 @@ extract_kata_deploy_ds() { ' "${RENDERED}" } +# Count nodeSelectorTerms under requiredDuringSchedulingIgnoredDuringExecution in a manifest. +count_required_node_selector_terms() { + local manifest="${1}" + echo "${manifest}" | awk ' + /requiredDuringSchedulingIgnoredDuringExecution:/ { in_req = 1; next } + in_req && /preferredDuringSchedulingIgnoredDuringExecution:/ { exit } + in_req && /^ [a-zA-Z]/ { exit } + in_req && /- match(Expressions|Fields):/ { count++ } + END { print count + 0 } + ' +} + # ============================================================================= # Template Rendering Tests (no cluster required) # ============================================================================= @@ -150,15 +162,66 @@ EOF render_chart -f "${values_file}" --set node-feature-discovery.enabled=true rm -f "${values_file}" - local ds + local ds term_count ds=$(extract_kata_deploy_ds) + term_count=$(count_required_node_selector_terms "${ds}") + [[ "${term_count}" -eq 6 ]] echo "${ds}" | grep -q "node.cloud/reserved" echo "${ds}" | grep -q "platform-team" echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX" echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.SVM" } +@test "Helm template: NFD enabled applies virtualization nodeAffinity when user sets no affinity" { + render_chart --set node-feature-discovery.enabled=true + + local ds term_count + ds=$(extract_kata_deploy_ds) + term_count=$(count_required_node_selector_terms "${ds}") + + [[ "${term_count}" -eq 6 ]] + echo "${ds}" | grep -q "affinity:" + echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX" + echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.SVM" +} + +@test "Helm template: NFD merge preserves podAntiAffinity" { + local values_file + values_file=$(mktemp) + cat > "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" <