diff --git a/docs/helm-configuration.md b/docs/helm-configuration.md index e2843c6a41..69f69b7e22 100644 --- a/docs/helm-configuration.md +++ b/docs/helm-configuration.md @@ -16,7 +16,7 @@ helm show values --version X.Y.Z oci://ghcr.io/kata-containers/kata-deploy-chart Kata ships with a number of pre-built artifacts and runtimes. You may selectively enable or disable specific shims. For example: -```yaml title="values.yaml" +```yaml shims: disableAll: true qemu: @@ -332,7 +332,7 @@ $ helm install kata-deploy \ You can also use a values file: -```yaml title="values.yaml" +```yaml nodeSelector: kata-containers: "enabled" node-type: "worker" @@ -342,6 +342,107 @@ nodeSelector: $ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}" ``` +### `podLabels` + +You can add extra labels to the kata-deploy DaemonSet pods. These are applied +in addition to the `name: kata-deploy` label that the chart uses internally. +The chart ignores a `name` key in `podLabels` and always sets the required +selector label itself. + +```sh +$ helm install kata-deploy \ + --set podLabels.team=platform \ + "${CHART}" --version "${VERSION}" +``` + +Or via a values file: + +```yaml +podLabels: + team: platform +``` + +### `podAnnotations` + +You can add annotations to the kata-deploy DaemonSet pods for whatever your +environment needs: Prometheus scrape hints, policy markers, revision metadata, +and so on. The chart does not set any annotations on the kata-deploy DaemonSet +pod template by default, so nothing is reserved or overwritten unless you set +`podAnnotations` yourself. + +```sh +$ helm install kata-deploy \ + --set-string podAnnotations.prometheus\.io/scrape=false \ + "${CHART}" --version "${VERSION}" +``` + +Or via a values file: + +```yaml +podAnnotations: + prometheus.io/scrape: "false" + example.com/owner: platform-team +``` + +### `affinity` + +Use `affinity` when you need more granular scheduling controls than +`nodeSelector` alone. `nodeSelector` only matches exact key/value pairs on a +node; affinity gives you `matchExpressions` (e.g. `In`, `NotIn`) and rules +about other pods on the same node. For example, you might want kata-deploy on +nodes reserved for your platform team but *not* on nodes that run the GPU +operator. + +```sh +# First, label the nodes where kata-deploy should run +$ kubectl label nodes worker-node-1 node.cloud/reserved=platform-team +$ kubectl label nodes worker-node-2 node.cloud/reserved=platform-team + +# Then install the chart with affinity +$ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}" +``` + +```yaml +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.cloud/reserved + operator: In + values: + - platform-team + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - gpu-operator + topologyKey: kubernetes.io/hostname +``` + +When `node-feature-discovery.enabled=true`, the chart also merges in a +`nodeAffinity` rule that requires hardware virtualization support. + +!!! note "How NFD and user nodeAffinity are combined" + Within a single `nodeSelectorTerm`, `matchExpressions` and `matchFields` are + **AND**-ed (all must match). Across `nodeSelectorTerms`, terms are **OR**-ed + (any one term may match). + + If you set `affinity.nodeAffinity` yourself, only your **required** + `nodeSelectorTerms` participate in the merge. They are combined with the + built-in virtualization terms as **(NFD OR-group) AND (user OR-group)**: + each built-in term is AND-ed with each of your required terms. Multiple + user required terms remain OR among themselves. NFD virtualization + requirements cannot be bypassed by user affinity. + + If you set `nodeAffinity` without `requiredDuringSchedulingIgnoredDuringExecution`, + the built-in NFD required terms are still applied. Other affinity fields + (`podAffinity`, `podAntiAffinity`, and `preferredDuringSchedulingIgnoredDuringExecution`) + are passed through unchanged. + ### Multiple Kata installations on the Same Node For debugging, testing and other use-case it is possible to deploy multiple diff --git a/tests/functional/kata-deploy/kata-deploy-lifecycle.bats b/tests/functional/kata-deploy/kata-deploy-lifecycle.bats index 4b88a254a5..0436afaf5e 100644 --- a/tests/functional/kata-deploy/kata-deploy-lifecycle.bats +++ b/tests/functional/kata-deploy/kata-deploy-lifecycle.bats @@ -200,8 +200,8 @@ EOF echo "# Uninstall complete, verifying cleanup..." >&3 # Wait for node to recover — containerd restart during cleanup may - # cause brief unavailability (especially on k3s/rke2). - kubectl wait nodes --timeout=120s --all --for condition=Ready=True + # cause brief unavailability (especially on k3s/rke2/microk8s). + kubectl wait nodes --timeout=300s --all --for condition=Ready=True # RuntimeClasses must be gone (filter out AKS-managed ones) local rc_count @@ -242,7 +242,7 @@ EOF teardown() { if [[ "${BATS_TEST_NAME}" == *"restart"* ]]; then - kubectl delete pod "${LIFECYCLE_POD_NAME}" --ignore-not-found=true --wait=false 2>/dev/null || true + kubectl delete pod "${LIFECYCLE_POD_NAME}" --ignore-not-found=true --timeout=120s 2>/dev/null || true fi } diff --git a/tests/functional/kata-deploy/kata-deploy-scheduling.bats b/tests/functional/kata-deploy/kata-deploy-scheduling.bats new file mode 100644 index 0000000000..b4af9f6929 --- /dev/null +++ b/tests/functional/kata-deploy/kata-deploy-scheduling.bats @@ -0,0 +1,374 @@ +#!/usr/bin/env bats +# Copyright (c) 2026 The Kata Containers Authors +# +# SPDX-License-Identifier: Apache-2.0 +# +# Helm template tests for kata-deploy DaemonSet scheduling options +# (podLabels, podAnnotations, affinity). No cluster required. + +load "${BATS_TEST_DIRNAME}/../../common.bash" + +source "${BATS_TEST_DIRNAME}/lib/helm-deploy.bash" + +CHART_PATH="$(get_chart_path)" +RENDERED="/tmp/kata-deploy-scheduling-rendered.yaml" + +render_chart() { + helm template kata-deploy "${CHART_PATH}" \ + --set image.reference=quay.io/kata-containers/kata-deploy \ + --set image.tag=latest \ + "$@" > "${RENDERED}" +} + +# Extract the kata-deploy DaemonSet manifest (not kata-monitor or NFD subchart). +extract_kata_deploy_ds() { + awk ' + /^kind: DaemonSet$/ { buf = $0 "\n"; in_ds = 1; has_name = 0; next } + in_ds { + buf = buf $0 "\n" + if ($0 ~ /^ name: kata-deploy$/) { has_name = 1 } + if ($0 ~ /^---$/) { + if (has_name) { printf "%s", buf; exit } + in_ds = 0; buf = ""; has_name = 0 + next + } + } + END { if (has_name && in_ds) { printf "%s", buf } } + ' "${RENDERED}" +} + +# Count nodeSelectorTerms under requiredDuringSchedulingIgnoredDuringExecution in a manifest. +count_required_node_selector_terms() { + local manifest="${1}" + echo "${manifest}" | awk ' + /requiredDuringSchedulingIgnoredDuringExecution:/ { in_req = 1; next } + in_req && /preferredDuringSchedulingIgnoredDuringExecution:/ { exit } + in_req && /^ [a-zA-Z]/ { exit } + in_req && /- match(Expressions|Fields):/ { count++ } + END { print count + 0 } + ' +} + +# ============================================================================= +# Template Rendering Tests (no cluster required) +# ============================================================================= + +@test "Helm template: default values keep single name pod label and no affinity" { + render_chart + + local ds + ds=$(extract_kata_deploy_ds) + + [[ -n "${ds}" ]] + echo "${ds}" | grep -q "name: kata-deploy" + echo "${ds}" | grep -A5 "template:" | grep -A3 "labels:" | grep -q "name: kata-deploy" + ! echo "${ds}" | grep -A10 "template:" | grep -A5 "metadata:" | grep -q "annotations:" + ! echo "${ds}" | grep -q "affinity:" +} + +@test "Helm template: podLabels are applied to pod template" { + render_chart --set podLabels.team=platform + + local ds + ds=$(extract_kata_deploy_ds) + + echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "name: kata-deploy" + echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "team: platform" +} + +@test "Helm template: podLabels cannot override required name selector label" { + render_chart --set podLabels.name=wrong + + local ds + ds=$(extract_kata_deploy_ds) + + ! echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep -q "name: wrong" + echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep -q "name: kata-deploy" + ! echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep "name:" | grep -qv "name: kata-deploy" +} + +@test "Helm template: podAnnotations are applied to pod template" { + local values_file + values_file=$(mktemp) + cat > "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" < "${values_file}" <