github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-09 22:24:51 +00:00
Code Issues Projects Releases Wiki Activity

kata-deploy: harden NFD nodeAffinity merge and document semantics

Browse Source 
Replace fragile per-term Helm merge with explicit NodeSelectorTerm
construction, document (NFD OR-group) AND (user OR-group) behavior, and
add bats coverage to prevent merge regressions.

Signed-off-by: Zachary Spar <zspar@coreweave.com>
This commit is contained in:
Zachary Spar
2026-07-06 12:00:47 -04:00
parent bf131b2448
commit 83497cf994
4 changed files with 225 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
docs/helm-configuration.md
View File

@@ -424,10 +424,24 @@ affinity:
```
When `node-feature-discovery.enabled=true`, the chart also merges in a
`nodeAffinity` rule that requires hardware virtualization support. If you set
`affinity.nodeAffinity` yourself, your required `nodeSelectorTerms` are
combined with the built-in virtualization terms. Other affinity fields
(`podAffinity`, `podAntiAffinity`, and so on) are passed through as-is.
`nodeAffinity` rule that requires hardware virtualization support.
!!! note "How NFD and user nodeAffinity are combined"
Within a single `nodeSelectorTerm`, `matchExpressions` and `matchFields` are
**AND**-ed (all must match). Across `nodeSelectorTerms`, terms are **OR**-ed
(any one term may match).
If you set `affinity.nodeAffinity` yourself, only your **required**
`nodeSelectorTerms` participate in the merge. They are combined with the
built-in virtualization terms as **(NFD OR-group) AND (user OR-group)**:
each built-in term is AND-ed with each of your required terms. Multiple
user required terms remain OR among themselves. NFD virtualization
requirements cannot be bypassed by user affinity.
If you set `nodeAffinity` without `requiredDuringSchedulingIgnoredDuringExecution`,
the built-in NFD required terms are still applied. Other affinity fields
(`podAffinity`, `podAntiAffinity`, and `preferredDuringSchedulingIgnoredDuringExecution`)
are passed through unchanged.
### Multiple Kata installations on the Same Node

189
tests/functional/kata-deploy/kata-deploy-scheduling.bats
View File

@@ -37,6 +37,18 @@ extract_kata_deploy_ds() {
' "${RENDERED}"
}
# Count nodeSelectorTerms under requiredDuringSchedulingIgnoredDuringExecution in a manifest.
count_required_node_selector_terms() {
local manifest="${1}"
echo "${manifest}" | awk '
/requiredDuringSchedulingIgnoredDuringExecution:/ { in_req = 1; next }
in_req && /preferredDuringSchedulingIgnoredDuringExecution:/ { exit }
in_req && /^ [a-zA-Z]/ { exit }
in_req && /- match(Expressions|Fields):/ { count++ }
END { print count + 0 }
'
}
# =============================================================================
# Template Rendering Tests (no cluster required)
# =============================================================================
@@ -150,15 +162,66 @@ EOF
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
rm -f "${values_file}"
local ds
local ds term_count
ds=$(extract_kata_deploy_ds)
term_count=$(count_required_node_selector_terms "${ds}")
[[ "${term_count}" -eq 6 ]]
echo "${ds}" | grep -q "node.cloud/reserved"
echo "${ds}" | grep -q "platform-team"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.SVM"
}
@test "Helm template: NFD enabled applies virtualization nodeAffinity when user sets no affinity" {
render_chart --set node-feature-discovery.enabled=true
local ds term_count
ds=$(extract_kata_deploy_ds)
term_count=$(count_required_node_selector_terms "${ds}")
[[ "${term_count}" -eq 6 ]]
echo "${ds}" | grep -q "affinity:"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.SVM"
}
@test "Helm template: NFD merge preserves podAntiAffinity" {
local values_file
values_file=$(mktemp)
cat > "${values_file}" <<EOF
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- platform-team
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- gpu-operator
topologyKey: kubernetes.io/hostname
EOF
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
rm -f "${values_file}"
local ds
ds=$(extract_kata_deploy_ds)
echo "${ds}" | grep -q "podAntiAffinity:"
echo "${ds}" | grep -q "gpu-operator"
echo "${ds}" | grep -q "platform-team"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
}
@test "Helm template: NFD merge preserves matchFields in nodeSelectorTerms" {
local values_file
values_file=$(mktemp)
@@ -185,3 +248,127 @@ EOF
echo "${ds}" | grep -q "worker-node-1"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
}
@test "Helm template: NFD merge cross-products multiple user OR terms" {
local values_file
values_file=$(mktemp)
cat > "${values_file}" <<EOF
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- platform-team
- matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- gpu-team
EOF
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
rm -f "${values_file}"
local ds term_count
ds=$(extract_kata_deploy_ds)
term_count=$(count_required_node_selector_terms "${ds}")
[[ "${term_count}" -eq 12 ]]
echo "${ds}" | grep -q "platform-team"
echo "${ds}" | grep -q "gpu-team"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
}
@test "Helm template: NFD merge omits empty matchFields" {
local values_file
values_file=$(mktemp)
cat > "${values_file}" <<EOF
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- platform-team
EOF
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
rm -f "${values_file}"
local ds
ds=$(extract_kata_deploy_ds)
echo "${ds}" | grep -q "node.cloud/reserved"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
! echo "${ds}" | grep -q 'matchFields: \[\]'
}
@test "Helm template: NFD merge preserves preferredDuringSchedulingIgnoredDuringExecution" {
local values_file
values_file=$(mktemp)
cat > "${values_file}" <<EOF
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- platform-team
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- preferred-team
EOF
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
rm -f "${values_file}"
local ds
ds=$(extract_kata_deploy_ds)
echo "${ds}" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:"
echo "${ds}" | grep -q "preferred-team"
echo "${ds}" | grep -q "weight: 100"
echo "${ds}" | grep -q "platform-team"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
}
@test "Helm template: NFD required applied when user has no required terms" {
local values_file
values_file=$(mktemp)
cat > "${values_file}" <<EOF
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 50
preference:
matchExpressions:
- key: node.cloud/reserved
operator: In
values:
- preferred-team
EOF
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
rm -f "${values_file}"
local ds term_count
ds=$(extract_kata_deploy_ds)
term_count=$(count_required_node_selector_terms "${ds}")
[[ "${term_count}" -eq 6 ]]
echo "${ds}" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:"
echo "${ds}" | grep -q "preferred-team"
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
}

19
tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl
View File

@@ -993,8 +993,13 @@ nodeAffinity:
{{/*
Merged affinity for the kata-deploy DaemonSet.
When NFD is enabled, the built-in virtualization nodeAffinity is always applied.
Kubernetes semantics:
- nodeSelectorTerms are OR within a group (match any one term)
- matchExpressions and matchFields are AND within a term (all must match)
If the user sets affinity.nodeAffinity, their required nodeSelectorTerms are
merged with the NFD terms (each NFD term is AND-ed with each user term).
combined with the NFD terms as (NFD OR-group) AND (user OR-group) via cross-
product: each NFD term is AND-ed with each user term. NFD virtualization
requirements cannot be bypassed by user affinity.
*/}}
{{- define "kata-deploy.daemonsetAffinity" -}}
{{- $affinity := .Values.affinity | default dict | deepCopy -}}
@@ -1013,9 +1018,15 @@ merged with the NFD terms (each NFD term is AND-ed with each user term).
{{- if $userTerms -}}
{{- range $nfdTerm := $nfdTerms -}}
{{- range $userTerm := $userTerms -}}
{{- $mergedTerm := merge (deepCopy $nfdTerm) (deepCopy $userTerm) -}}
{{- $_ := set $mergedTerm "matchExpressions" (concat ($nfdTerm.matchExpressions | default list) ($userTerm.matchExpressions | default list)) -}}
{{- $_ := set $mergedTerm "matchFields" (concat ($nfdTerm.matchFields | default list) ($userTerm.matchFields | default list)) -}}
{{- $mergedTerm := dict -}}
{{- $exprs := concat ($nfdTerm.matchExpressions | default list) ($userTerm.matchExpressions | default list) -}}
{{- $fields := concat ($nfdTerm.matchFields | default list) ($userTerm.matchFields | default list) -}}
{{- if $exprs -}}
{{- $_ := set $mergedTerm "matchExpressions" $exprs -}}
{{- end -}}
{{- if $fields -}}
{{- $_ := set $mergedTerm "matchFields" $fields -}}
{{- end -}}
{{- $mergedTerms = append $mergedTerms $mergedTerm -}}
{{- end -}}
{{- end -}}

5
tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml
View File

@@ -216,7 +216,10 @@ podAnnotations: {}
# nodeSelector matches exact key/value pairs; affinity supports matchExpressions
# and rules about other pods on the same node (e.g. podAntiAffinity).
# When node-feature-discovery.enabled=true, the chart merges in a virtualization
# nodeAffinity rule; user affinity.nodeAffinity required terms are combined with it.
# nodeAffinity rule. User required nodeSelectorTerms are AND-combined with the
# built-in virtualization terms; multiple user required terms remain OR among
# themselves. NFD virtualization requirements cannot be bypassed. podAffinity,
# podAntiAffinity, and preferredDuringSchedulingIgnoredDuringExecution pass through.
# Examples:
# affinity:
# nodeAffinity: