mirror of
https://github.com/kata-containers/kata-containers.git
synced 2026-07-10 07:05:51 +00:00
Merge pull request #13212 from zachspar/spar/kata-deploy-daemonset-labels-and-affinity
kata-deploy: add podLabels, podAnnotations and affinity to DaemonSet
This commit is contained in:
Fabiano Fidêncio
GitHub
@@ -16,7 +16,7 @@ helm show values --version X.Y.Z oci://ghcr.io/kata-containers/kata-deploy-chart
|
||||
|
||||
Kata ships with a number of pre-built artifacts and runtimes. You may selectively enable or disable specific shims. For example:
|
||||
|
||||
```yaml title="values.yaml"
|
||||
```yaml
|
||||
shims:
|
||||
disableAll: true
|
||||
qemu:
|
||||
@@ -332,7 +332,7 @@ $ helm install kata-deploy \
|
||||
|
||||
You can also use a values file:
|
||||
|
||||
```yaml title="values.yaml"
|
||||
```yaml
|
||||
nodeSelector:
|
||||
kata-containers: "enabled"
|
||||
node-type: "worker"
|
||||
@@ -342,6 +342,107 @@ nodeSelector:
|
||||
$ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}"
|
||||
```
|
||||
|
||||
### `podLabels`
|
||||
|
||||
You can add extra labels to the kata-deploy DaemonSet pods. These are applied
|
||||
in addition to the `name: kata-deploy` label that the chart uses internally.
|
||||
The chart ignores a `name` key in `podLabels` and always sets the required
|
||||
selector label itself.
|
||||
|
||||
```sh
|
||||
$ helm install kata-deploy \
|
||||
--set podLabels.team=platform \
|
||||
"${CHART}" --version "${VERSION}"
|
||||
```
|
||||
|
||||
Or via a values file:
|
||||
|
||||
```yaml
|
||||
podLabels:
|
||||
team: platform
|
||||
```
|
||||
|
||||
### `podAnnotations`
|
||||
|
||||
You can add annotations to the kata-deploy DaemonSet pods for whatever your
|
||||
environment needs: Prometheus scrape hints, policy markers, revision metadata,
|
||||
and so on. The chart does not set any annotations on the kata-deploy DaemonSet
|
||||
pod template by default, so nothing is reserved or overwritten unless you set
|
||||
`podAnnotations` yourself.
|
||||
|
||||
```sh
|
||||
$ helm install kata-deploy \
|
||||
--set-string podAnnotations.prometheus\.io/scrape=false \
|
||||
"${CHART}" --version "${VERSION}"
|
||||
```
|
||||
|
||||
Or via a values file:
|
||||
|
||||
```yaml
|
||||
podAnnotations:
|
||||
prometheus.io/scrape: "false"
|
||||
example.com/owner: platform-team
|
||||
```
|
||||
|
||||
### `affinity`
|
||||
|
||||
Use `affinity` when you need more granular scheduling controls than
|
||||
`nodeSelector` alone. `nodeSelector` only matches exact key/value pairs on a
|
||||
node; affinity gives you `matchExpressions` (e.g. `In`, `NotIn`) and rules
|
||||
about other pods on the same node. For example, you might want kata-deploy on
|
||||
nodes reserved for your platform team but *not* on nodes that run the GPU
|
||||
operator.
|
||||
|
||||
```sh
|
||||
# First, label the nodes where kata-deploy should run
|
||||
$ kubectl label nodes worker-node-1 node.cloud/reserved=platform-team
|
||||
$ kubectl label nodes worker-node-2 node.cloud/reserved=platform-team
|
||||
|
||||
# Then install the chart with affinity
|
||||
$ helm install kata-deploy -f values.yaml "${CHART}" --version "${VERSION}"
|
||||
```
|
||||
|
||||
```yaml
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- gpu-operator
|
||||
topologyKey: kubernetes.io/hostname
|
||||
```
|
||||
|
||||
When `node-feature-discovery.enabled=true`, the chart also merges in a
|
||||
`nodeAffinity` rule that requires hardware virtualization support.
|
||||
|
||||
!!! note "How NFD and user nodeAffinity are combined"
|
||||
Within a single `nodeSelectorTerm`, `matchExpressions` and `matchFields` are
|
||||
**AND**-ed (all must match). Across `nodeSelectorTerms`, terms are **OR**-ed
|
||||
(any one term may match).
|
||||
|
||||
If you set `affinity.nodeAffinity` yourself, only your **required**
|
||||
`nodeSelectorTerms` participate in the merge. They are combined with the
|
||||
built-in virtualization terms as **(NFD OR-group) AND (user OR-group)**:
|
||||
each built-in term is AND-ed with each of your required terms. Multiple
|
||||
user required terms remain OR among themselves. NFD virtualization
|
||||
requirements cannot be bypassed by user affinity.
|
||||
|
||||
If you set `nodeAffinity` without `requiredDuringSchedulingIgnoredDuringExecution`,
|
||||
the built-in NFD required terms are still applied. Other affinity fields
|
||||
(`podAffinity`, `podAntiAffinity`, and `preferredDuringSchedulingIgnoredDuringExecution`)
|
||||
are passed through unchanged.
|
||||
|
||||
### Multiple Kata installations on the Same Node
|
||||
|
||||
For debugging, testing and other use-case it is possible to deploy multiple
|
||||
|
||||
@@ -200,8 +200,8 @@ EOF
|
||||
echo "# Uninstall complete, verifying cleanup..." >&3
|
||||
|
||||
# Wait for node to recover — containerd restart during cleanup may
|
||||
# cause brief unavailability (especially on k3s/rke2).
|
||||
kubectl wait nodes --timeout=120s --all --for condition=Ready=True
|
||||
# cause brief unavailability (especially on k3s/rke2/microk8s).
|
||||
kubectl wait nodes --timeout=300s --all --for condition=Ready=True
|
||||
|
||||
# RuntimeClasses must be gone (filter out AKS-managed ones)
|
||||
local rc_count
|
||||
@@ -242,7 +242,7 @@ EOF
|
||||
|
||||
teardown() {
|
||||
if [[ "${BATS_TEST_NAME}" == *"restart"* ]]; then
|
||||
kubectl delete pod "${LIFECYCLE_POD_NAME}" --ignore-not-found=true --wait=false 2>/dev/null || true
|
||||
kubectl delete pod "${LIFECYCLE_POD_NAME}" --ignore-not-found=true --timeout=120s 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
374
tests/functional/kata-deploy/kata-deploy-scheduling.bats
Normal file
374
tests/functional/kata-deploy/kata-deploy-scheduling.bats
Normal file
@@ -0,0 +1,374 @@
|
||||
#!/usr/bin/env bats
|
||||
# Copyright (c) 2026 The Kata Containers Authors
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Helm template tests for kata-deploy DaemonSet scheduling options
|
||||
# (podLabels, podAnnotations, affinity). No cluster required.
|
||||
|
||||
load "${BATS_TEST_DIRNAME}/../../common.bash"
|
||||
|
||||
source "${BATS_TEST_DIRNAME}/lib/helm-deploy.bash"
|
||||
|
||||
CHART_PATH="$(get_chart_path)"
|
||||
RENDERED="/tmp/kata-deploy-scheduling-rendered.yaml"
|
||||
|
||||
render_chart() {
|
||||
helm template kata-deploy "${CHART_PATH}" \
|
||||
--set image.reference=quay.io/kata-containers/kata-deploy \
|
||||
--set image.tag=latest \
|
||||
"$@" > "${RENDERED}"
|
||||
}
|
||||
|
||||
# Extract the kata-deploy DaemonSet manifest (not kata-monitor or NFD subchart).
|
||||
extract_kata_deploy_ds() {
|
||||
awk '
|
||||
/^kind: DaemonSet$/ { buf = $0 "\n"; in_ds = 1; has_name = 0; next }
|
||||
in_ds {
|
||||
buf = buf $0 "\n"
|
||||
if ($0 ~ /^ name: kata-deploy$/) { has_name = 1 }
|
||||
if ($0 ~ /^---$/) {
|
||||
if (has_name) { printf "%s", buf; exit }
|
||||
in_ds = 0; buf = ""; has_name = 0
|
||||
next
|
||||
}
|
||||
}
|
||||
END { if (has_name && in_ds) { printf "%s", buf } }
|
||||
' "${RENDERED}"
|
||||
}
|
||||
|
||||
# Count nodeSelectorTerms under requiredDuringSchedulingIgnoredDuringExecution in a manifest.
|
||||
count_required_node_selector_terms() {
|
||||
local manifest="${1}"
|
||||
echo "${manifest}" | awk '
|
||||
/requiredDuringSchedulingIgnoredDuringExecution:/ { in_req = 1; next }
|
||||
in_req && /preferredDuringSchedulingIgnoredDuringExecution:/ { exit }
|
||||
in_req && /^ [a-zA-Z]/ { exit }
|
||||
in_req && /- match(Expressions|Fields):/ { count++ }
|
||||
END { print count + 0 }
|
||||
'
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Template Rendering Tests (no cluster required)
|
||||
# =============================================================================
|
||||
|
||||
@test "Helm template: default values keep single name pod label and no affinity" {
|
||||
render_chart
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
[[ -n "${ds}" ]]
|
||||
echo "${ds}" | grep -q "name: kata-deploy"
|
||||
echo "${ds}" | grep -A5 "template:" | grep -A3 "labels:" | grep -q "name: kata-deploy"
|
||||
! echo "${ds}" | grep -A10 "template:" | grep -A5 "metadata:" | grep -q "annotations:"
|
||||
! echo "${ds}" | grep -q "affinity:"
|
||||
}
|
||||
|
||||
@test "Helm template: podLabels are applied to pod template" {
|
||||
render_chart --set podLabels.team=platform
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "name: kata-deploy"
|
||||
echo "${ds}" | grep -A5 "template:" | grep -A4 "labels:" | grep -q "team: platform"
|
||||
}
|
||||
|
||||
@test "Helm template: podLabels cannot override required name selector label" {
|
||||
render_chart --set podLabels.name=wrong
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
! echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep -q "name: wrong"
|
||||
echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep -q "name: kata-deploy"
|
||||
! echo "${ds}" | grep -A8 "template:" | grep -A6 "labels:" | grep "name:" | grep -qv "name: kata-deploy"
|
||||
}
|
||||
|
||||
@test "Helm template: podAnnotations are applied to pod template" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
podAnnotations:
|
||||
example.com/owner: platform-team
|
||||
prometheus.io/scrape: "false"
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}"
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -A10 "template:" | grep -A5 "metadata:" | grep -q "annotations:"
|
||||
echo "${ds}" | grep -q "example.com/owner: platform-team"
|
||||
echo "${ds}" | grep -q 'prometheus.io/scrape: "false"'
|
||||
}
|
||||
|
||||
@test "Helm template: user affinity is applied to pod spec" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- gpu-operator
|
||||
topologyKey: kubernetes.io/hostname
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}"
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -q "affinity:"
|
||||
echo "${ds}" | grep -q "node.cloud/reserved"
|
||||
echo "${ds}" | grep -q "platform-team"
|
||||
echo "${ds}" | grep -q "podAntiAffinity:"
|
||||
echo "${ds}" | grep -q "gpu-operator"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD enabled merges virtualization nodeAffinity with user nodeAffinity" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds term_count
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
term_count=$(count_required_node_selector_terms "${ds}")
|
||||
|
||||
[[ "${term_count}" -eq 6 ]]
|
||||
echo "${ds}" | grep -q "node.cloud/reserved"
|
||||
echo "${ds}" | grep -q "platform-team"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.SVM"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD enabled applies virtualization nodeAffinity when user sets no affinity" {
|
||||
render_chart --set node-feature-discovery.enabled=true
|
||||
|
||||
local ds term_count
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
term_count=$(count_required_node_selector_terms "${ds}")
|
||||
|
||||
[[ "${term_count}" -eq 6 ]]
|
||||
echo "${ds}" | grep -q "affinity:"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.SVM"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD merge preserves podAntiAffinity" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- gpu-operator
|
||||
topologyKey: kubernetes.io/hostname
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -q "podAntiAffinity:"
|
||||
echo "${ds}" | grep -q "gpu-operator"
|
||||
echo "${ds}" | grep -q "platform-team"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD merge preserves matchFields in nodeSelectorTerms" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchFields:
|
||||
- key: metadata.name
|
||||
operator: In
|
||||
values:
|
||||
- worker-node-1
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -q "matchFields:"
|
||||
echo "${ds}" | grep -q "metadata.name"
|
||||
echo "${ds}" | grep -q "worker-node-1"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD merge cross-products multiple user OR terms" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- gpu-team
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds term_count
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
term_count=$(count_required_node_selector_terms "${ds}")
|
||||
|
||||
[[ "${term_count}" -eq 12 ]]
|
||||
echo "${ds}" | grep -q "platform-team"
|
||||
echo "${ds}" | grep -q "gpu-team"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD merge omits empty matchFields" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -q "node.cloud/reserved"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
! echo "${ds}" | grep -q 'matchFields: \[\]'
|
||||
}
|
||||
|
||||
@test "Helm template: NFD merge preserves preferredDuringSchedulingIgnoredDuringExecution" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- platform-team
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- preferred-team
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
|
||||
echo "${ds}" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:"
|
||||
echo "${ds}" | grep -q "preferred-team"
|
||||
echo "${ds}" | grep -q "weight: 100"
|
||||
echo "${ds}" | grep -q "platform-team"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
}
|
||||
|
||||
@test "Helm template: NFD required applied when user has no required terms" {
|
||||
local values_file
|
||||
values_file=$(mktemp)
|
||||
cat > "${values_file}" <<EOF
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 50
|
||||
preference:
|
||||
matchExpressions:
|
||||
- key: node.cloud/reserved
|
||||
operator: In
|
||||
values:
|
||||
- preferred-team
|
||||
EOF
|
||||
|
||||
render_chart -f "${values_file}" --set node-feature-discovery.enabled=true
|
||||
rm -f "${values_file}"
|
||||
|
||||
local ds term_count
|
||||
ds=$(extract_kata_deploy_ds)
|
||||
term_count=$(count_required_node_selector_terms "${ds}")
|
||||
|
||||
[[ "${term_count}" -eq 6 ]]
|
||||
echo "${ds}" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:"
|
||||
echo "${ds}" | grep -q "preferred-team"
|
||||
echo "${ds}" | grep -q "feature.node.kubernetes.io/cpu-cpuid.VMX"
|
||||
}
|
||||
@@ -22,6 +22,7 @@ else
|
||||
"kata-deploy.bats" \
|
||||
"kata-deploy-custom-runtimes.bats" \
|
||||
"kata-deploy-lifecycle.bats" \
|
||||
"kata-deploy-scheduling.bats" \
|
||||
)
|
||||
fi
|
||||
|
||||
|
||||
@@ -921,3 +921,123 @@ Returns "true" when at least one default shim has a non-empty dropIn value.
|
||||
{{- end -}}
|
||||
{{- if $has -}}true{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
NFD virtualization nodeAffinity for the kata-deploy DaemonSet.
|
||||
Applied when node-feature-discovery is managed by this chart (enabled: true).
|
||||
Kata Containers requires hardware virtualization support to function.
|
||||
|
||||
Note: Virtualization checks are ONLY enforced when node-feature-discovery is
|
||||
managed by kata-deploy. If node-feature-discovery is installed
|
||||
independently (enabled: false), no checks are applied because we cannot
|
||||
guarantee the external node-feature-discovery configuration and labels.
|
||||
|
||||
NOTE: For kata-remote/peer-pods support in the future, add a condition here:
|
||||
if and (index .Values "node-feature-discovery" "enabled") (not .Values.cloud-api-adaptor.enabled)
|
||||
*/}}
|
||||
{{- define "kata-deploy.nfdVirtualizationNodeAffinity" -}}
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
# x86_64: Intel VT-x (VMX) support
|
||||
- matchExpressions:
|
||||
- key: feature.node.kubernetes.io/cpu-cpuid.VMX
|
||||
operator: In
|
||||
values:
|
||||
- "true"
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "amd64"
|
||||
# x86_64: AMD-V (SVM) support
|
||||
- matchExpressions:
|
||||
- key: feature.node.kubernetes.io/cpu-cpuid.SVM
|
||||
operator: In
|
||||
values:
|
||||
- "true"
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "amd64"
|
||||
# aarch64: Allow all ARM64 nodes (virtualization check not yet implemented)
|
||||
# TODO: Implement proper virtualization detection for aarch64
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "arm64"
|
||||
- "aarch64"
|
||||
# s390x: Allow all s390x nodes (virtualization check not yet implemented)
|
||||
# TODO: Implement proper virtualization detection for s390x
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "s390x"
|
||||
# ppc64le: Allow all ppc64le nodes (virtualization check not yet implemented)
|
||||
# TODO: Implement proper virtualization detection for ppc64le
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "ppc64le"
|
||||
# riscv64: Allow all RISC-V nodes (virtualization support not yet available)
|
||||
# TODO: Implement virtualization detection when RISC-V virt support is available
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "riscv64"
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Merged affinity for the kata-deploy DaemonSet.
|
||||
When NFD is enabled, the built-in virtualization nodeAffinity is always applied.
|
||||
Kubernetes semantics:
|
||||
- nodeSelectorTerms are OR within a group (match any one term)
|
||||
- matchExpressions and matchFields are AND within a term (all must match)
|
||||
If the user sets affinity.nodeAffinity, their required nodeSelectorTerms are
|
||||
combined with the NFD terms as (NFD OR-group) AND (user OR-group) via cross-
|
||||
product: each NFD term is AND-ed with each user term. NFD virtualization
|
||||
requirements cannot be bypassed by user affinity.
|
||||
*/}}
|
||||
{{- define "kata-deploy.daemonsetAffinity" -}}
|
||||
{{- $affinity := .Values.affinity | default dict | deepCopy -}}
|
||||
{{- if index .Values "node-feature-discovery" "enabled" -}}
|
||||
{{- $nfd := include "kata-deploy.nfdVirtualizationNodeAffinity" . | fromYaml -}}
|
||||
{{- $nfdNodeAffinity := $nfd.nodeAffinity -}}
|
||||
{{- if not (hasKey $affinity "nodeAffinity") -}}
|
||||
{{- $affinity = merge $affinity $nfd -}}
|
||||
{{- else -}}
|
||||
{{- $userNodeAffinity := $affinity.nodeAffinity | deepCopy -}}
|
||||
{{- $nfdRequired := $nfdNodeAffinity.requiredDuringSchedulingIgnoredDuringExecution | default dict -}}
|
||||
{{- $nfdTerms := $nfdRequired.nodeSelectorTerms | default list -}}
|
||||
{{- $userRequired := $userNodeAffinity.requiredDuringSchedulingIgnoredDuringExecution | default dict -}}
|
||||
{{- $userTerms := $userRequired.nodeSelectorTerms | default list -}}
|
||||
{{- $mergedTerms := list -}}
|
||||
{{- if $userTerms -}}
|
||||
{{- range $nfdTerm := $nfdTerms -}}
|
||||
{{- range $userTerm := $userTerms -}}
|
||||
{{- $mergedTerm := dict -}}
|
||||
{{- $exprs := concat ($nfdTerm.matchExpressions | default list) ($userTerm.matchExpressions | default list) -}}
|
||||
{{- $fields := concat ($nfdTerm.matchFields | default list) ($userTerm.matchFields | default list) -}}
|
||||
{{- if $exprs -}}
|
||||
{{- $_ := set $mergedTerm "matchExpressions" $exprs -}}
|
||||
{{- end -}}
|
||||
{{- if $fields -}}
|
||||
{{- $_ := set $mergedTerm "matchFields" $fields -}}
|
||||
{{- end -}}
|
||||
{{- $mergedTerms = append $mergedTerms $mergedTerm -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- else -}}
|
||||
{{- $mergedTerms = $nfdTerms -}}
|
||||
{{- end -}}
|
||||
{{- $_ := set $userNodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" (dict "nodeSelectorTerms" $mergedTerms) -}}
|
||||
{{- $_ := set $affinity "nodeAffinity" $userNodeAffinity -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if $affinity -}}
|
||||
{{- $affinity | toYaml -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -39,10 +39,20 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- with .Values.podLabels }}
|
||||
{{- $podLabels := omit . "name" }}
|
||||
{{- with $podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.env.multiInstallSuffix }}
|
||||
name: {{ .Chart.Name }}-{{ .Values.env.multiInstallSuffix }}
|
||||
{{- else }}
|
||||
name: {{ .Chart.Name }}
|
||||
{{- end }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
@@ -65,72 +75,10 @@ spec:
|
||||
{{- with .Values.priorityClassName }}
|
||||
priorityClassName: {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- if index .Values "node-feature-discovery" "enabled" }}
|
||||
# When node-feature-discovery is managed by this chart (enabled: true), enforce virtualization requirements.
|
||||
# Kata Containers requires hardware virtualization support to function.
|
||||
#
|
||||
# Note: Virtualization checks are ONLY enforced when node-feature-discovery is
|
||||
# managed by kata-deploy
|
||||
#
|
||||
# If node-feature-discovery is installed independently (enabled: false),
|
||||
# no checks are applied because we cannot guarantee the external
|
||||
# node-feature-discovery configuration and labels.
|
||||
#
|
||||
# NOTE: For kata-remote/peer-pods support in the future, add a condition here:
|
||||
# if and (index .Values "node-feature-discovery" "enabled") (not .Values.cloud-api-adaptor.enabled)
|
||||
{{- $dsAffinity := include "kata-deploy.daemonsetAffinity" . | trim -}}
|
||||
{{- if $dsAffinity }}
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
# x86_64: Intel VT-x (VMX) support
|
||||
- matchExpressions:
|
||||
- key: feature.node.kubernetes.io/cpu-cpuid.VMX
|
||||
operator: In
|
||||
values:
|
||||
- "true"
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "amd64"
|
||||
# x86_64: AMD-V (SVM) support
|
||||
- matchExpressions:
|
||||
- key: feature.node.kubernetes.io/cpu-cpuid.SVM
|
||||
operator: In
|
||||
values:
|
||||
- "true"
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "amd64"
|
||||
# aarch64: Allow all ARM64 nodes (virtualization check not yet implemented)
|
||||
# TODO: Implement proper virtualization detection for aarch64
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "arm64"
|
||||
- "aarch64"
|
||||
# s390x: Allow all s390x nodes (virtualization check not yet implemented)
|
||||
# TODO: Implement proper virtualization detection for s390x
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "s390x"
|
||||
# ppc64le: Allow all ppc64le nodes (virtualization check not yet implemented)
|
||||
# TODO: Implement proper virtualization detection for ppc64le
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "ppc64le"
|
||||
# riscv64: Allow all RISC-V nodes (virtualization support not yet available)
|
||||
# TODO: Implement virtualization detection when RISC-V virt support is available
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values:
|
||||
- "riscv64"
|
||||
{{- $dsAffinity | nindent 8 }}
|
||||
{{- end }}
|
||||
hostPID: true
|
||||
# Allow SIGTERM cleanup (uninstall) to complete before force-kill.
|
||||
|
||||
@@ -198,6 +198,49 @@ tolerations: []
|
||||
# - "katacontainers.io/kata-runtime:NoSchedule"
|
||||
startupTaints: []
|
||||
|
||||
# Extra labels for the kata-deploy DaemonSet pods (in addition to the
|
||||
# required "name: kata-deploy" label used by the chart). A "name" key here
|
||||
# is ignored.
|
||||
# Examples:
|
||||
# podLabels:
|
||||
# team: platform
|
||||
podLabels: {}
|
||||
|
||||
# Annotations for the kata-deploy DaemonSet pods.
|
||||
# Examples:
|
||||
# podAnnotations:
|
||||
# prometheus.io/scrape: "false"
|
||||
podAnnotations: {}
|
||||
|
||||
# Affinity rules for the kata-deploy DaemonSet.
|
||||
# nodeSelector matches exact key/value pairs; affinity supports matchExpressions
|
||||
# and rules about other pods on the same node (e.g. podAntiAffinity).
|
||||
# When node-feature-discovery.enabled=true, the chart merges in a virtualization
|
||||
# nodeAffinity rule. User required nodeSelectorTerms are AND-combined with the
|
||||
# built-in virtualization terms; multiple user required terms remain OR among
|
||||
# themselves. NFD virtualization requirements cannot be bypassed. podAffinity,
|
||||
# podAntiAffinity, and preferredDuringSchedulingIgnoredDuringExecution pass through.
|
||||
# Examples:
|
||||
# affinity:
|
||||
# nodeAffinity:
|
||||
# requiredDuringSchedulingIgnoredDuringExecution:
|
||||
# nodeSelectorTerms:
|
||||
# - matchExpressions:
|
||||
# - key: node.cloud/reserved
|
||||
# operator: In
|
||||
# values:
|
||||
# - platform-team
|
||||
# podAntiAffinity:
|
||||
# requiredDuringSchedulingIgnoredDuringExecution:
|
||||
# - labelSelector:
|
||||
# matchExpressions:
|
||||
# - key: app
|
||||
# operator: In
|
||||
# values:
|
||||
# - gpu-operator
|
||||
# topologyKey: kubernetes.io/hostname
|
||||
affinity: {}
|
||||
|
||||
# Priority class name for the kata-deploy DaemonSet pods.
|
||||
#
|
||||
# kata-deploy is an infrastructure DaemonSet that installs Kata runtime
|
||||
|
||||
Reference in New Issue
Block a user