mirror of
https://github.com/kata-containers/kata-containers.git
synced 2026-02-21 14:22:24 +00:00
kata-deploy: Add TEE nodeSelectors for TEE shims when NFD is detected
When NFD is detected (deployed by the chart or existing in the cluster),
apply shim-specific nodeSelectors only for TEE runtime classes (snp,
tdx, and se).
Non-TEE shims keep existing behavior (e.g. runtimeClass.nodeSelector for
nvidia GPU from f3bba0885 is unchanged).
Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
This commit is contained in:
@@ -52,7 +52,10 @@ mem/B # For terms like "virtio-mem"
|
||||
memdisk/B
|
||||
MDEV/AB
|
||||
NEMU/AB
|
||||
NFD/AB # Node Feature Discovery
|
||||
NIC/AB
|
||||
nodeSelector/B # Kubernetes RuntimeClass scheduling field
|
||||
nodeSelectors/B
|
||||
nv/AB # NVIDIA abbreviation (lowercase)
|
||||
NVDIMM/AB
|
||||
OCI/AB
|
||||
@@ -74,15 +77,20 @@ QEMU/AB
|
||||
RBAC/AB
|
||||
RDMA/AB
|
||||
RNG/AB
|
||||
RuntimeClass/B # Kubernetes resource (node.k8s.io)
|
||||
RuntimeClasses/B
|
||||
SaaS/B # Software as a Service
|
||||
SCSI/AB
|
||||
SDK/AB
|
||||
seccomp # secure computing mode
|
||||
SHA/AB
|
||||
SEL/AB # IBM Secure Execution for Linux
|
||||
SPDX/AB
|
||||
SRIOV/AB
|
||||
SEV-SNP/B # AMD Secure Encrypted Virtualization - Secure Nested Paging
|
||||
SVG/AB
|
||||
TBD/AB
|
||||
TEE/AB # Trusted Execution Environment
|
||||
TOC/AB
|
||||
TOML/AB
|
||||
TTY/AB
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
409
|
||||
417
|
||||
ACPI/AB
|
||||
ACS/AB
|
||||
API/AB
|
||||
@@ -93,6 +93,7 @@ Mellanox/B
|
||||
Minikube/B
|
||||
MonitorTest/A
|
||||
NEMU/AB
|
||||
NFD/AB
|
||||
NIC/AB
|
||||
NVDIMM/AB
|
||||
NVIDIA/A
|
||||
@@ -134,10 +135,14 @@ RBAC/AB
|
||||
RDMA/AB
|
||||
RHEL/B
|
||||
RNG/AB
|
||||
RuntimeClass/B
|
||||
RuntimeClasses/B
|
||||
Rustlang/B
|
||||
SCSI/AB
|
||||
SDK/AB
|
||||
SEL/AB
|
||||
SELinux/B
|
||||
SEV-SNP/B
|
||||
SHA/AB
|
||||
SLES/B
|
||||
SPDX/AB
|
||||
@@ -153,6 +158,7 @@ Submodule/A
|
||||
Sysbench/B
|
||||
TBD/AB
|
||||
TDX
|
||||
TEE/AB
|
||||
TOC/AB
|
||||
TOML/AB
|
||||
TTY/AB
|
||||
@@ -306,6 +312,8 @@ nack/AB
|
||||
namespace/ABCD
|
||||
netlink
|
||||
netns/AB
|
||||
nodeSelector/B
|
||||
nodeSelectors/B
|
||||
nv/AB
|
||||
nvidia/A
|
||||
onwards
|
||||
|
||||
@@ -229,6 +229,7 @@ shims:
|
||||
agent:
|
||||
httpsProxy: ""
|
||||
noProxy: ""
|
||||
# Optional: set runtimeClass.nodeSelector to pin TEE to specific nodes (always applied). If unset, NFD TEE labels are auto-injected when NFD is detected.
|
||||
|
||||
# Default shim per architecture
|
||||
defaultShim:
|
||||
@@ -311,8 +312,8 @@ helm install kata-deploy oci://ghcr.io/kata-containers/kata-deploy-charts/kata-d
|
||||
Includes:
|
||||
- `qemu-snp` - AMD SEV-SNP (amd64)
|
||||
- `qemu-tdx` - Intel TDX (amd64)
|
||||
- `qemu-se` - IBM Secure Execution (s390x)
|
||||
- `qemu-se-runtime-rs` - IBM Secure Execution Rust runtime (s390x)
|
||||
- `qemu-se` - IBM Secure Execution for Linux (SEL) (s390x)
|
||||
- `qemu-se-runtime-rs` - IBM Secure Execution for Linux (SEL) Rust runtime (s390x)
|
||||
- `qemu-cca` - Arm Confidential Compute Architecture (arm64)
|
||||
- `qemu-coco-dev` - Confidential Containers development (amd64, s390x)
|
||||
- `qemu-coco-dev-runtime-rs` - Confidential Containers development Rust runtime (amd64, s390x)
|
||||
@@ -334,6 +335,27 @@ Includes:
|
||||
|
||||
**Note**: These example files are located in the chart directory. When installing from the OCI registry, you'll need to download them separately or clone the repository to access them.
|
||||
|
||||
### RuntimeClass Node Selectors for TEE Shims
|
||||
|
||||
**Manual configuration:** Any `nodeSelector` you set under `shims.<shim>.runtimeClass.nodeSelector`
|
||||
is **always applied** to that shim's RuntimeClass, whether or not NFD is present. Use this when
|
||||
you want to pin TEE workloads to specific nodes (e.g. without NFD, or with custom labels).
|
||||
|
||||
**Auto-inject when NFD is present:** If you do *not* set a `runtimeClass.nodeSelector` for a
|
||||
TEE shim, the chart can **automatically inject** NFD-based labels when NFD is detected in the
|
||||
cluster (deployed by this chart with `node-feature-discovery.enabled=true` or found externally):
|
||||
- AMD SEV-SNP shims: `amd.feature.node.kubernetes.io/snp: "true"`
|
||||
- Intel TDX shims: `intel.feature.node.kubernetes.io/tdx: "true"`
|
||||
- IBM Secure Execution for Linux (SEL) shims (s390x): `feature.node.kubernetes.io/cpu-security.se.enabled: "true"`
|
||||
|
||||
The chart uses Helm's `lookup` function to detect NFD (by looking for the
|
||||
`node-feature-discovery-worker` DaemonSet). Auto-inject only runs when NFD is detected and
|
||||
no manual `runtimeClass.nodeSelector` is set for that shim.
|
||||
|
||||
**Note**: NFD detection requires cluster access. During `helm template` (dry-run without a
|
||||
cluster), external NFD is not seen, so auto-injected labels are not added. Manual
|
||||
`runtimeClass.nodeSelector` values are still applied in all cases.
|
||||
|
||||
## `RuntimeClass` Management
|
||||
|
||||
**NEW**: Starting with Kata Containers v3.23.0, `runtimeClasses` are managed by
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
{{- $multiInstallSuffix := .Values.env.multiInstallSuffix }}
|
||||
{{- $createDefaultRC := .Values.runtimeClasses.createDefault }}
|
||||
{{- $defaultRCName := .Values.runtimeClasses.defaultName }}
|
||||
{{- $nfdEnabled := index .Values "node-feature-discovery" "enabled" | default false }}
|
||||
{{- /* Detect if NFD is installed externally (returns namespace or empty string) */ -}}
|
||||
{{- $externalNFDNamespace := include "kata-deploy.detectExistingNFD" . | trim -}}
|
||||
{{- /* Apply NFD TEE nodeSelectors only for TEE shims when NFD is detected (managed by us or external) */ -}}
|
||||
{{- $useShimNodeSelectors := or $nfdEnabled (ne $externalNFDNamespace "") -}}
|
||||
|
||||
{{- /* Get enabled shims from structured config using null-aware logic */ -}}
|
||||
{{- $disableAll := .Values.shims.disableAll | default false -}}
|
||||
@@ -79,10 +84,25 @@ overhead:
|
||||
scheduling:
|
||||
nodeSelector:
|
||||
katacontainers.io/kata-runtime: "true"
|
||||
{{- /* Apply manual nodeSelectors when present (always). Auto-inject NFD TEE labels only when NFD is detected and no manual runtimeClass.nodeSelector exists. */ -}}
|
||||
{{- $isSeShim := or (hasSuffix "-se" $shim) (hasSuffix "-se-runtime-rs" $shim) -}}
|
||||
{{- $isTeeShim := or (contains "snp" $shim) (contains "tdx" $shim) $isSeShim -}}
|
||||
{{- $isPureTeeShim := and $isTeeShim (not (contains "nvidia-gpu" $shim)) -}}
|
||||
{{- if and $shimConfig.runtimeClass $shimConfig.runtimeClass.nodeSelector }}
|
||||
{{- range $key, $value := $shimConfig.runtimeClass.nodeSelector }}
|
||||
{{ $key }}: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- else if and $useShimNodeSelectors $isPureTeeShim }}
|
||||
{{- /* Auto-inject NFD TEE labels for pure TEE shims when NFD is detected and no manual nodeSelector in values */ -}}
|
||||
{{- if contains "snp" $shim }}
|
||||
amd.feature.node.kubernetes.io/snp: "true"
|
||||
{{- end }}
|
||||
{{- if contains "tdx" $shim }}
|
||||
intel.feature.node.kubernetes.io/tdx: "true"
|
||||
{{- end }}
|
||||
{{- if $isSeShim }}
|
||||
feature.node.kubernetes.io/cpu-security.se.enabled: "true"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -17,6 +17,7 @@ shims:
|
||||
disableAll: true
|
||||
|
||||
# Enable TEE shims (qemu-snp, qemu-snp-runtime-rs, qemu-tdx, qemu-tdx-runtime-rs, qemu-se, qemu-se-runtime-rs, qemu-cca, qemu-coco-dev, qemu-coco-dev-runtime-rs)
|
||||
# NFD TEE labels (snp, tdx, se) are auto-injected into RuntimeClasses when NFD is detected; no need to set nodeSelector here.
|
||||
qemu-snp:
|
||||
enabled: true
|
||||
supportedArches:
|
||||
|
||||
Reference in New Issue
Block a user