github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-25 14:18:54 +00:00
Code Issues Projects Releases Wiki Activity

nvidia/runtime-rs: use EROFS without virtio-fs

Browse Source 
Disable filesystem sharing for the non-confidential NVIDIA runtime-rs
handler and use the EROFS snapshotter as its Kubernetes image-layer
transport. This moves the runtime class toward a guest-owned storage
model instead of relying on virtio-fs for container image layers and
writable cache volumes.

Configure kata-deploy's NVIDIA GPU values to install EROFS and select
it for the qemu-nvidia-gpu-runtime-rs Kubernetes handler. Use
memory-backed writable layers and dm-verity for lower-layer integrity.

Adjust the NVIDIA GPU Kubernetes CI matrix so the non-confidential
runtime-rs job exercises EROFS, while the Go and TEE jobs keep their
existing snapshotter choices.

Keep the Docker smoke test path on virtio-fs. The EROFS setup is
targeted at the Kubernetes runtime-rs handler, where containerd can use
the EROFS snapshotter for image layers.

Adjust the runtime-rs NIM test selection and manifests closer to the
TEE case, where filesystem sharing is already disabled and cache
storage is guest-owned and ephemeral.

Signed-off-by: Manuel Huber <manuelh@nvidia.com>
Assisted-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Manuel Huber
2026-07-14 18:28:28 +00:00
committed by Fabiano Fidêncio
parent 34a1e12c10
commit c6f90de4c4
10 changed files with 293 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml vendored
View File

@@ -45,10 +45,10 @@ jobs:
fail-fast: false
matrix:
environment: [
{ name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false },
{ name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false },
{ name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true },
{ name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true },
{ name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false, snapshotter: "" },
{ name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false, snapshotter: "erofs" },
{ name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" },
{ name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" },
]
concurrency:
group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }}
@@ -62,8 +62,9 @@ jobs:
KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
KUBERNETES: kubeadm
KBS: ${{ matrix.environment.coco && 'true' || 'false' }}
SNAPSHOTTER: ${{ matrix.environment.coco && 'nydus' || '' }}
USE_EXPERIMENTAL_SNAPSHOTTER_SETUP: ${{ matrix.environment.coco && 'true' || 'false' }}
SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.snapshotter == 'erofs' && 'memory' || '' }}
EROFS_DMVERITY: ${{ matrix.environment.snapshotter == 'erofs' && 'dmverity' || '' }}
K8S_TEST_HOST_TYPE: baremetal
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

2
src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
View File

@@ -195,7 +195,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@
# - virtio-fs (default)
# - virtio-fs-nydus
# - none
shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
shared_fs = "none"
# Path to vhost-user-fs daemon.
virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"

2
tests/common.bash
View File

@@ -812,7 +812,7 @@ function enabling_hypervisor() {
# shared_fs=none + EROFS snapshotter path is covered by Kubernetes CI instead.
function configure_nvidia_runtime_rs_shared_fs_dropin() {
case "${KATA_HYPERVISOR:-}" in
qemu-nvidia-cpu-runtime-rs) ;;
qemu-nvidia-cpu-runtime-rs|qemu-nvidia-gpu-runtime-rs) ;;
*) return 0 ;;
esac

2
tests/integration/kubernetes/gha-run.sh
View File

@@ -200,7 +200,7 @@ function deploy_kata() {
EXPERIMENTAL_FORCE_GUEST_PULL=false
fi
if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-cpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then
if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-cpu-runtime-rs" || "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then
SNAPSHOTTER="erofs"
fi

12
tests/integration/kubernetes/k8s-nvidia-nim-service.bats
View File

@@ -22,7 +22,9 @@ NIM_SERVICE_NAME="meta-llama-3-2-1b-instruct"
export NIM_SERVICE_NAME
POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=600s
[[ "${TEE}" = "true" ]] && POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s
if [[ "${TEE}" = "true" ]] || is_runtime_rs; then
POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s
fi
export POD_READY_TIMEOUT_LLAMA_3_2_1B=${POD_READY_TIMEOUT_LLAMA_3_2_1B:-${POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED}}
export LOCAL_NIM_CACHE_LLAMA_3_2_1B="${LOCAL_NIM_CACHE_LLAMA_3_2_1B:-${LOCAL_NIM_CACHE:-/opt/nim/.cache}-llama-3-2-1b}"
@@ -167,8 +169,12 @@ setup() {
# Same pattern as k8s-nvidia-nim.bats: choose manifest by TEE; each YAML has literal secret names.
local tee_suffix=""
[[ "${TEE}" = "true" ]] && tee_suffix="-tee"
export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml.in"
export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml"
local runtime_rs_suffix=""
if is_runtime_rs && [[ "${TEE}" != "true" ]]; then
runtime_rs_suffix="-runtime-rs"
fi
export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml.in"
export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml"
if [[ "${TEE}" = "true" ]]; then
setup_kbs_credentials

9
tests/integration/kubernetes/k8s-nvidia-nim.bats
View File

@@ -28,6 +28,8 @@ POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=600s
if [[ "${TEE}" = "true" ]]; then
POD_NAME_EMBEDQA="${POD_NAME_EMBEDQA}-tee"
POD_NAME_INSTRUCT="${POD_NAME_INSTRUCT}-tee"
fi
if [[ "${TEE}" = "true" ]] || is_runtime_rs; then
POD_READY_TIMEOUT_EMBEDQA_PREDEFINED=1000s
POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=1000s
fi
@@ -179,6 +181,13 @@ setup_file() {
export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml.in"
export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml"
if is_runtime_rs && [[ "${TEE}" != "true" ]]; then
export POD_INSTRUCT_YAML_IN="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml.in"
export POD_INSTRUCT_YAML="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml"
export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml.in"
export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml"
fi
dpkg -s jq >/dev/null 2>&1 || sudo apt -y install jq
setup_langchain_flow

92
tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in Normal file
View File

@@ -0,0 +1,92 @@
# Copyright (c) 2025 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
---
apiVersion: v1
kind: Pod
metadata:
name: ${POD_NAME_INSTRUCT}
labels:
app: ${POD_NAME_INSTRUCT}
spec:
# block-plain emptyDir volumes are formatted and mounted in the guest.
# Set fsGroup so the agent makes the cache writable by the NIM user.
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
restartPolicy: Never
runtimeClassName: kata
imagePullSecrets:
- name: ngc-secret-instruct
containers:
- name: ${POD_NAME_INSTRUCT}
image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1
# Ports exposed by the container:
ports:
- containerPort: 8000
name: http-openai
livenessProbe:
httpGet:
path: /v1/health/live
port: http-openai
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /v1/health/ready
port: http-openai
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
startupProbe:
httpGet:
path: /v1/health/ready
port: http-openai
initialDelaySeconds: 360
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 30
env:
- name: NGC_API_KEY
valueFrom:
secretKeyRef:
name: ngc-api-key-instruct
key: api-key
# GPU resource limit (for NVIDIA GPU)
resources:
limits:
nvidia.com/pgpu: "1"
cpu: "16"
memory: "48Gi"
volumeMounts:
- name: nim-cache
mountPath: /opt/nim/.cache
volumes:
- name: nim-cache
emptyDir:
sizeLimit: 64Gi
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-secret-instruct
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: ${DOCKER_CONFIG_JSON}
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-api-key-instruct
type: Opaque
data:
api-key: "${NGC_API_KEY_BASE64}"

49
tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in Normal file
View File

@@ -0,0 +1,49 @@
# Copyright (c) 2026 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
---
apiVersion: apps.nvidia.com/v1alpha1
kind: NIMService
metadata:
name: ${NIM_SERVICE_NAME}
spec:
image:
repository: nvcr.io/nim/meta/llama-3.2-1b-instruct
tag: "1.12.0"
pullPolicy: IfNotPresent
pullSecrets:
- ngc-secret-llama-3-2-1b
authSecret: ngc-api-key-llama-3-2-1b
storage:
emptyDir:
sizeLimit: 10Gi
replicas: 1
resources:
limits:
nvidia.com/pgpu: "1"
cpu: "8"
memory: "16Gi"
expose:
service:
type: ClusterIP
port: 8000
runtimeClassName: kata
userID: 1000
groupID: 1000
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-secret-llama-3-2-1b
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: ${DOCKER_CONFIG_JSON}
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-api-key-llama-3-2-1b
type: Opaque
data:
NGC_API_KEY: "${NGC_API_KEY_BASE64}"

103
tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in Normal file
View File

@@ -0,0 +1,103 @@
# Copyright (c) 2025 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
---
apiVersion: v1
kind: Pod
metadata:
name: ${POD_NAME_EMBEDQA}
labels:
app: ${POD_NAME_EMBEDQA}
spec:
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
restartPolicy: Always
runtimeClassName: kata
serviceAccountName: default
imagePullSecrets:
- name: ngc-secret-embedqa
containers:
- name: ${POD_NAME_EMBEDQA}
image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1
imagePullPolicy: IfNotPresent
env:
- name: NIM_CACHE_PATH
value: "/opt/nim/.cache"
- name: NGC_API_KEY
valueFrom:
secretKeyRef:
name: ngc-api-key-embedqa
key: api-key
- name: NIM_HTTP_API_PORT
value: "8000"
- name: NIM_JSONL_LOGGING
value: "1"
- name: NIM_LOG_LEVEL
value: "INFO"
ports:
- containerPort: 8000
name: http
livenessProbe:
httpGet:
path: /v1/health/live
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /v1/health/ready
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 10
successThreshold: 1
failureThreshold: 3
startupProbe:
httpGet:
path: /v1/health/ready
port: 8000
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 180
resources:
limits:
nvidia.com/pgpu: "1"
cpu: "16"
memory: "24Gi"
volumeMounts:
- name: nim-cache
mountPath: /opt/nim/.cache
volumes:
- name: nim-cache
emptyDir:
sizeLimit: 40Gi
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-secret-embedqa
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: ${DOCKER_CONFIG_JSON}
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-api-key-embedqa
type: Opaque
data:
api-key: "${NGC_API_KEY_BASE64}"

24
tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml
View File

@@ -14,7 +14,26 @@ debug: false
deploymentMode: job
snapshotter:
setup: ["nydus"]
setup: ["nydus", "erofs"]
# EROFS fs-verity requires the backing filesystem to support fs-verity, and we
# cannot guarantee that on an arbitrary node. Rather than gate this profile on a
# host feature we do not control, we disable fs-verity so it works on the widest
# possible range of setups; we take that hit deliberately to keep the profile
# generic. Combined with memory-backed EROFS rw layers, ship this as the default
# containerd erofs snapshotter drop-in for the NVIDIA GPU runtime-rs handler so
# it does not need to be injected out-of-band. It is loaded after kata-deploy's
# generated config, so it overrides the built-in enable_fsverity default.
#
# NOTE: This drop-in targets containerd's built-in EROFS snapshotter, which is
# only available on containerd >= 2.2.0 (config version 3, using the
# auto-imported conf.d drop-in mechanism). This is the same minimum kata-deploy
# enforces for the EROFS snapshotter.
containerd:
userDropIn: |
[plugins.'io.containerd.snapshotter.v1.erofs']
enable_fsverity = false
default_size = "0"
# Enable NVIDIA GPU shims
# Disable all shims at once, then enable only the ones we need
@@ -41,7 +60,7 @@ shims:
- amd64
allowedHypervisorAnnotations: []
containerd:
snapshotter: ""
snapshotter: "erofs"
runtimeClass:
# This label is automatically added by gpu-operator. Override it
# if you want to use a different label.
@@ -144,4 +163,3 @@ runtimeClasses:
enabled: true
createDefault: false
defaultName: "kata"