nvidia: test runtime-rs with EROFS

Run the non-confidential NVIDIA runtime-rs CI path with the EROFS snapshotter. Use a memory-backed writable layer and dm-verity for lower-layer integrity. Disable filesystem sharing for the runtime-rs handler and use block-plain emptyDirs for NIM caches. Keep the Go runtime on its existing hostPath manifests and select the emptyDir variants for non-TEE runtime-rs tests. Signed-off-by: Manuel Huber <manuelh@nvidia.com> Assisted-by: OpenAI Codex <codex@openai.com>
2026-07-11 09:07:49 +00:00 · 2026-07-06 17:38:42 +00:00
parent 4adabbd28a
commit 5a5e09ecac
12 changed files with 298 additions and 14 deletions
--- a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml
+++ b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml
@@ -45,10 +45,10 @@ jobs:
      fail-fast: false
      matrix:
        environment: [
-          { name: nvidia-gpu,                  vmm: qemu-nvidia-gpu,                runner: amd64-nvidia-a100,     coco: false },
-          { name: nvidia-gpu-runtime-rs,       vmm: qemu-nvidia-gpu-runtime-rs,     runner: amd64-nvidia-a100,     coco: false },
-          { name: nvidia-gpu-snp,              vmm: qemu-nvidia-gpu-snp,            runner: amd64-nvidia-h100-snp, coco: true },
-          { name: nvidia-gpu-snp-runtime-rs,   vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true },
+          { name: nvidia-gpu,                  vmm: qemu-nvidia-gpu,                runner: amd64-nvidia-a100,     coco: false, snapshotter: "" },
+          { name: nvidia-gpu-runtime-rs,       vmm: qemu-nvidia-gpu-runtime-rs,     runner: amd64-nvidia-a100,     coco: false, snapshotter: "erofs" },
+          { name: nvidia-gpu-snp,              vmm: qemu-nvidia-gpu-snp,            runner: amd64-nvidia-h100-snp, coco: true,  snapshotter: "nydus" },
+          { name: nvidia-gpu-snp-runtime-rs,   vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true,  snapshotter: "nydus" },
        ]
    concurrency:
      group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }}
@@ -62,8 +62,9 @@ jobs:
      KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
      KUBERNETES: kubeadm
      KBS: ${{ matrix.environment.coco && 'true' || 'false' }}
-      SNAPSHOTTER: ${{ matrix.environment.coco && 'nydus' || '' }}
-      USE_EXPERIMENTAL_SNAPSHOTTER_SETUP: ${{ matrix.environment.coco && 'true' || 'false' }}
+      SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
+      EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.snapshotter == 'erofs' && 'memory' || '' }}
+      EROFS_DMVERITY: ${{ matrix.environment.snapshotter == 'erofs' && 'dmverity' || '' }}
      K8S_TEST_HOST_TYPE: baremetal
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
--- a/src/runtime-rs/Makefile
+++ b/src/runtime-rs/Makefile
@@ -202,6 +202,7 @@ DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"kernel_params\", \"kernel_veri
 DEFDISABLEGUESTSECCOMP := true
 DEFEMPTYDIRMODE := shared-fs
 DEFEMPTYDIRMODE_COCO := block-encrypted
+DEFEMPTYDIRMODE_NV := block-plain
 ##VAR DEFAULTEXPFEATURES=[features] Default experimental features enabled
 DEFAULTEXPFEATURES := []
 DEFDISABLESELINUX := false
@@ -722,6 +723,7 @@ USER_VARS += DEFNETWORKMODEL_QEMU
 USER_VARS += DEFNETWORKMODEL_FC
 USER_VARS += DEFEMPTYDIRMODE
 USER_VARS += DEFEMPTYDIRMODE_COCO
+USER_VARS += DEFEMPTYDIRMODE_NV
 USER_VARS += DEFDISABLEGUESTSECCOMP
 USER_VARS += DEFDISABLESELINUX
 USER_VARS += DEFDISABLEGUESTSELINUX
--- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
+++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
@@ -195,7 +195,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@
 #   - virtio-fs (default)
 #   - virtio-fs-nydus
 #   - none
-shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
+shared_fs = "none"

 # Path to vhost-user-fs daemon.
 virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
@@ -816,7 +816,7 @@ vfio_mode = "@DEFVFIOMODE_NV@"
 #   - block-plain
 #     Plugs a block device to be mounted directly in the guest.
 #
-emptydir_mode = "@DEFEMPTYDIRMODE@"
+emptydir_mode = "@DEFEMPTYDIRMODE_NV@"

 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
--- a/tests/gha-run-k8s-common.sh
+++ b/tests/gha-run-k8s-common.sh
@@ -874,6 +874,11 @@ function helm_helper() {
 			HELM_CONTAINERD_USER_DROP_IN="[plugins.'io.containerd.snapshotter.v1.erofs']"$'\n'
 			HELM_CONTAINERD_USER_DROP_IN+="  default_size = \"${erofs_default_size}\""

+			# NVIDIA CI hosts are not configured with ext4 fs-verity support.
+			if [[ "${KATA_HYPERVISOR:-}" == *"nvidia-gpu"* ]]; then
+				HELM_CONTAINERD_USER_DROP_IN+=$'\n'"  enable_fsverity = false"
+			fi
+
 			HELM_CONTAINERD_USER_DROP_IN="${HELM_CONTAINERD_USER_DROP_IN}" \
 				yq -i '.containerd.userDropIn = strenv(HELM_CONTAINERD_USER_DROP_IN)' "${values_yaml}"

--- a/tests/integration/docker/gha-run.sh
+++ b/tests/integration/docker/gha-run.sh
@@ -32,6 +32,19 @@ function enable_kata_debug() {
 	sudo sed -i -e 's/^#\?[[:space:]]*enable_debug[[:space:]]*=.*/enable_debug = true/g' "${cfg}"
 }

+function configure_docker_nvidia_runtime_rs() {
+	[[ "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] || return 0
+
+	local -r cfg="${KATA_CONFIG_PATH:-}"
+	[[ -z "${cfg}" || ! -e "${cfg}" ]] && return 0
+
+	info "Configuring NVIDIA runtime-rs for docker smoke test in ${cfg}"
+	sudo sed -i \
+		-e 's/^#\?[[:space:]]*shared_fs[[:space:]]*=.*/shared_fs = "virtio-fs"/g' \
+		-e 's/^#\?[[:space:]]*emptydir_mode[[:space:]]*=.*/emptydir_mode = "shared-fs"/g' \
+		"${cfg}"
+}
+
 function dump_kata_debug() {
 	local -r kata_runtime="$1"

@@ -92,6 +105,7 @@ function run() {
 	info "Running docker smoke test tests using ${KATA_HYPERVISOR} hypervisor"

 	enabling_hypervisor
+	configure_docker_nvidia_runtime_rs
 	enable_kata_debug

 	info "Running docker with runc"
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -200,6 +200,10 @@ function deploy_kata() {
 		EXPERIMENTAL_FORCE_GUEST_PULL=false
 	fi

+	if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then
+		SNAPSHOTTER="erofs"
+	fi
+
 	ANNOTATIONS="default_vcpus"
 	if [[ "${KATA_HYPERVISOR}" == *azure* ]]; then
 		ANNOTATIONS="image kernel default_vcpus cc_init_data"
--- a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats
+++ b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats
@@ -22,7 +22,9 @@ NIM_SERVICE_NAME="meta-llama-3-2-1b-instruct"
 export NIM_SERVICE_NAME

 POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=600s
-[[ "${TEE}" = "true" ]] && POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s
+if [[ "${TEE}" = "true" ]] || is_runtime_rs; then
+    POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s
+fi
 export POD_READY_TIMEOUT_LLAMA_3_2_1B=${POD_READY_TIMEOUT_LLAMA_3_2_1B:-${POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED}}

 export LOCAL_NIM_CACHE_LLAMA_3_2_1B="${LOCAL_NIM_CACHE_LLAMA_3_2_1B:-${LOCAL_NIM_CACHE:-/opt/nim/.cache}-llama-3-2-1b}"
@@ -167,8 +169,12 @@ setup() {
    # Same pattern as k8s-nvidia-nim.bats: choose manifest by TEE; each YAML has literal secret names.
    local tee_suffix=""
    [[ "${TEE}" = "true" ]] && tee_suffix="-tee"
-    export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml.in"
-    export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml"
+    local runtime_rs_suffix=""
+    if is_runtime_rs && [[ "${TEE}" != "true" ]]; then
+        runtime_rs_suffix="-runtime-rs"
+    fi
+    export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml.in"
+    export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml"

    if [[ "${TEE}" = "true" ]]; then
        setup_kbs_credentials
--- a/tests/integration/kubernetes/k8s-nvidia-nim.bats
+++ b/tests/integration/kubernetes/k8s-nvidia-nim.bats
@@ -28,6 +28,8 @@ POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=600s
 if [[ "${TEE}" = "true" ]]; then
    POD_NAME_EMBEDQA="${POD_NAME_EMBEDQA}-tee"
    POD_NAME_INSTRUCT="${POD_NAME_INSTRUCT}-tee"
+fi
+if [[ "${TEE}" = "true" ]] || is_runtime_rs; then
    POD_READY_TIMEOUT_EMBEDQA_PREDEFINED=1000s
    POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=1000s
 fi
@@ -179,6 +181,13 @@ setup_file() {
    export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml.in"
    export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml"

+    if is_runtime_rs && [[ "${TEE}" != "true" ]]; then
+        export POD_INSTRUCT_YAML_IN="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml.in"
+        export POD_INSTRUCT_YAML="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml"
+        export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml.in"
+        export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml"
+    fi
+
    dpkg -s jq >/dev/null 2>&1 || sudo apt -y install jq

    setup_langchain_flow
--- a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in
+++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in
@@ -0,0 +1,92 @@
+# Copyright (c) 2025 NVIDIA Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${POD_NAME_INSTRUCT}
+  labels:
+    app: ${POD_NAME_INSTRUCT}
+spec:
+  # block-plain emptyDir volumes are formatted and mounted in the guest.
+  # Set fsGroup so the agent makes the cache writable by the NIM user.
+  securityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
+  restartPolicy: Never
+  runtimeClassName: kata
+  imagePullSecrets:
+    - name: ngc-secret-instruct
+  containers:
+  - name: ${POD_NAME_INSTRUCT}
+    image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1
+    # Ports exposed by the container:
+    ports:
+      - containerPort: 8000
+        name: http-openai
+    livenessProbe:
+      httpGet:
+        path: /v1/health/live
+        port: http-openai
+      initialDelaySeconds: 15
+      periodSeconds: 10
+      timeoutSeconds: 1
+      successThreshold: 1
+      failureThreshold: 3
+    readinessProbe:
+      httpGet:
+        path: /v1/health/ready
+        port: http-openai
+      initialDelaySeconds: 15
+      periodSeconds: 10
+      timeoutSeconds: 1
+      successThreshold: 1
+      failureThreshold: 3
+    startupProbe:
+      httpGet:
+        path: /v1/health/ready
+        port: http-openai
+      initialDelaySeconds: 360
+      periodSeconds: 10
+      timeoutSeconds: 1
+      successThreshold: 1
+      failureThreshold: 30
+    env:
+      - name: NGC_API_KEY
+        valueFrom:
+          secretKeyRef:
+            name: ngc-api-key-instruct
+            key: api-key
+    # GPU resource limit (for NVIDIA GPU)
+    resources:
+      limits:
+        nvidia.com/pgpu: "1"
+        cpu: "16"
+        memory: "48Gi"
+    volumeMounts:
+      - name: nim-cache
+        mountPath: /opt/nim/.cache
+
+  volumes:
+  - name: nim-cache
+    emptyDir:
+      sizeLimit: 64Gi
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ngc-secret-instruct
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: ${DOCKER_CONFIG_JSON}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ngc-api-key-instruct
+type: Opaque
+data:
+  api-key: "${NGC_API_KEY_BASE64}"
--- a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in
+++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in
@@ -0,0 +1,49 @@
+# Copyright (c) 2026 NVIDIA Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: apps.nvidia.com/v1alpha1
+kind: NIMService
+metadata:
+  name: ${NIM_SERVICE_NAME}
+spec:
+  image:
+    repository: nvcr.io/nim/meta/llama-3.2-1b-instruct
+    tag: "1.12.0"
+    pullPolicy: IfNotPresent
+    pullSecrets:
+      - ngc-secret-llama-3-2-1b
+  authSecret: ngc-api-key-llama-3-2-1b
+  storage:
+    emptyDir:
+      sizeLimit: 10Gi
+  replicas: 1
+  resources:
+    limits:
+      nvidia.com/pgpu: "1"
+      cpu: "8"
+      memory: "16Gi"
+  expose:
+    service:
+      type: ClusterIP
+      port: 8000
+  runtimeClassName: kata
+  userID: 1000
+  groupID: 1000
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ngc-secret-llama-3-2-1b
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: ${DOCKER_CONFIG_JSON}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ngc-api-key-llama-3-2-1b
+type: Opaque
+data:
+  NGC_API_KEY: "${NGC_API_KEY_BASE64}"
--- a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in
+++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in
@@ -0,0 +1,103 @@
+# Copyright (c) 2025 NVIDIA Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${POD_NAME_EMBEDQA}
+  labels:
+    app: ${POD_NAME_EMBEDQA}
+spec:
+  securityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
+  restartPolicy: Always
+  runtimeClassName: kata
+  serviceAccountName: default
+  imagePullSecrets:
+    - name: ngc-secret-embedqa
+  containers:
+  - name: ${POD_NAME_EMBEDQA}
+    image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1
+    imagePullPolicy: IfNotPresent
+    env:
+      - name: NIM_CACHE_PATH
+        value: "/opt/nim/.cache"
+      - name: NGC_API_KEY
+        valueFrom:
+          secretKeyRef:
+            name: ngc-api-key-embedqa
+            key: api-key
+      - name: NIM_HTTP_API_PORT
+        value: "8000"
+      - name: NIM_JSONL_LOGGING
+        value: "1"
+      - name: NIM_LOG_LEVEL
+        value: "INFO"
+    ports:
+      - containerPort: 8000
+        name: http
+
+    livenessProbe:
+      httpGet:
+        path: /v1/health/live
+        port: 8000
+      initialDelaySeconds: 15
+      periodSeconds: 10
+      timeoutSeconds: 1
+      successThreshold: 1
+      failureThreshold: 3
+
+    readinessProbe:
+      httpGet:
+        path: /v1/health/ready
+        port: 8000
+      initialDelaySeconds: 15
+      periodSeconds: 10
+      timeoutSeconds: 10
+      successThreshold: 1
+      failureThreshold: 3
+
+    startupProbe:
+      httpGet:
+        path: /v1/health/ready
+        port: 8000
+      initialDelaySeconds: 60
+      periodSeconds: 10
+      timeoutSeconds: 1
+      successThreshold: 1
+      failureThreshold: 180
+
+    resources:
+      limits:
+        nvidia.com/pgpu: "1"
+        cpu: "16"
+        memory: "24Gi"
+
+    volumeMounts:
+      - name: nim-cache
+        mountPath: /opt/nim/.cache
+
+  volumes:
+  - name: nim-cache
+    emptyDir:
+      sizeLimit: 40Gi
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ngc-secret-embedqa
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: ${DOCKER_CONFIG_JSON}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ngc-api-key-embedqa
+type: Opaque
+data:
+  api-key: "${NGC_API_KEY_BASE64}"
--- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml
+++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml
@@ -14,7 +14,7 @@ debug: false
 deploymentMode: job

 snapshotter:
-  setup: ["nydus"]
+  setup: ["nydus", "erofs"]

 # Enable NVIDIA GPU shims
 # Disable all shims at once, then enable only the ones we need
@@ -41,7 +41,7 @@ shims:
      - amd64
    allowedHypervisorAnnotations: []
    containerd:
-      snapshotter: ""
+      snapshotter: "erofs"
    runtimeClass:
      # This label is automatically added by gpu-operator. Override it
      # if you want to use a different label.
@@ -144,4 +144,3 @@ runtimeClasses:
  enabled: true
  createDefault: false
  defaultName: "kata"
-