From c6f90de4c4c8abf85ade8d1508c883fae3e6ad1f Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Tue, 14 Jul 2026 18:28:28 +0000 Subject: [PATCH] nvidia/runtime-rs: use EROFS without virtio-fs Disable filesystem sharing for the non-confidential NVIDIA runtime-rs handler and use the EROFS snapshotter as its Kubernetes image-layer transport. This moves the runtime class toward a guest-owned storage model instead of relying on virtio-fs for container image layers and writable cache volumes. Configure kata-deploy's NVIDIA GPU values to install EROFS and select it for the qemu-nvidia-gpu-runtime-rs Kubernetes handler. Use memory-backed writable layers and dm-verity for lower-layer integrity. Adjust the NVIDIA GPU Kubernetes CI matrix so the non-confidential runtime-rs job exercises EROFS, while the Go and TEE jobs keep their existing snapshotter choices. Keep the Docker smoke test path on virtio-fs. The EROFS setup is targeted at the Kubernetes runtime-rs handler, where containerd can use the EROFS snapshotter for image layers. Adjust the runtime-rs NIM test selection and manifests closer to the TEE case, where filesystem sharing is already disabled and cache storage is guest-owned and ephemeral. Signed-off-by: Manuel Huber Assisted-by: OpenAI Codex --- .../run-k8s-tests-on-nvidia-gpu.yaml | 13 ++- ...uration-qemu-nvidia-gpu-runtime-rs.toml.in | 2 +- tests/common.bash | 2 +- tests/integration/kubernetes/gha-run.sh | 2 +- .../kubernetes/k8s-nvidia-nim-service.bats | 12 +- .../kubernetes/k8s-nvidia-nim.bats | 9 ++ ...m-llama-3-1-8b-instruct-runtime-rs.yaml.in | 92 ++++++++++++++++ ...3-2-1b-instruct-service-runtime-rs.yaml.in | 49 +++++++++ ...ma-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in | 103 ++++++++++++++++++ .../try-kata-nvidia-gpu.values.yaml | 24 +++- 10 files changed, 293 insertions(+), 15 deletions(-) create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in create mode 100644 tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in diff --git a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml index e465096b53..aef09019ce 100644 --- a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml +++ b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml @@ -45,10 +45,10 @@ jobs: fail-fast: false matrix: environment: [ - { name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false }, - { name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false }, - { name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true }, - { name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true }, + { name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false, snapshotter: "" }, + { name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false, snapshotter: "erofs" }, + { name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" }, + { name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }} @@ -62,8 +62,9 @@ jobs: KATA_HYPERVISOR: ${{ matrix.environment.vmm }} KUBERNETES: kubeadm KBS: ${{ matrix.environment.coco && 'true' || 'false' }} - SNAPSHOTTER: ${{ matrix.environment.coco && 'nydus' || '' }} - USE_EXPERIMENTAL_SNAPSHOTTER_SETUP: ${{ matrix.environment.coco && 'true' || 'false' }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter }} + EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.snapshotter == 'erofs' && 'memory' || '' }} + EROFS_DMVERITY: ${{ matrix.environment.snapshotter == 'erofs' && 'dmverity' || '' }} K8S_TEST_HOST_TYPE: baremetal steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in index 86f4e6e595..232bcfe906 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in @@ -195,7 +195,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-fs-nydus # - none -shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" +shared_fs = "none" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" diff --git a/tests/common.bash b/tests/common.bash index 708cbc7ae0..50dfcb1e6f 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -812,7 +812,7 @@ function enabling_hypervisor() { # shared_fs=none + EROFS snapshotter path is covered by Kubernetes CI instead. function configure_nvidia_runtime_rs_shared_fs_dropin() { case "${KATA_HYPERVISOR:-}" in - qemu-nvidia-cpu-runtime-rs) ;; + qemu-nvidia-cpu-runtime-rs|qemu-nvidia-gpu-runtime-rs) ;; *) return 0 ;; esac diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index fa6059e511..607c0511ce 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -200,7 +200,7 @@ function deploy_kata() { EXPERIMENTAL_FORCE_GUEST_PULL=false fi - if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-cpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then + if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-cpu-runtime-rs" || "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then SNAPSHOTTER="erofs" fi diff --git a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats index cb6b5e15ff..5f745465a6 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats @@ -22,7 +22,9 @@ NIM_SERVICE_NAME="meta-llama-3-2-1b-instruct" export NIM_SERVICE_NAME POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=600s -[[ "${TEE}" = "true" ]] && POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s +if [[ "${TEE}" = "true" ]] || is_runtime_rs; then + POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s +fi export POD_READY_TIMEOUT_LLAMA_3_2_1B=${POD_READY_TIMEOUT_LLAMA_3_2_1B:-${POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED}} export LOCAL_NIM_CACHE_LLAMA_3_2_1B="${LOCAL_NIM_CACHE_LLAMA_3_2_1B:-${LOCAL_NIM_CACHE:-/opt/nim/.cache}-llama-3-2-1b}" @@ -167,8 +169,12 @@ setup() { # Same pattern as k8s-nvidia-nim.bats: choose manifest by TEE; each YAML has literal secret names. local tee_suffix="" [[ "${TEE}" = "true" ]] && tee_suffix="-tee" - export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml.in" - export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml" + local runtime_rs_suffix="" + if is_runtime_rs && [[ "${TEE}" != "true" ]]; then + runtime_rs_suffix="-runtime-rs" + fi + export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml.in" + export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml" if [[ "${TEE}" = "true" ]]; then setup_kbs_credentials diff --git a/tests/integration/kubernetes/k8s-nvidia-nim.bats b/tests/integration/kubernetes/k8s-nvidia-nim.bats index f9d1d0b4d8..62148e9923 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim.bats @@ -28,6 +28,8 @@ POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=600s if [[ "${TEE}" = "true" ]]; then POD_NAME_EMBEDQA="${POD_NAME_EMBEDQA}-tee" POD_NAME_INSTRUCT="${POD_NAME_INSTRUCT}-tee" +fi +if [[ "${TEE}" = "true" ]] || is_runtime_rs; then POD_READY_TIMEOUT_EMBEDQA_PREDEFINED=1000s POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=1000s fi @@ -179,6 +181,13 @@ setup_file() { export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml.in" export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml" + if is_runtime_rs && [[ "${TEE}" != "true" ]]; then + export POD_INSTRUCT_YAML_IN="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml.in" + export POD_INSTRUCT_YAML="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml" + export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml.in" + export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml" + fi + dpkg -s jq >/dev/null 2>&1 || sudo apt -y install jq setup_langchain_flow diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in new file mode 100644 index 0000000000..33d0217239 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in @@ -0,0 +1,92 @@ +# Copyright (c) 2025 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: ${POD_NAME_INSTRUCT} + labels: + app: ${POD_NAME_INSTRUCT} +spec: + # block-plain emptyDir volumes are formatted and mounted in the guest. + # Set fsGroup so the agent makes the cache writable by the NIM user. + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + restartPolicy: Never + runtimeClassName: kata + imagePullSecrets: + - name: ngc-secret-instruct + containers: + - name: ${POD_NAME_INSTRUCT} + image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1 + # Ports exposed by the container: + ports: + - containerPort: 8000 + name: http-openai + livenessProbe: + httpGet: + path: /v1/health/live + port: http-openai + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v1/health/ready + port: http-openai + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /v1/health/ready + port: http-openai + initialDelaySeconds: 360 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 30 + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key-instruct + key: api-key + # GPU resource limit (for NVIDIA GPU) + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "16" + memory: "48Gi" + volumeMounts: + - name: nim-cache + mountPath: /opt/nim/.cache + + volumes: + - name: nim-cache + emptyDir: + sizeLimit: 64Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-instruct +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-instruct +type: Opaque +data: + api-key: "${NGC_API_KEY_BASE64}" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in new file mode 100644 index 0000000000..61128f272c --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in @@ -0,0 +1,49 @@ +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: apps.nvidia.com/v1alpha1 +kind: NIMService +metadata: + name: ${NIM_SERVICE_NAME} +spec: + image: + repository: nvcr.io/nim/meta/llama-3.2-1b-instruct + tag: "1.12.0" + pullPolicy: IfNotPresent + pullSecrets: + - ngc-secret-llama-3-2-1b + authSecret: ngc-api-key-llama-3-2-1b + storage: + emptyDir: + sizeLimit: 10Gi + replicas: 1 + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "8" + memory: "16Gi" + expose: + service: + type: ClusterIP + port: 8000 + runtimeClassName: kata + userID: 1000 + groupID: 1000 +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-llama-3-2-1b +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-llama-3-2-1b +type: Opaque +data: + NGC_API_KEY: "${NGC_API_KEY_BASE64}" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in new file mode 100644 index 0000000000..846199e42d --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in @@ -0,0 +1,103 @@ +# Copyright (c) 2025 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: ${POD_NAME_EMBEDQA} + labels: + app: ${POD_NAME_EMBEDQA} +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + restartPolicy: Always + runtimeClassName: kata + serviceAccountName: default + imagePullSecrets: + - name: ngc-secret-embedqa + containers: + - name: ${POD_NAME_EMBEDQA} + image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1 + imagePullPolicy: IfNotPresent + env: + - name: NIM_CACHE_PATH + value: "/opt/nim/.cache" + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key-embedqa + key: api-key + - name: NIM_HTTP_API_PORT + value: "8000" + - name: NIM_JSONL_LOGGING + value: "1" + - name: NIM_LOG_LEVEL + value: "INFO" + ports: + - containerPort: 8000 + name: http + + livenessProbe: + httpGet: + path: /v1/health/live + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + + startupProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 180 + + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "16" + memory: "24Gi" + + volumeMounts: + - name: nim-cache + mountPath: /opt/nim/.cache + + volumes: + - name: nim-cache + emptyDir: + sizeLimit: 40Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-embedqa +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-embedqa +type: Opaque +data: + api-key: "${NGC_API_KEY_BASE64}" diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml index 125ba85166..c5a9f6d698 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml @@ -14,7 +14,26 @@ debug: false deploymentMode: job snapshotter: - setup: ["nydus"] + setup: ["nydus", "erofs"] + +# EROFS fs-verity requires the backing filesystem to support fs-verity, and we +# cannot guarantee that on an arbitrary node. Rather than gate this profile on a +# host feature we do not control, we disable fs-verity so it works on the widest +# possible range of setups; we take that hit deliberately to keep the profile +# generic. Combined with memory-backed EROFS rw layers, ship this as the default +# containerd erofs snapshotter drop-in for the NVIDIA GPU runtime-rs handler so +# it does not need to be injected out-of-band. It is loaded after kata-deploy's +# generated config, so it overrides the built-in enable_fsverity default. +# +# NOTE: This drop-in targets containerd's built-in EROFS snapshotter, which is +# only available on containerd >= 2.2.0 (config version 3, using the +# auto-imported conf.d drop-in mechanism). This is the same minimum kata-deploy +# enforces for the EROFS snapshotter. +containerd: + userDropIn: | + [plugins.'io.containerd.snapshotter.v1.erofs'] + enable_fsverity = false + default_size = "0" # Enable NVIDIA GPU shims # Disable all shims at once, then enable only the ones we need @@ -41,7 +60,7 @@ shims: - amd64 allowedHypervisorAnnotations: [] containerd: - snapshotter: "" + snapshotter: "erofs" runtimeClass: # This label is automatically added by gpu-operator. Override it # if you want to use a different label. @@ -144,4 +163,3 @@ runtimeClasses: enabled: true createDefault: false defaultName: "kata" -