diff --git a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml index e465096b53..df29a254b0 100644 --- a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml +++ b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml @@ -45,10 +45,10 @@ jobs: fail-fast: false matrix: environment: [ - { name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false }, - { name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false }, - { name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true }, - { name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true }, + { name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false, snapshotter: "" }, + { name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false, snapshotter: "blockfile", blockfile_scratch_size: 64G }, + { name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" }, + { name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }} @@ -62,8 +62,8 @@ jobs: KATA_HYPERVISOR: ${{ matrix.environment.vmm }} KUBERNETES: kubeadm KBS: ${{ matrix.environment.coco && 'true' || 'false' }} - SNAPSHOTTER: ${{ matrix.environment.coco && 'nydus' || '' }} - USE_EXPERIMENTAL_SNAPSHOTTER_SETUP: ${{ matrix.environment.coco && 'true' || 'false' }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter }} + BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE: ${{ matrix.environment.blockfile_scratch_size || '10G' }} K8S_TEST_HOST_TYPE: baremetal steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -105,6 +105,10 @@ jobs: timeout-minutes: 10 run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client + - name: Configure ${{ matrix.environment.snapshotter }} snapshotter + if: matrix.environment.snapshotter == 'blockfile' + run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter + - name: Deploy Kata timeout-minutes: 20 run: bash tests/integration/kubernetes/gha-run.sh deploy-kata diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 9febef291c..271f40e5cc 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -466,6 +466,9 @@ endif DEFROOTFSTYPE_NV := $(ROOTFSTYPE_EROFS) # NVIDIA profile: rootfs block driver (avoid virtio-pmem/DAX for this stack) VMROOTFSDRIVER_NV := virtio-blk-pci + # NVIDIA runtime-rs with blockfile snapshotter: pass container rootfs as a block + # device (virtio-blk-pci) with no virtio-fs shared mount. + DEFSHAREDFS_QEMU_NVIDIA_GPU_RUNTIME_RS := none # Agent client: hybrid vsock retry delay (ms); vsock path uses this as initial backoff (clamped in agent) DEFDIALTIMEOUTMS_NV := 1000 # Go qemu-nvidia-gpu dial_timeout is 1200s; runtime-rs uses ms for reconnect_timeout_ms @@ -588,6 +591,7 @@ USER_VARS += CONFIG_QEMU_SE_IN USER_VARS += CONFIG_REMOTE_IN USER_VARS += CONFIG_QEMU_COCO_DEV_IN USER_VARS += CONFIG_QEMU_NVIDIA_GPU_IN +USER_VARS += DEFSHAREDFS_QEMU_NVIDIA_GPU_RUNTIME_RS USER_VARS += DESTDIR USER_VARS += HYPERVISOR USER_VARS += USE_BUILTIN_DB diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in index 3738301bcd..c8abb5be05 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in @@ -178,7 +178,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-fs-nydus # - none -shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" +shared_fs = "@DEFSHAREDFS_QEMU_NVIDIA_GPU_RUNTIME_RS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" @@ -228,7 +228,7 @@ virtio_fs_cache = "@DEFVIRTIOFSCACHE@" # - virtio-blk-ccw # - virtio-scsi # - nvidmm -block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@" +block_device_driver = "@VMROOTFSDRIVER_NV@" # aio is the I/O mechanism used by qemu # Options: diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index f8850e810d..8166ba3af9 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -885,6 +885,21 @@ function helm_helper() { yq -i ".snapshotter.erofsMergeMode = \"${EROFS_MERGE_MODE}\"" "${values_yaml}" fi + if [[ "${SNAPSHOTTER}" == "blockfile" ]]; then + BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE="${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE:-/opt/containerd/blockfile-scratch.img}" + BLOCKFILE_SNAPSHOTTER_ROOT_PATH="${BLOCKFILE_SNAPSHOTTER_ROOT_PATH:-/var/lib/containerd/io.containerd.snapshotter.v1.blockfile}" + + HELM_CONTAINERD_USER_DROP_IN="[plugins.'io.containerd.snapshotter.v1.blockfile']"$'\n' + HELM_CONTAINERD_USER_DROP_IN+=" root_path = \"${BLOCKFILE_SNAPSHOTTER_ROOT_PATH}\""$'\n' + HELM_CONTAINERD_USER_DROP_IN+=" scratch_file = \"${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}\""$'\n' + HELM_CONTAINERD_USER_DROP_IN+=" fs_type = \"ext4\""$'\n' + HELM_CONTAINERD_USER_DROP_IN+=" mount_options = []"$'\n' + HELM_CONTAINERD_USER_DROP_IN+=" recreate_scratch = false" + + HELM_CONTAINERD_USER_DROP_IN="${HELM_CONTAINERD_USER_DROP_IN}" \ + yq -i '.containerd.userDropIn = strenv(HELM_CONTAINERD_USER_DROP_IN)' "${values_yaml}" + fi + if [[ -z "${HELM_SHIMS}" ]]; then die "A list of shims is expected but none was provided" fi diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 528be36a30..5e13ef48e4 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -30,6 +30,9 @@ export KBS="${KBS:-false}" export KBS_INGRESS="${KBS_INGRESS:-}" export KUBERNETES="${KUBERNETES:-}" export SNAPSHOTTER="${SNAPSHOTTER:-}" +export BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE="${BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE:-10G}" +export BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE="${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE:-/opt/containerd/blockfile-scratch.img}" +export BLOCKFILE_SNAPSHOTTER_ROOT_PATH="${BLOCKFILE_SNAPSHOTTER_ROOT_PATH:-/var/lib/containerd/io.containerd.snapshotter.v1.blockfile}" export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}" export NO_PROXY="${NO_PROXY:-${no_proxy:-}}" export PULL_TYPE="${PULL_TYPE:-default}" @@ -149,11 +152,24 @@ EOF sudo dmsetup status -v } +function _setup_blockfile_device() { + sudo mkdir -p "$(dirname "${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}")" + sudo truncate -s "${BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE}" "${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}" + sudo mkfs.ext4 -F "${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}" +} + function configure_snapshotter() { echo "::group::Configuring ${SNAPSHOTTER}" case "${SNAPSHOTTER}" in devmapper) configure_devmapper ;; + blockfile) + # Create the formatted scratch image required by containerd's blockfile + # snapshotter. Plugin config and per-shim snapshotter mapping are applied + # by kata-deploy via helm_helper (containerd.userDropIn and + # SNAPSHOTTER_HANDLER_MAPPING). + _setup_blockfile_device + ;; *) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;; esac @@ -198,6 +214,10 @@ function deploy_kata() { EXPERIMENTAL_FORCE_GUEST_PULL=false fi + if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then + SNAPSHOTTER="blockfile" + fi + ANNOTATIONS="default_vcpus" if [[ "${KATA_HYPERVISOR}" == *azure* ]]; then ANNOTATIONS="image kernel default_vcpus cc_init_data" diff --git a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats index 6661c5f067..9d39408e6b 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats @@ -167,8 +167,12 @@ setup() { # Same pattern as k8s-nvidia-nim.bats: choose manifest by TEE; each YAML has literal secret names. local tee_suffix="" [[ "${TEE}" = "true" ]] && tee_suffix="-tee" - export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml.in" - export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml" + local runtime_rs_suffix="" + if is_runtime_rs && [[ "${TEE}" != "true" ]]; then + runtime_rs_suffix="-runtime-rs" + fi + export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml.in" + export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml" if [[ "${TEE}" = "true" ]]; then setup_kbs_credentials diff --git a/tests/integration/kubernetes/k8s-nvidia-nim.bats b/tests/integration/kubernetes/k8s-nvidia-nim.bats index 71e034b684..856e21bf10 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim.bats @@ -179,6 +179,13 @@ setup_file() { export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml.in" export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml" + if is_runtime_rs && [[ "${TEE}" != "true" ]]; then + export POD_INSTRUCT_YAML_IN="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml.in" + export POD_INSTRUCT_YAML="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml" + export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml.in" + export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml" + fi + dpkg -s jq >/dev/null 2>&1 || sudo apt -y install jq setup_langchain_flow diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in new file mode 100644 index 0000000000..b806ff8e17 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in @@ -0,0 +1,82 @@ +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# runtime-rs non-TEE blockfile variant. Container rootfs is an ext4 block device +# passed through virtio-blk-pci (shared_fs = none), so NIM cache writes stay on +# the container rootfs. +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: ${POD_NAME_INSTRUCT} + labels: + app: ${POD_NAME_INSTRUCT} +spec: + restartPolicy: Never + runtimeClassName: kata + imagePullSecrets: + - name: ngc-secret-instruct + containers: + - name: ${POD_NAME_INSTRUCT} + image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1 + # Ports exposed by the container: + ports: + - containerPort: 8000 + name: http-openai + livenessProbe: + httpGet: + path: /v1/health/live + port: http-openai + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v1/health/ready + port: http-openai + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /v1/health/ready + port: http-openai + initialDelaySeconds: 150 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 45 + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key-instruct + key: api-key + # GPU resource limit (for NVIDIA GPU) + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "16" + memory: "128Gi" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-instruct +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-instruct +type: Opaque +data: + api-key: "${NGC_API_KEY_BASE64}" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in new file mode 100644 index 0000000000..2ddb14f8cc --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in @@ -0,0 +1,55 @@ +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# runtime-rs non-TEE blockfile variant. Container rootfs is an ext4 block device +# passed through virtio-blk-pci (shared_fs = none), so NIM cache writes stay on +# the container rootfs. +# +--- +apiVersion: apps.nvidia.com/v1alpha1 +kind: NIMService +metadata: + name: ${NIM_SERVICE_NAME} +spec: + image: + repository: nvcr.io/nim/meta/llama-3.2-1b-instruct + tag: "1.12.0" + pullPolicy: IfNotPresent + pullSecrets: + - ngc-secret-llama-3-2-1b + authSecret: ngc-api-key-llama-3-2-1b + storage: + # The operator requires a storage target. For this runtime-rs blockfile + # variant, writes are still served by the container rootfs block device. + emptyDir: + sizeLimit: 10Gi + replicas: 1 + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "8" + memory: "56Gi" + expose: + service: + type: ClusterIP + port: 8000 + runtimeClassName: kata + userID: 1000 + groupID: 1000 +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-llama-3-2-1b +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-llama-3-2-1b +type: Opaque +data: + NGC_API_KEY: "${NGC_API_KEY_BASE64}" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in new file mode 100644 index 0000000000..3b990f4ea5 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in @@ -0,0 +1,98 @@ +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# runtime-rs non-TEE blockfile variant. Container rootfs is an ext4 block device +# passed through virtio-blk-pci (shared_fs = none), so NIM cache writes stay on +# the container rootfs. +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: ${POD_NAME_EMBEDQA} + labels: + app: ${POD_NAME_EMBEDQA} +spec: + # unlike the instruct manifest, this image needs securityContext to + # avoid NVML/GPU permission failures + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + restartPolicy: Always + runtimeClassName: kata + serviceAccountName: default + imagePullSecrets: + - name: ngc-secret-embedqa + containers: + - name: ${POD_NAME_EMBEDQA} + image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1 + imagePullPolicy: IfNotPresent + env: + - name: NIM_CACHE_PATH + value: "/opt/nim/.cache" + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key-embedqa + key: api-key + - name: NIM_HTTP_API_PORT + value: "8000" + - name: NIM_JSONL_LOGGING + value: "1" + - name: NIM_LOG_LEVEL + value: "INFO" + ports: + - containerPort: 8000 + name: http + + livenessProbe: + httpGet: + path: /v1/health/live + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + startupProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 40 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 180 + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "16" + memory: "48Gi" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-embedqa +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-embedqa +type: Opaque +data: + api-key: "${NGC_API_KEY_BASE64}" diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml index 125ba85166..66e67f3359 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml @@ -41,7 +41,7 @@ shims: - amd64 allowedHypervisorAnnotations: [] containerd: - snapshotter: "" + snapshotter: "blockfile" runtimeClass: # This label is automatically added by gpu-operator. Override it # if you want to use a different label.