diff --git a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml index e465096b53..aef09019ce 100644 --- a/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml +++ b/.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml @@ -45,10 +45,10 @@ jobs: fail-fast: false matrix: environment: [ - { name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false }, - { name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false }, - { name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true }, - { name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true }, + { name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false, snapshotter: "" }, + { name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false, snapshotter: "erofs" }, + { name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" }, + { name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }} @@ -62,8 +62,9 @@ jobs: KATA_HYPERVISOR: ${{ matrix.environment.vmm }} KUBERNETES: kubeadm KBS: ${{ matrix.environment.coco && 'true' || 'false' }} - SNAPSHOTTER: ${{ matrix.environment.coco && 'nydus' || '' }} - USE_EXPERIMENTAL_SNAPSHOTTER_SETUP: ${{ matrix.environment.coco && 'true' || 'false' }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter }} + EROFS_SNAPSHOTTER_MODE: ${{ matrix.environment.snapshotter == 'erofs' && 'memory' || '' }} + EROFS_DMVERITY: ${{ matrix.environment.snapshotter == 'erofs' && 'dmverity' || '' }} K8S_TEST_HOST_TYPE: baremetal steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 5e333ea446..02310361f7 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -202,6 +202,7 @@ DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"kernel_params\", \"kernel_veri DEFDISABLEGUESTSECCOMP := true DEFEMPTYDIRMODE := shared-fs DEFEMPTYDIRMODE_COCO := block-encrypted +DEFEMPTYDIRMODE_NV := block-plain ##VAR DEFAULTEXPFEATURES=[features] Default experimental features enabled DEFAULTEXPFEATURES := [] DEFDISABLESELINUX := false @@ -722,6 +723,7 @@ USER_VARS += DEFNETWORKMODEL_QEMU USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFEMPTYDIRMODE USER_VARS += DEFEMPTYDIRMODE_COCO +USER_VARS += DEFEMPTYDIRMODE_NV USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFDISABLESELINUX USER_VARS += DEFDISABLEGUESTSELINUX diff --git a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in index ce7025f407..a679ee2b07 100644 --- a/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in @@ -195,7 +195,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-fs-nydus # - none -shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" +shared_fs = "none" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" @@ -816,7 +816,7 @@ vfio_mode = "@DEFVFIOMODE_NV@" # - block-plain # Plugs a block device to be mounted directly in the guest. # -emptydir_mode = "@DEFEMPTYDIRMODE@" +emptydir_mode = "@DEFEMPTYDIRMODE_NV@" # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index e0554a2dd0..90cc20a9db 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -874,6 +874,11 @@ function helm_helper() { HELM_CONTAINERD_USER_DROP_IN="[plugins.'io.containerd.snapshotter.v1.erofs']"$'\n' HELM_CONTAINERD_USER_DROP_IN+=" default_size = \"${erofs_default_size}\"" + # NVIDIA CI hosts are not configured with ext4 fs-verity support. + if [[ "${KATA_HYPERVISOR:-}" == *"nvidia-gpu"* ]]; then + HELM_CONTAINERD_USER_DROP_IN+=$'\n'" enable_fsverity = false" + fi + HELM_CONTAINERD_USER_DROP_IN="${HELM_CONTAINERD_USER_DROP_IN}" \ yq -i '.containerd.userDropIn = strenv(HELM_CONTAINERD_USER_DROP_IN)' "${values_yaml}" diff --git a/tests/integration/docker/gha-run.sh b/tests/integration/docker/gha-run.sh index 09693052cc..fc1dff6d79 100755 --- a/tests/integration/docker/gha-run.sh +++ b/tests/integration/docker/gha-run.sh @@ -32,6 +32,19 @@ function enable_kata_debug() { sudo sed -i -e 's/^#\?[[:space:]]*enable_debug[[:space:]]*=.*/enable_debug = true/g' "${cfg}" } +function configure_docker_nvidia_runtime_rs() { + [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] || return 0 + + local -r cfg="${KATA_CONFIG_PATH:-}" + [[ -z "${cfg}" || ! -e "${cfg}" ]] && return 0 + + info "Configuring NVIDIA runtime-rs for docker smoke test in ${cfg}" + sudo sed -i \ + -e 's/^#\?[[:space:]]*shared_fs[[:space:]]*=.*/shared_fs = "virtio-fs"/g' \ + -e 's/^#\?[[:space:]]*emptydir_mode[[:space:]]*=.*/emptydir_mode = "shared-fs"/g' \ + "${cfg}" +} + function dump_kata_debug() { local -r kata_runtime="$1" @@ -92,6 +105,7 @@ function run() { info "Running docker smoke test tests using ${KATA_HYPERVISOR} hypervisor" enabling_hypervisor + configure_docker_nvidia_runtime_rs enable_kata_debug info "Running docker with runc" diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index f20969875e..53aeea9398 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -200,6 +200,10 @@ function deploy_kata() { EXPERIMENTAL_FORCE_GUEST_PULL=false fi + if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then + SNAPSHOTTER="erofs" + fi + ANNOTATIONS="default_vcpus" if [[ "${KATA_HYPERVISOR}" == *azure* ]]; then ANNOTATIONS="image kernel default_vcpus cc_init_data" diff --git a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats index 6661c5f067..7040db0cbe 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim-service.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim-service.bats @@ -22,7 +22,9 @@ NIM_SERVICE_NAME="meta-llama-3-2-1b-instruct" export NIM_SERVICE_NAME POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=600s -[[ "${TEE}" = "true" ]] && POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s +if [[ "${TEE}" = "true" ]] || is_runtime_rs; then + POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED=1200s +fi export POD_READY_TIMEOUT_LLAMA_3_2_1B=${POD_READY_TIMEOUT_LLAMA_3_2_1B:-${POD_READY_TIMEOUT_LLAMA_3_2_1B_PREDEFINED}} export LOCAL_NIM_CACHE_LLAMA_3_2_1B="${LOCAL_NIM_CACHE_LLAMA_3_2_1B:-${LOCAL_NIM_CACHE:-/opt/nim/.cache}-llama-3-2-1b}" @@ -167,8 +169,12 @@ setup() { # Same pattern as k8s-nvidia-nim.bats: choose manifest by TEE; each YAML has literal secret names. local tee_suffix="" [[ "${TEE}" = "true" ]] && tee_suffix="-tee" - export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml.in" - export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml" + local runtime_rs_suffix="" + if is_runtime_rs && [[ "${TEE}" != "true" ]]; then + runtime_rs_suffix="-runtime-rs" + fi + export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml.in" + export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml" if [[ "${TEE}" = "true" ]]; then setup_kbs_credentials diff --git a/tests/integration/kubernetes/k8s-nvidia-nim.bats b/tests/integration/kubernetes/k8s-nvidia-nim.bats index 71e034b684..f56470c39a 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim.bats @@ -28,6 +28,8 @@ POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=600s if [[ "${TEE}" = "true" ]]; then POD_NAME_EMBEDQA="${POD_NAME_EMBEDQA}-tee" POD_NAME_INSTRUCT="${POD_NAME_INSTRUCT}-tee" +fi +if [[ "${TEE}" = "true" ]] || is_runtime_rs; then POD_READY_TIMEOUT_EMBEDQA_PREDEFINED=1000s POD_READY_TIMEOUT_INSTRUCT_PREDEFINED=1000s fi @@ -179,6 +181,13 @@ setup_file() { export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml.in" export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml" + if is_runtime_rs && [[ "${TEE}" != "true" ]]; then + export POD_INSTRUCT_YAML_IN="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml.in" + export POD_INSTRUCT_YAML="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml" + export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml.in" + export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml" + fi + dpkg -s jq >/dev/null 2>&1 || sudo apt -y install jq setup_langchain_flow diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in new file mode 100644 index 0000000000..33d0217239 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in @@ -0,0 +1,92 @@ +# Copyright (c) 2025 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: ${POD_NAME_INSTRUCT} + labels: + app: ${POD_NAME_INSTRUCT} +spec: + # block-plain emptyDir volumes are formatted and mounted in the guest. + # Set fsGroup so the agent makes the cache writable by the NIM user. + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + restartPolicy: Never + runtimeClassName: kata + imagePullSecrets: + - name: ngc-secret-instruct + containers: + - name: ${POD_NAME_INSTRUCT} + image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1 + # Ports exposed by the container: + ports: + - containerPort: 8000 + name: http-openai + livenessProbe: + httpGet: + path: /v1/health/live + port: http-openai + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v1/health/ready + port: http-openai + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /v1/health/ready + port: http-openai + initialDelaySeconds: 360 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 30 + env: + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key-instruct + key: api-key + # GPU resource limit (for NVIDIA GPU) + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "16" + memory: "48Gi" + volumeMounts: + - name: nim-cache + mountPath: /opt/nim/.cache + + volumes: + - name: nim-cache + emptyDir: + sizeLimit: 64Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-instruct +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-instruct +type: Opaque +data: + api-key: "${NGC_API_KEY_BASE64}" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in new file mode 100644 index 0000000000..61128f272c --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in @@ -0,0 +1,49 @@ +# Copyright (c) 2026 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: apps.nvidia.com/v1alpha1 +kind: NIMService +metadata: + name: ${NIM_SERVICE_NAME} +spec: + image: + repository: nvcr.io/nim/meta/llama-3.2-1b-instruct + tag: "1.12.0" + pullPolicy: IfNotPresent + pullSecrets: + - ngc-secret-llama-3-2-1b + authSecret: ngc-api-key-llama-3-2-1b + storage: + emptyDir: + sizeLimit: 10Gi + replicas: 1 + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "8" + memory: "16Gi" + expose: + service: + type: ClusterIP + port: 8000 + runtimeClassName: kata + userID: 1000 + groupID: 1000 +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-llama-3-2-1b +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-llama-3-2-1b +type: Opaque +data: + NGC_API_KEY: "${NGC_API_KEY_BASE64}" diff --git a/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in new file mode 100644 index 0000000000..846199e42d --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in @@ -0,0 +1,103 @@ +# Copyright (c) 2025 NVIDIA Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: ${POD_NAME_EMBEDQA} + labels: + app: ${POD_NAME_EMBEDQA} +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + restartPolicy: Always + runtimeClassName: kata + serviceAccountName: default + imagePullSecrets: + - name: ngc-secret-embedqa + containers: + - name: ${POD_NAME_EMBEDQA} + image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1 + imagePullPolicy: IfNotPresent + env: + - name: NIM_CACHE_PATH + value: "/opt/nim/.cache" + - name: NGC_API_KEY + valueFrom: + secretKeyRef: + name: ngc-api-key-embedqa + key: api-key + - name: NIM_HTTP_API_PORT + value: "8000" + - name: NIM_JSONL_LOGGING + value: "1" + - name: NIM_LOG_LEVEL + value: "INFO" + ports: + - containerPort: 8000 + name: http + + livenessProbe: + httpGet: + path: /v1/health/live + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + + startupProbe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 180 + + resources: + limits: + nvidia.com/pgpu: "1" + cpu: "16" + memory: "24Gi" + + volumeMounts: + - name: nim-cache + mountPath: /opt/nim/.cache + + volumes: + - name: nim-cache + emptyDir: + sizeLimit: 40Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-secret-embedqa +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: ${DOCKER_CONFIG_JSON} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ngc-api-key-embedqa +type: Opaque +data: + api-key: "${NGC_API_KEY_BASE64}" diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml index 125ba85166..d3b71f37af 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml @@ -14,7 +14,7 @@ debug: false deploymentMode: job snapshotter: - setup: ["nydus"] + setup: ["nydus", "erofs"] # Enable NVIDIA GPU shims # Disable all shims at once, then enable only the ones we need @@ -41,7 +41,7 @@ shims: - amd64 allowedHypervisorAnnotations: [] containerd: - snapshotter: "" + snapshotter: "erofs" runtimeClass: # This label is automatically added by gpu-operator. Override it # if you want to use a different label. @@ -144,4 +144,3 @@ runtimeClasses: enabled: true createDefault: false defaultName: "kata" -