github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-09 22:24:51 +00:00
Code Issues Projects Releases Wiki Activity

nvidia: use blockfile snapshotter for runtime-rs

Browse Source 
Signed-off-by: Manuel Huber <manuelh@nvidia.com>
This commit is contained in:
Manuel Huber
2026-06-23 20:23:55 +00:00
parent 392b802f61
commit 8f308bab9b
11 changed files with 300 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/run-k8s-tests-on-nvidia-gpu.yaml vendored
View File

@@ -45,10 +45,10 @@ jobs:
fail-fast: false
matrix:
environment: [
{ name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false },
{ name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false },
{ name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true },
{ name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true },
{ name: nvidia-gpu, vmm: qemu-nvidia-gpu, runner: amd64-nvidia-a100, coco: false, snapshotter: "" },
{ name: nvidia-gpu-runtime-rs, vmm: qemu-nvidia-gpu-runtime-rs, runner: amd64-nvidia-a100, coco: false, snapshotter: "blockfile", blockfile_scratch_size: 64G },
{ name: nvidia-gpu-snp, vmm: qemu-nvidia-gpu-snp, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" },
{ name: nvidia-gpu-snp-runtime-rs, vmm: qemu-nvidia-gpu-snp-runtime-rs, runner: amd64-nvidia-h100-snp, coco: true, snapshotter: "nydus" },
]
concurrency:
group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-${{ toJSON(matrix) }}
@@ -62,8 +62,8 @@ jobs:
KATA_HYPERVISOR: ${{ matrix.environment.vmm }}
KUBERNETES: kubeadm
KBS: ${{ matrix.environment.coco && 'true' || 'false' }}
SNAPSHOTTER: ${{ matrix.environment.coco && 'nydus' || '' }}
USE_EXPERIMENTAL_SNAPSHOTTER_SETUP: ${{ matrix.environment.coco && 'true' || 'false' }}
SNAPSHOTTER: ${{ matrix.environment.snapshotter }}
BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE: ${{ matrix.environment.blockfile_scratch_size || '10G' }}
K8S_TEST_HOST_TYPE: baremetal
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -105,6 +105,10 @@ jobs:
timeout-minutes: 10
run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
- name: Configure ${{ matrix.environment.snapshotter }} snapshotter
if: matrix.environment.snapshotter == 'blockfile'
run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
- name: Deploy Kata
timeout-minutes: 20
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata

4
src/runtime-rs/Makefile
View File

@@ -466,6 +466,9 @@ endif
DEFROOTFSTYPE_NV := $(ROOTFSTYPE_EROFS)
# NVIDIA profile: rootfs block driver (avoid virtio-pmem/DAX for this stack)
VMROOTFSDRIVER_NV := virtio-blk-pci
# NVIDIA runtime-rs with blockfile snapshotter: pass container rootfs as a block
# device (virtio-blk-pci) with no virtio-fs shared mount.
DEFSHAREDFS_QEMU_NVIDIA_GPU_RUNTIME_RS := none
# Agent client: hybrid vsock retry delay (ms); vsock path uses this as initial backoff (clamped in agent)
DEFDIALTIMEOUTMS_NV := 1000
# Go qemu-nvidia-gpu dial_timeout is 1200s; runtime-rs uses ms for reconnect_timeout_ms
@@ -588,6 +591,7 @@ USER_VARS += CONFIG_QEMU_SE_IN
USER_VARS += CONFIG_REMOTE_IN
USER_VARS += CONFIG_QEMU_COCO_DEV_IN
USER_VARS += CONFIG_QEMU_NVIDIA_GPU_IN
USER_VARS += DEFSHAREDFS_QEMU_NVIDIA_GPU_RUNTIME_RS
USER_VARS += DESTDIR
USER_VARS += HYPERVISOR
USER_VARS += USE_BUILTIN_DB

4
src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
View File

@@ -178,7 +178,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@
# - virtio-fs (default)
# - virtio-fs-nydus
# - none
shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@"
shared_fs = "@DEFSHAREDFS_QEMU_NVIDIA_GPU_RUNTIME_RS@"
# Path to vhost-user-fs daemon.
virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
@@ -228,7 +228,7 @@ virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
# - virtio-blk-ccw
# - virtio-scsi
# - nvidmm
block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@"
block_device_driver = "@VMROOTFSDRIVER_NV@"
# aio is the I/O mechanism used by qemu
# Options:

15
tests/gha-run-k8s-common.sh
View File

@@ -885,6 +885,21 @@ function helm_helper() {
yq -i ".snapshotter.erofsMergeMode = \"${EROFS_MERGE_MODE}\"" "${values_yaml}"
fi
if [[ "${SNAPSHOTTER}" == "blockfile" ]]; then
BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE="${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE:-/opt/containerd/blockfile-scratch.img}"
BLOCKFILE_SNAPSHOTTER_ROOT_PATH="${BLOCKFILE_SNAPSHOTTER_ROOT_PATH:-/var/lib/containerd/io.containerd.snapshotter.v1.blockfile}"
HELM_CONTAINERD_USER_DROP_IN="[plugins.'io.containerd.snapshotter.v1.blockfile']"$'\n'
HELM_CONTAINERD_USER_DROP_IN+=" root_path = \"${BLOCKFILE_SNAPSHOTTER_ROOT_PATH}\""$'\n'
HELM_CONTAINERD_USER_DROP_IN+=" scratch_file = \"${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}\""$'\n'
HELM_CONTAINERD_USER_DROP_IN+=" fs_type = \"ext4\""$'\n'
HELM_CONTAINERD_USER_DROP_IN+=" mount_options = []"$'\n'
HELM_CONTAINERD_USER_DROP_IN+=" recreate_scratch = false"
HELM_CONTAINERD_USER_DROP_IN="${HELM_CONTAINERD_USER_DROP_IN}" \
yq -i '.containerd.userDropIn = strenv(HELM_CONTAINERD_USER_DROP_IN)' "${values_yaml}"
fi
if [[ -z "${HELM_SHIMS}" ]]; then
die "A list of shims is expected but none was provided"
fi

20
tests/integration/kubernetes/gha-run.sh
View File

@@ -30,6 +30,9 @@ export KBS="${KBS:-false}"
export KBS_INGRESS="${KBS_INGRESS:-}"
export KUBERNETES="${KUBERNETES:-}"
export SNAPSHOTTER="${SNAPSHOTTER:-}"
export BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE="${BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE:-10G}"
export BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE="${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE:-/opt/containerd/blockfile-scratch.img}"
export BLOCKFILE_SNAPSHOTTER_ROOT_PATH="${BLOCKFILE_SNAPSHOTTER_ROOT_PATH:-/var/lib/containerd/io.containerd.snapshotter.v1.blockfile}"
export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}"
export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"
export PULL_TYPE="${PULL_TYPE:-default}"
@@ -149,11 +152,24 @@ EOF
sudo dmsetup status -v
}
function _setup_blockfile_device() {
sudo mkdir -p "$(dirname "${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}")"
sudo truncate -s "${BLOCKFILE_SNAPSHOTTER_SCRATCH_SIZE}" "${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}"
sudo mkfs.ext4 -F "${BLOCKFILE_SNAPSHOTTER_SCRATCH_FILE}"
}
function configure_snapshotter() {
echo "::group::Configuring ${SNAPSHOTTER}"
case "${SNAPSHOTTER}" in
devmapper) configure_devmapper ;;
blockfile)
# Create the formatted scratch image required by containerd's blockfile
# snapshotter. Plugin config and per-shim snapshotter mapping are applied
# by kata-deploy via helm_helper (containerd.userDropIn and
# SNAPSHOTTER_HANDLER_MAPPING).
_setup_blockfile_device
;;
*) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;;
esac
@@ -198,6 +214,10 @@ function deploy_kata() {
EXPERIMENTAL_FORCE_GUEST_PULL=false
fi
if [[ "${KATA_HYPERVISOR}" == "qemu-nvidia-gpu-runtime-rs" ]] && [[ -z "${SNAPSHOTTER}" ]]; then
SNAPSHOTTER="blockfile"
fi
ANNOTATIONS="default_vcpus"
if [[ "${KATA_HYPERVISOR}" == *azure* ]]; then
ANNOTATIONS="image kernel default_vcpus cc_init_data"

8
tests/integration/kubernetes/k8s-nvidia-nim-service.bats
View File

@@ -167,8 +167,12 @@ setup() {
# Same pattern as k8s-nvidia-nim.bats: choose manifest by TEE; each YAML has literal secret names.
local tee_suffix=""
[[ "${TEE}" = "true" ]] && tee_suffix="-tee"
export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml.in"
export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}.yaml"
local runtime_rs_suffix=""
if is_runtime_rs && [[ "${TEE}" != "true" ]]; then
runtime_rs_suffix="-runtime-rs"
fi
export NIM_YAML_IN="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml.in"
export NIM_YAML="${pod_config_dir}/nvidia-nim-llama-3-2-1b-instruct-service${tee_suffix}${runtime_rs_suffix}.yaml"
if [[ "${TEE}" = "true" ]]; then
setup_kbs_credentials

7
tests/integration/kubernetes/k8s-nvidia-nim.bats
View File

@@ -179,6 +179,13 @@ setup_file() {
export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml.in"
export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}.yaml"
if is_runtime_rs && [[ "${TEE}" != "true" ]]; then
export POD_INSTRUCT_YAML_IN="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml.in"
export POD_INSTRUCT_YAML="${pod_config_dir}/${POD_NAME_INSTRUCT}-runtime-rs.yaml"
export POD_EMBEDQA_YAML_IN="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml.in"
export POD_EMBEDQA_YAML="${pod_config_dir}/${POD_NAME_EMBEDQA}-runtime-rs.yaml"
fi
dpkg -s jq >/dev/null 2>&1 || sudo apt -y install jq
setup_langchain_flow

82
tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-1-8b-instruct-runtime-rs.yaml.in Normal file
View File

@@ -0,0 +1,82 @@
# Copyright (c) 2026 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# runtime-rs non-TEE blockfile variant. Container rootfs is an ext4 block device
# passed through virtio-blk-pci (shared_fs = none), so NIM cache writes stay on
# the container rootfs.
#
---
apiVersion: v1
kind: Pod
metadata:
name: ${POD_NAME_INSTRUCT}
labels:
app: ${POD_NAME_INSTRUCT}
spec:
restartPolicy: Never
runtimeClassName: kata
imagePullSecrets:
- name: ngc-secret-instruct
containers:
- name: ${POD_NAME_INSTRUCT}
image: nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1
# Ports exposed by the container:
ports:
- containerPort: 8000
name: http-openai
livenessProbe:
httpGet:
path: /v1/health/live
port: http-openai
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /v1/health/ready
port: http-openai
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
startupProbe:
httpGet:
path: /v1/health/ready
port: http-openai
initialDelaySeconds: 150
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 45
env:
- name: NGC_API_KEY
valueFrom:
secretKeyRef:
name: ngc-api-key-instruct
key: api-key
# GPU resource limit (for NVIDIA GPU)
resources:
limits:
nvidia.com/pgpu: "1"
cpu: "16"
memory: "128Gi"
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-secret-instruct
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: ${DOCKER_CONFIG_JSON}
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-api-key-instruct
type: Opaque
data:
api-key: "${NGC_API_KEY_BASE64}"

55
tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-1b-instruct-service-runtime-rs.yaml.in Normal file
View File

@@ -0,0 +1,55 @@
# Copyright (c) 2026 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# runtime-rs non-TEE blockfile variant. Container rootfs is an ext4 block device
# passed through virtio-blk-pci (shared_fs = none), so NIM cache writes stay on
# the container rootfs.
#
---
apiVersion: apps.nvidia.com/v1alpha1
kind: NIMService
metadata:
name: ${NIM_SERVICE_NAME}
spec:
image:
repository: nvcr.io/nim/meta/llama-3.2-1b-instruct
tag: "1.12.0"
pullPolicy: IfNotPresent
pullSecrets:
- ngc-secret-llama-3-2-1b
authSecret: ngc-api-key-llama-3-2-1b
storage:
# The operator requires a storage target. For this runtime-rs blockfile
# variant, writes are still served by the container rootfs block device.
emptyDir:
sizeLimit: 10Gi
replicas: 1
resources:
limits:
nvidia.com/pgpu: "1"
cpu: "8"
memory: "56Gi"
expose:
service:
type: ClusterIP
port: 8000
runtimeClassName: kata
userID: 1000
groupID: 1000
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-secret-llama-3-2-1b
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: ${DOCKER_CONFIG_JSON}
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-api-key-llama-3-2-1b
type: Opaque
data:
NGC_API_KEY: "${NGC_API_KEY_BASE64}"

98
tests/integration/kubernetes/runtimeclass_workloads/nvidia-nim-llama-3-2-nv-embedqa-1b-v2-runtime-rs.yaml.in Normal file
View File

@@ -0,0 +1,98 @@
# Copyright (c) 2026 NVIDIA Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# runtime-rs non-TEE blockfile variant. Container rootfs is an ext4 block device
# passed through virtio-blk-pci (shared_fs = none), so NIM cache writes stay on
# the container rootfs.
#
---
apiVersion: v1
kind: Pod
metadata:
name: ${POD_NAME_EMBEDQA}
labels:
app: ${POD_NAME_EMBEDQA}
spec:
# unlike the instruct manifest, this image needs securityContext to
# avoid NVML/GPU permission failures
securityContext:
runAsUser: 1000
runAsGroup: 1000
restartPolicy: Always
runtimeClassName: kata
serviceAccountName: default
imagePullSecrets:
- name: ngc-secret-embedqa
containers:
- name: ${POD_NAME_EMBEDQA}
image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1
imagePullPolicy: IfNotPresent
env:
- name: NIM_CACHE_PATH
value: "/opt/nim/.cache"
- name: NGC_API_KEY
valueFrom:
secretKeyRef:
name: ngc-api-key-embedqa
key: api-key
- name: NIM_HTTP_API_PORT
value: "8000"
- name: NIM_JSONL_LOGGING
value: "1"
- name: NIM_LOG_LEVEL
value: "INFO"
ports:
- containerPort: 8000
name: http
livenessProbe:
httpGet:
path: /v1/health/live
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
readinessProbe:
httpGet:
path: /v1/health/ready
port: 8000
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 3
startupProbe:
httpGet:
path: /v1/health/ready
port: 8000
initialDelaySeconds: 40
periodSeconds: 10
timeoutSeconds: 1
successThreshold: 1
failureThreshold: 180
resources:
limits:
nvidia.com/pgpu: "1"
cpu: "16"
memory: "48Gi"
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-secret-embedqa
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: ${DOCKER_CONFIG_JSON}
---
apiVersion: v1
kind: Secret
metadata:
name: ngc-api-key-embedqa
type: Opaque
data:
api-key: "${NGC_API_KEY_BASE64}"

2
tools/packaging/kata-deploy/helm-chart/kata-deploy/try-kata-nvidia-gpu.values.yaml
View File

@@ -41,7 +41,7 @@ shims:
- amd64
allowedHypervisorAnnotations: []
containerd:
snapshotter: ""
snapshotter: "blockfile"
runtimeClass:
# This label is automatically added by gpu-operator. Override it
# if you want to use a different label.