From e17f96251dc35b0bb1a1e16b42ca6f20db877827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Fri, 16 Jan 2026 13:43:33 -0600 Subject: [PATCH 1/4] runtime{,-rs}/clh: Disable virtio-pmem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This disables virtio-pmem support for Cloud Hypervisor by changing Kata config defaults and removing the relevant code paths. Signed-off-by: Aurélien Bombo --- src/runtime-rs/Makefile | 2 +- .../configuration-cloud-hypervisor.toml.in | 2 + .../hypervisor/ch-config/src/convert.rs | 21 +++------- .../crates/hypervisor/src/ch/inner.rs | 7 +++- .../hypervisor/src/ch/inner_hypervisor.rs | 9 +---- src/runtime/Makefile | 2 + src/runtime/config/configuration-clh.toml.in | 4 +- src/runtime/virtcontainers/clh.go | 39 +++++++------------ src/runtime/virtcontainers/clh_test.go | 1 + 9 files changed, 36 insertions(+), 51 deletions(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index bd98178c0b..eea49c54f9 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -298,7 +298,7 @@ ifneq (,$(CLHCMD)) KERNELTYPE_CLH = uncompressed KERNEL_NAME_CLH = $(call MAKE_KERNEL_NAME,$(KERNELTYPE_CLH)) KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH) - VMROOTFSDRIVER_CLH := virtio-pmem + VMROOTFSDRIVER_CLH := virtio-blk-pci DEFSANDBOXCGROUPONLY_CLH := true DEFSTATICRESOURCEMGMT_CLH := false diff --git a/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in b/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in index 06a069eaf5..4b981d47c0 100644 --- a/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in +++ b/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in @@ -22,6 +22,8 @@ rootfs_type = @DEFROOTFSTYPE@ # Block storage driver to be used for the VM rootfs is backed # by a block device. +# +# virtio-pmem is not supported with Cloud Hypervisor. vm_rootfs_driver = "@VMROOTFSDRIVER_CLH@" # Path to the firmware. diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs index eae5807936..cd6998a649 100644 --- a/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs @@ -118,13 +118,11 @@ impl TryFrom for VmConfig { // Note how CH handles the different image types: // - // - A standard image is specified in PmemConfig. // - An initrd/initramfs is specified in PayloadConfig. - // - A confidential guest image is specified by a DiskConfig. + // - An image is specified in DiskConfig. + // Note: pmem is not used as it's not properly supported by Cloud Hypervisor. // - If TDX is enabled, the firmware (`td-shim` [1]) must be // specified in PayloadConfig. - // - A confidential guest initrd is specified by a PayloadConfig with - // firmware. // // [1] - https://github.com/confidential-containers/td-shim let boot_info = cfg.boot_info; @@ -140,14 +138,6 @@ impl TryFrom for VmConfig { return Err(VmConfigError::NoBootFile); } - let pmem = if use_initrd || guest_protection_is_tdx(guest_protection_to_use.clone()) { - None - } else { - let pmem = PmemConfig::try_from(&boot_info).map_err(VmConfigError::PmemError)?; - - Some(vec![pmem]) - }; - let payload = Some( PayloadConfig::try_from(( boot_info.clone(), @@ -159,7 +149,7 @@ impl TryFrom for VmConfig { let mut disks: Vec = vec![]; - if use_image && guest_protection_is_tdx(guest_protection_to_use.clone()) { + if use_image { let disk = DiskConfig::try_from(boot_info).map_err(VmConfigError::DiskError)?; disks.push(disk); @@ -199,7 +189,6 @@ impl TryFrom for VmConfig { fs, net, devices: host_devices, - pmem, disks, vsock: Some(vsock), rng, @@ -1656,7 +1645,6 @@ mod tests { let (memory_info_confidential_guest, mem_config_confidential_guest) = make_memory_objects(79, usable_max_mem_bytes, true); - let (_, pmem_config_with_image) = make_bootinfo_pmemconfig_objects(image); let (machine_info, rng_config) = make_machineinfo_rngconfig_objects(entropy_source); let payload_firmware = None; @@ -1664,6 +1652,7 @@ mod tests { let (boot_info_with_initrd, payload_config_with_initrd) = make_bootinfo_payloadconfig_objects(kernel, initramfs, payload_firmware, None); + let (_, disk_config_with_image) = make_bootinfo_diskconfig_objects(image); let (_, disk_config_confidential_guest_image) = make_bootinfo_diskconfig_objects(image); let boot_info_tdx_image = BootInfo { @@ -1762,7 +1751,7 @@ mod tests { vsock: Some(valid_vsock.clone()), // rootfs image specific - pmem: Some(vec![pmem_config_with_image]), + disks: Some(vec![disk_config_with_image]), payload: Some(PayloadConfig { kernel: Some(PathBuf::from(kernel)), diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs index c7f31bd0d8..3b9ed8a002 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs @@ -123,7 +123,12 @@ impl CloudHypervisorInner { } } - pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) { + pub fn set_hypervisor_config(&mut self, mut config: HypervisorConfig) { + // virtio-pmem is not supported for Cloud Hypervisor. + if config.boot_info.vm_rootfs_driver == crate::VM_ROOTFS_DRIVER_PMEM { + config.boot_info.vm_rootfs_driver = crate::VM_ROOTFS_DRIVER_BLK.to_string(); + } + self.config = config; } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index f2debb5cc1..2ba7378413 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -15,7 +15,6 @@ use crate::utils::vm_cleanup; use crate::utils::{bytes_to_megs, get_jailer_root, get_sandbox_path, megs_to_bytes}; use crate::MemoryConfig; use crate::VM_ROOTFS_DRIVER_BLK; -use crate::VM_ROOTFS_DRIVER_PMEM; use crate::{VcpuThreadIds, VmmState}; use anyhow::{anyhow, Context, Result}; use ch_config::ch_api::cloud_hypervisor_vm_netdev_add_with_fds; @@ -130,12 +129,8 @@ impl CloudHypervisorInner { let confidential_guest = cfg.security_info.confidential_guest; // Note that the configuration option hypervisor.block_device_driver is not used. - let rootfs_driver = if confidential_guest { - // PMEM is not available with TDX. - VM_ROOTFS_DRIVER_BLK - } else { - VM_ROOTFS_DRIVER_PMEM - }; + // NVDIMM is not supported for Cloud Hypervisor. + let rootfs_driver = VM_ROOTFS_DRIVER_BLK; let rootfs_type = match cfg.boot_info.rootfs_type.is_empty() { true => DEFAULT_CH_ROOTFS_TYPE, diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 9d85319af0..31c65627d3 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -288,6 +288,7 @@ DEFSTATICRESOURCEMGMT_NV = true DEFDISABLEIMAGENVDIMM ?= false DEFDISABLEIMAGENVDIMM_NV = true +DEFDISABLEIMAGENVDIMM_CLH ?= true DEFBINDMOUNTS := [] @@ -788,6 +789,7 @@ USER_VARS += DEFVFIOMODE_SE USER_VARS += BUILDFLAGS USER_VARS += DEFDISABLEIMAGENVDIMM USER_VARS += DEFDISABLEIMAGENVDIMM_NV +USER_VARS += DEFDISABLEIMAGENVDIMM_CLH USER_VARS += DEFCCAMEASUREMENTALGO USER_VARS += DEFSHAREDFS_QEMU_CCA_VIRTIOFS USER_VARS += DEFPODRESOURCEAPISOCK diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 3b61b1c377..8718b56f39 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -222,8 +222,8 @@ hypervisor_loglevel = 1 # If false and nvdimm is supported, use nvdimm device to plug guest image. # Otherwise virtio-block device is used. # -# nvdimm is not supported when `confidential_guest = true`. -disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM@ +# nvdimm is not supported with Cloud Hypervisor or when `confidential_guest = true`. +disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_CLH@ # Enable hot-plugging of VFIO devices to a root-port. # The default setting is "no-port" diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index a77af24950..844004928b 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -332,6 +332,9 @@ func (clh *cloudHypervisor) getClhStopSandboxTimeout() time.Duration { func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error { clh.config = *config + // We don't support NVDIMM with Cloud Hypervisor. + clh.config.DisableImageNvdimm = true + return nil } @@ -584,8 +587,8 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net // Set initial amount of cpu's for the virtual machine clh.vmconfig.Cpus = chclient.NewCpusConfig(int32(clh.config.NumVCPUs()), int32(clh.config.DefaultMaxVCPUs)) - disableNvdimm := (clh.config.DisableImageNvdimm || clh.config.ConfidentialGuest) - enableDax := !disableNvdimm + disableNvdimm := true + enableDax := false params, err := getNonUserDefinedKernelParams(hypervisorConfig.RootfsType, disableNvdimm, enableDax, clh.config.Debug, clh.config.ConfidentialGuest, clh.config.IOMMU, hypervisorConfig.KernelVerityParams) if err != nil { @@ -607,31 +610,19 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net } if assetType == types.ImageAsset { - if clh.config.DisableImageNvdimm || clh.config.ConfidentialGuest { - disk := chclient.NewDiskConfig() - disk.Path = &assetPath - disk.SetReadonly(true) + disk := chclient.NewDiskConfig() + disk.Path = &assetPath + disk.SetReadonly(true) - diskRateLimiterConfig := clh.getDiskRateLimiterConfig() - if diskRateLimiterConfig != nil { - disk.SetRateLimiterConfig(*diskRateLimiterConfig) - } + diskRateLimiterConfig := clh.getDiskRateLimiterConfig() + if diskRateLimiterConfig != nil { + disk.SetRateLimiterConfig(*diskRateLimiterConfig) + } - if clh.vmconfig.Disks != nil { - *clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk) - } else { - clh.vmconfig.Disks = &[]chclient.DiskConfig{*disk} - } + if clh.vmconfig.Disks != nil { + *clh.vmconfig.Disks = append(*clh.vmconfig.Disks, *disk) } else { - pmem := chclient.NewPmemConfig(assetPath) - *pmem.DiscardWrites = true - pmem.SetIommu(clh.config.IOMMU) - - if clh.vmconfig.Pmem != nil { - *clh.vmconfig.Pmem = append(*clh.vmconfig.Pmem, *pmem) - } else { - clh.vmconfig.Pmem = &[]chclient.PmemConfig{*pmem} - } + clh.vmconfig.Disks = &[]chclient.DiskConfig{*disk} } } else { // assetType == types.InitrdAsset diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index f4fb50e17e..0a90982d32 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -69,6 +69,7 @@ func newClhConfig() (HypervisorConfig, error) { NetRateLimiterOpsMaxRate: int64(0), NetRateLimiterOpsOneTimeBurst: int64(0), HotPlugVFIO: config.NoPort, + DisableImageNvdimm: true, }, nil } From c727332b0ee001eb5467dc74459329602422587f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Mon, 3 Nov 2025 23:43:30 -0600 Subject: [PATCH 2/4] runtime/qemu/arm64: Align NVDIMM usage on amd64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nowadays on arm64 we use a modern QEMU version which supports the features we require for NVDIMM, so we remove the arm64-specific code and use the generic implementation. Signed-off-by: Aurélien Bombo --- .../virtcontainers/qemu_arch_base_test.go | 32 ++++++++++++++++++ src/runtime/virtcontainers/qemu_arm64.go | 30 ----------------- src/runtime/virtcontainers/qemu_arm64_test.go | 33 ------------------- 3 files changed, 32 insertions(+), 63 deletions(-) diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 3090e765b4..dfaebb8dab 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -332,6 +332,38 @@ func TestQemuArchBaseAppendImage(t *testing.T) { assert.Equal(expectedOut, devices) } +func TestQemuArchBaseAppendNvdimmImage(t *testing.T) { + var devices []govmmQemu.Device + assert := assert.New(t) + qemuArchBase := newQemuArchBase() + + image, err := os.CreateTemp("", "img") + assert.NoError(err) + defer image.Close() + defer os.Remove(image.Name()) + + imageStat, err := image.Stat() + assert.NoError(err) + + devices, err = qemuArchBase.appendNvdimmImage(devices, image.Name()) + assert.NoError(err) + assert.Len(devices, 1) + + expectedOut := []govmmQemu.Device{ + govmmQemu.Object{ + Driver: govmmQemu.NVDIMM, + Type: govmmQemu.MemoryBackendFile, + DeviceID: "nv0", + ID: "mem0", + MemPath: image.Name(), + Size: (uint64)(imageStat.Size()), + ReadOnly: true, + }, + } + + assert.Equal(expectedOut, devices) +} + func TestQemuArchBaseAppendBridges(t *testing.T) { var devices []govmmQemu.Device assert := assert.New(t) diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index 3f72637a4a..93d8e8fd64 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -10,7 +10,6 @@ package virtcontainers import ( "context" "fmt" - "os" "time" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" @@ -109,35 +108,6 @@ func (q *qemuArm64) appendImage(ctx context.Context, devices []govmmQemu.Device, return q.appendBlockImage(ctx, devices, path) } -// There is no nvdimm/readonly feature in qemu 5.1 which is used by arm64 for now, -// so we temporarily add this specific implementation for arm64 here until -// the qemu used by arm64 is capable for that feature -func (q *qemuArm64) appendNvdimmImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { - imageFile, err := os.Open(path) - if err != nil { - return nil, err - } - defer imageFile.Close() - - imageStat, err := imageFile.Stat() - if err != nil { - return nil, err - } - - object := govmmQemu.Object{ - Driver: govmmQemu.NVDIMM, - Type: govmmQemu.MemoryBackendFile, - DeviceID: "nv0", - ID: "mem0", - MemPath: path, - Size: (uint64)(imageStat.Size()), - } - - devices = append(devices, object) - - return devices, nil -} - func (q *qemuArm64) setIgnoreSharedMemoryMigrationCaps(_ context.Context, _ *govmmQemu.QMP) error { // x-ignore-shared not support in arm64 for now return nil diff --git a/src/runtime/virtcontainers/qemu_arm64_test.go b/src/runtime/virtcontainers/qemu_arm64_test.go index 66f53bf7fe..de62fc48c2 100644 --- a/src/runtime/virtcontainers/qemu_arm64_test.go +++ b/src/runtime/virtcontainers/qemu_arm64_test.go @@ -130,39 +130,6 @@ func TestQemuArm64AppendImage(t *testing.T) { assert.Equal(expectedOut, devices) } -func TestQemuArm64AppendNvdimmImage(t *testing.T) { - var devices []govmmQemu.Device - assert := assert.New(t) - - f, err := os.CreateTemp("", "img") - assert.NoError(err) - defer func() { _ = f.Close() }() - defer func() { _ = os.Remove(f.Name()) }() - - imageStat, err := f.Stat() - assert.NoError(err) - - cfg := qemuConfig(QemuVirt) - cfg.ImagePath = f.Name() - arm64, err := newQemuArch(cfg) - assert.NoError(err) - - expectedOut := []govmmQemu.Device{ - govmmQemu.Object{ - Driver: govmmQemu.NVDIMM, - Type: govmmQemu.MemoryBackendFile, - DeviceID: "nv0", - ID: "mem0", - MemPath: f.Name(), - Size: (uint64)(imageStat.Size()), - }, - } - - devices, err = arm64.appendNvdimmImage(devices, f.Name()) - assert.NoError(err) - assert.Equal(expectedOut, devices) -} - func TestQemuArm64WithInitrd(t *testing.T) { assert := assert.New(t) From 48aa077e8cb41226598c3e9f3f46a72d2f57ee4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Wed, 4 Feb 2026 11:32:56 -0600 Subject: [PATCH 3/4] runtime{,-rs}/qemu/arm64: Disable DAX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enabling full-featured QEMU NVDIMM support on ARM with DAX enabled causes a kernel panic in caches_clean_inval_pou (see below, different issue from 33b1f07), so we disable DAX in that environment. [ 1.222529] EXT4-fs (pmem0p1): mounted filesystem e5a4892c-dac8-42ee-ba55-27d4ff2f38c3 ro with ordered data mode. Quota mode: disabled. [ 1.222695] VFS: Mounted root (ext4 filesystem) readonly on device 259:1. [ 1.224890] devtmpfs: mounted [ 1.225175] Freeing unused kernel memory: 1920K [ 1.226102] Run /sbin/init as init process [ 1.226164] with arguments: [ 1.226204] /sbin/init [ 1.226235] with environment: [ 1.226268] HOME=/ [ 1.226295] TERM=linux [ 1.230974] Internal error: synchronous external abort: 0000000096000010 [#1] SMP [ 1.231963] CPU: 0 UID: 0 PID: 1 Comm: init Tainted: G M 6.18.5 #1 NONE [ 1.232965] Tainted: [M]=MACHINE_CHECK [ 1.233428] pstate: 43400005 (nZcv daif +PAN -UAO +TCO +DIT -SSBS BTYPE=--) [ 1.234273] pc : caches_clean_inval_pou+0x68/0x84 [ 1.234862] lr : sync_icache_aliases+0x30/0x38 [ 1.235412] sp : ffff80008000b9a0 [ 1.235842] x29: ffff80008000b9a0 x28: 0000000000000000 x27: 00000000019a00e1 [ 1.236912] x26: ffff80008000bc08 x25: ffff80008000baf0 x24: fffffdffc0000000 [ 1.238064] x23: ffff000001671ab0 x22: ffff000001663480 x21: fffffdffc23401c0 [ 1.239356] x20: fffffdffc23401c0 x19: fffffdffc23401c0 x18: 0000000000000000 [ 1.240626] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 [ 1.241762] x14: ffffaae8f021b3b0 x13: 0000000000000000 x12: ffffaae8f021b3b0 [ 1.242874] x11: ffffffffffffffff x10: 0000000000000000 x9 : 0000ffffbb53c000 [ 1.244022] x8 : 0000000000000000 x7 : 0000000000000012 x6 : ffff55178f5e5000 [ 1.245157] x5 : ffff80008000b970 x4 : ffff00007fa4f680 x3 : ffff00008d007000 [ 1.246257] x2 : 0000000000000040 x1 : ffff00008d008000 x0 : ffff00008d007000 [ 1.247387] Call trace: [ 1.248056] caches_clean_inval_pou+0x68/0x84 (P) [ 1.248923] __sync_icache_dcache+0x7c/0x9c [ 1.249578] insert_page_into_pte_locked+0x1e4/0x284 [ 1.250432] insert_page+0xa8/0xc0 [ 1.251080] vmf_insert_page_mkwrite+0x40/0x7c [ 1.251832] dax_iomap_pte_fault+0x598/0x804 [ 1.252646] dax_iomap_fault+0x28/0x30 [ 1.253293] ext4_dax_huge_fault+0x80/0x2dc [ 1.253988] ext4_dax_fault+0x10/0x3c [ 1.254679] __do_fault+0x38/0x12c [ 1.255293] __handle_mm_fault+0x530/0xcf0 [ 1.255990] handle_mm_fault+0xe4/0x230 [ 1.256697] do_page_fault+0x17c/0x4dc [ 1.257487] do_translation_fault+0x30/0x38 [ 1.258184] do_mem_abort+0x40/0x8c [ 1.258895] el0_ia+0x4c/0x170 [ 1.259420] el0t_64_sync_handler+0xd8/0xdc [ 1.260154] el0t_64_sync+0x168/0x16c [ 1.260795] Code: d2800082 9ac32042 d1000443 8a230003 (d50b7523) [ 1.261756] ---[ end trace 0000000000000000 ]--- Signed-off-by: Aurélien Bombo --- .../hypervisor/src/ch/inner_hypervisor.rs | 1 + .../crates/hypervisor/src/dragonball/inner.rs | 1 + .../hypervisor/src/firecracker/fc_api.rs | 1 + .../crates/hypervisor/src/kernel_param.rs | 61 ++++++++++++++----- .../hypervisor/src/qemu/cmdline_generator.rs | 7 +++ src/runtime/virtcontainers/qemu_arm64.go | 7 ++- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 2ba7378413..57b88c27e4 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -150,6 +150,7 @@ impl CloudHypervisorInner { &cfg.boot_info.kernel_verity_params, rootfs_driver, rootfs_type, + true, )?; let mut console_params = if enable_debug { diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index d810d9e07f..01d3ca400e 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -150,6 +150,7 @@ impl DragonballInner { &self.config.boot_info.kernel_verity_params, &rootfs_driver, &self.config.boot_info.rootfs_type, + true, )?; kernel_params.append(&mut rootfs_params); } diff --git a/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs b/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs index 8fe9de784a..06f2a1da81 100644 --- a/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs +++ b/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs @@ -90,6 +90,7 @@ impl FcInner { &self.config.boot_info.kernel_verity_params, &self.config.blockdev_info.block_device_driver, &self.config.boot_info.rootfs_type, + true, )?; kernel_params.append(&mut rootfs_params); kernel_params.append(&mut KernelParams::from_string( diff --git a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs index 2edd9f4051..6b34ea1f2f 100644 --- a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs +++ b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs @@ -10,8 +10,8 @@ use crate::{ VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_BLK_CCW, VM_ROOTFS_DRIVER_MMIO, VM_ROOTFS_DRIVER_PMEM, VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM, }; -use kata_types::config::LOG_VPORT_OPTION; use kata_types::config::hypervisor::{parse_kernel_verity_params, VERITY_BLOCK_SIZE_BYTES}; +use kata_types::config::LOG_VPORT_OPTION; use kata_types::fs::{ VM_ROOTFS_FILESYSTEM_EROFS, VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS, }; @@ -66,8 +66,7 @@ struct KernelVerityConfig { } fn new_kernel_verity_params(params_string: &str) -> Result> { - let cfg = parse_kernel_verity_params(params_string) - .map_err(|err| anyhow!(err.to_string()))?; + let cfg = parse_kernel_verity_params(params_string).map_err(|err| anyhow!(err.to_string()))?; Ok(cfg.map(|params| KernelVerityConfig { root_hash: params.root_hash, @@ -145,6 +144,7 @@ impl KernelParams { kernel_verity_params: &str, rootfs_driver: &str, rootfs_type: &str, + use_dax: bool, ) -> Result { let mut params = vec![]; @@ -153,16 +153,29 @@ impl KernelParams { params.push(Param::new("root", VM_ROOTFS_ROOT_PMEM)); match rootfs_type { VM_ROOTFS_FILESYSTEM_EXT4 => { - params.push(Param::new( - "rootflags", - "dax,data=ordered,errors=remount-ro ro", - )); + if use_dax { + params.push(Param::new( + "rootflags", + "dax,data=ordered,errors=remount-ro ro", + )); + } else { + params + .push(Param::new("rootflags", "data=ordered,errors=remount-ro ro")); + } } VM_ROOTFS_FILESYSTEM_XFS => { - params.push(Param::new("rootflags", "dax ro")); + if use_dax { + params.push(Param::new("rootflags", "dax ro")); + } else { + params.push(Param::new("rootflags", "ro")); + } } VM_ROOTFS_FILESYSTEM_EROFS => { - params.push(Param::new("rootflags", "dax ro")); + if use_dax { + params.push(Param::new("rootflags", "dax ro")); + } else { + params.push(Param::new("rootflags", "ro")); + } } _ => { return Err(anyhow!("Unsupported rootfs type {}", rootfs_type)); @@ -346,6 +359,7 @@ mod tests { struct TestData<'a> { rootfs_driver: &'a str, rootfs_type: &'a str, + use_dax: bool, expect_params: KernelParams, result: Result<()>, } @@ -353,10 +367,11 @@ mod tests { #[test] fn test_rootfs_kernel_params() { let tests = &[ - // EXT4 + // EXT4 with DAX TestData { rootfs_driver: VM_ROOTFS_DRIVER_PMEM, rootfs_type: VM_ROOTFS_FILESYSTEM_EXT4, + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_PMEM), @@ -370,6 +385,7 @@ mod tests { TestData { rootfs_driver: VM_ROOTFS_DRIVER_BLK, rootfs_type: VM_ROOTFS_FILESYSTEM_EXT4, + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_BLK), @@ -380,14 +396,15 @@ mod tests { }, result: Ok(()), }, - // XFS + // XFS without DAX TestData { rootfs_driver: VM_ROOTFS_DRIVER_PMEM, rootfs_type: VM_ROOTFS_FILESYSTEM_XFS, + use_dax: false, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_PMEM), - Param::new("rootflags", "dax ro"), + Param::new("rootflags", "ro"), Param::new("rootfstype", VM_ROOTFS_FILESYSTEM_XFS), ] .to_vec(), @@ -397,6 +414,7 @@ mod tests { TestData { rootfs_driver: VM_ROOTFS_DRIVER_BLK, rootfs_type: VM_ROOTFS_FILESYSTEM_XFS, + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_BLK), @@ -407,10 +425,11 @@ mod tests { }, result: Ok(()), }, - // EROFS + // EROFS with DAX TestData { rootfs_driver: VM_ROOTFS_DRIVER_PMEM, rootfs_type: VM_ROOTFS_FILESYSTEM_EROFS, + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_PMEM), @@ -424,6 +443,7 @@ mod tests { TestData { rootfs_driver: VM_ROOTFS_DRIVER_BLK, rootfs_type: VM_ROOTFS_FILESYSTEM_EROFS, + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_BLK), @@ -438,6 +458,7 @@ mod tests { TestData { rootfs_driver: "foo", rootfs_type: VM_ROOTFS_FILESYSTEM_EXT4, + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_BLK), @@ -452,6 +473,7 @@ mod tests { TestData { rootfs_driver: VM_ROOTFS_DRIVER_BLK, rootfs_type: "foo", + use_dax: true, expect_params: KernelParams { params: [ Param::new("root", VM_ROOTFS_ROOT_BLK), @@ -466,8 +488,12 @@ mod tests { for (i, t) in tests.iter().enumerate() { let msg = format!("test[{i}]: {t:?}"); - let result = - KernelParams::new_rootfs_kernel_params("", t.rootfs_driver, t.rootfs_type); + let result = KernelParams::new_rootfs_kernel_params( + "", + t.rootfs_driver, + t.rootfs_type, + t.use_dax, + ); let msg = format!("{msg}, result: {result:?}"); if t.result.is_ok() { assert!(result.is_ok(), "{}", msg); @@ -486,6 +512,7 @@ mod tests { "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096", VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_FILESYSTEM_EXT4, + false, )?; let params_string = params.to_string()?; assert!(params_string.contains("dm-mod.create=")); @@ -496,6 +523,7 @@ mod tests { "root_hash=abc,data_blocks=1,data_block_size=4096,hash_block_size=4096", VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_FILESYSTEM_EXT4, + false, ) .err() .expect("expected missing salt error"); @@ -505,6 +533,7 @@ mod tests { "root_hash=abc,salt=def,data_block_size=4096,hash_block_size=4096", VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_FILESYSTEM_EXT4, + false, ) .err() .expect("expected missing data_blocks error"); @@ -514,6 +543,7 @@ mod tests { "root_hash=abc,salt=def,data_blocks=foo,data_block_size=4096,hash_block_size=4096", VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_FILESYSTEM_EXT4, + false, ) .err() .expect("expected invalid data_blocks error"); @@ -523,6 +553,7 @@ mod tests { "root_hash=abc,salt=def,data_blocks=1,data_block_size=4096,hash_block_size=4096,badfield", VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_FILESYSTEM_EXT4, + false, ) .err() .expect("expected invalid entry error"); diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs index 31886b1988..2c9622c93c 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs @@ -179,10 +179,17 @@ impl Kernel { let mut kernel_params = KernelParams::new(config.debug_info.enable_debug); if config.boot_info.initrd.is_empty() { + // DAX is disabled on ARM due to a kernel panic in caches_clean_inval_pou. + #[cfg(target_arch = "aarch64")] + let use_dax = false; + #[cfg(not(target_arch = "aarch64"))] + let use_dax = true; + let mut rootfs_params = KernelParams::new_rootfs_kernel_params( &config.boot_info.kernel_verity_params, &config.boot_info.vm_rootfs_driver, &config.boot_info.rootfs_type, + use_dax, ) .context("adding rootfs/verity params failed")?; kernel_params.append(&mut rootfs_params); diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index 93d8e8fd64..1a3ee7f74e 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -68,9 +68,10 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { kernelParamsDebug: kernelParamsDebug, kernelParams: kernelParams, disableNvdimm: config.DisableImageNvdimm, - dax: true, - protection: noneProtection, - legacySerial: config.LegacySerial, + // DAX is disabled on ARM due to a kernel panic in caches_clean_inval_pou. + dax: false, + protection: noneProtection, + legacySerial: config.LegacySerial, }, measurementAlgo: config.MeasurementAlgo, } From 336b922d4f7fe5dc94087de2af3992a1a786d0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Tue, 3 Feb 2026 14:55:45 -0600 Subject: [PATCH 4/4] tests/cbl-mariner: Stop disabling NVDIMM explicitly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not needed anymore since now disable_image_nvdimm=true for Cloud Hypervisor. Signed-off-by: Aurélien Bombo --- tests/integration/kubernetes/gha-run.sh | 2 +- tests/integration/kubernetes/setup.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index c55ad7d6c1..6e197a8336 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -175,7 +175,7 @@ function deploy_kata() { ANNOTATIONS="default_vcpus" if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then - ANNOTATIONS="image kernel default_vcpus disable_image_nvdimm cc_init_data" + ANNOTATIONS="image kernel default_vcpus cc_init_data" fi if [[ "${KATA_HYPERVISOR}" = "qemu" ]]; then ANNOTATIONS="image initrd kernel default_vcpus" diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 15ed301da3..549ffcacf9 100644 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -108,12 +108,8 @@ add_cbl_mariner_annotation_to_yaml() { local -r mariner_annotation_image="io.katacontainers.config.hypervisor.image" local -r mariner_image_path="/opt/kata/share/kata-containers/kata-containers-mariner.img" - local -r mariner_annotation_disable_image_nvdimm="io.katacontainers.config.hypervisor.disable_image_nvdimm" - local -r mariner_disable_image_nvdimm=true - add_annotations_to_yaml "${yaml_file}" "${mariner_annotation_kernel}" "${mariner_kernel_path}" add_annotations_to_yaml "${yaml_file}" "${mariner_annotation_image}" "${mariner_image_path}" - add_annotations_to_yaml "${yaml_file}" "${mariner_annotation_disable_image_nvdimm}" "${mariner_disable_image_nvdimm}" } add_cbl_mariner_specific_annotations() {