From 1dac6b135754ee833faee95184bcaa7bc7bbd2ff Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Mon, 5 Feb 2024 18:55:59 +0100 Subject: [PATCH 1/6] runtime-rs: Configure s390x specific flags for Makefile s390x supports a different machine type `s390-ccw-virtio` and it is not required to configure cpu features by default for the platform. A hypervisor `dragonball` is not supported on s390x so that `DBCMD` is not necessary. `vm-rootfs_driver` should be set to `virtio-blk-ccw`. This commit is to set the architecture-specific flags for Makefile. Fixes: #9158 Signed-off-by: Hyounggyu Choi --- src/runtime-rs/Makefile | 4 ++++ src/runtime-rs/arch/s390x-options.mk | 7 ++----- src/runtime-rs/crates/hypervisor/src/kernel_param.rs | 7 ++++--- src/runtime-rs/crates/hypervisor/src/lib.rs | 1 + 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 09ba3bc388..9bfd6169e9 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -273,7 +273,11 @@ ifneq (,$(QEMUCMD)) # qemu-specific options DEFSANDBOXCGROUPONLY_QEMU := false +ifeq ($(ARCH), s390x) + VMROOTFSDRIVER_QEMU := virtio-blk-ccw +else VMROOTFSDRIVER_QEMU := virtio-pmem +endif DEFVCPUS_QEMU := 1 DEFMAXVCPUS_QEMU := 0 DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs diff --git a/src/runtime-rs/arch/s390x-options.mk b/src/runtime-rs/arch/s390x-options.mk index f6381eee22..8cda86cc7d 100644 --- a/src/runtime-rs/arch/s390x-options.mk +++ b/src/runtime-rs/arch/s390x-options.mk @@ -4,12 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 # -MACHINETYPE := +MACHINETYPE := s390-ccw-virtio KERNELPARAMS := MACHINEACCELERATORS := -CPUFEATURES := pmu=off +CPUFEATURES := QEMUCMD := qemu-system-s390x - -# dragonball binary name -DBCMD := dragonball diff --git a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs index 9521ebac7e..c6a93034ba 100644 --- a/src/runtime-rs/crates/hypervisor/src/kernel_param.rs +++ b/src/runtime-rs/crates/hypervisor/src/kernel_param.rs @@ -7,8 +7,9 @@ use anyhow::{anyhow, Result}; use crate::{ - VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_MMIO, VM_ROOTFS_DRIVER_PMEM, VM_ROOTFS_FILESYSTEM_EROFS, - VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS, VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM, + VM_ROOTFS_DRIVER_BLK, VM_ROOTFS_DRIVER_BLK_CCW, VM_ROOTFS_DRIVER_MMIO, VM_ROOTFS_DRIVER_PMEM, + VM_ROOTFS_FILESYSTEM_EROFS, VM_ROOTFS_FILESYSTEM_EXT4, VM_ROOTFS_FILESYSTEM_XFS, + VM_ROOTFS_ROOT_BLK, VM_ROOTFS_ROOT_PMEM, }; use kata_types::config::LOG_VPORT_OPTION; @@ -89,7 +90,7 @@ impl KernelParams { } } } - VM_ROOTFS_DRIVER_BLK | VM_ROOTFS_DRIVER_MMIO => { + VM_ROOTFS_DRIVER_BLK | VM_ROOTFS_DRIVER_BLK_CCW | VM_ROOTFS_DRIVER_MMIO => { params.push(Param::new("root", VM_ROOTFS_ROOT_BLK)); match rootfs_type { VM_ROOTFS_FILESYSTEM_EXT4 | VM_ROOTFS_FILESYSTEM_XFS => { diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index aed86810b5..f7aec752d6 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -34,6 +34,7 @@ pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH; // Config which driver to use as vm root dev const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk-pci"; +const VM_ROOTFS_DRIVER_BLK_CCW: &str = "virtio-blk-ccw"; const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem"; const VM_ROOTFS_DRIVER_MMIO: &str = "virtio-blk-mmio"; From 7b3d1adb8c6d95b539d6899c320432b6cd3d1d30 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Mon, 26 Feb 2024 13:56:46 +0100 Subject: [PATCH 2/6] libs: Bump sysinfo to v0.30.5 It has been observed that the runtime stops running around `sysinfo::total_memory()` while adjusting a config on s390x. This is to update the crate to the latest version which happened to resolve the issue. (No explicit release note for this) Signed-off-by: Hyounggyu Choi --- src/libs/kata-types/Cargo.toml | 2 +- .../kata-types/src/config/hypervisor/mod.rs | 2 +- src/runtime-rs/Cargo.lock | 34 +++++++++++++++---- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index c139726ccd..c875264106 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -26,7 +26,7 @@ serde_json = "1.0.73" thiserror = "1.0" toml = "0.5.8" serde-enum-str = "0.4" -sysinfo = "0.29.11" +sysinfo = "0.30.5" oci = { path = "../oci" } safe-path = { path = "../safe-path" } diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index b571e26c6f..c23fe02b69 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -33,7 +33,7 @@ use std::collections::HashMap; use std::io::{self, Result}; use std::path::Path; use std::sync::{Arc, Mutex}; -use sysinfo::{System, SystemExt}; +use sysinfo::System; mod dragonball; pub use self::dragonball::{DragonballConfig, HYPERVISOR_NAME_DRAGONBALL}; diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 06791f5915..0e3942c0a6 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -1673,7 +1673,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows 0.48.0", ] [[package]] @@ -1874,9 +1874,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libz-sys" @@ -2749,7 +2749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "059a34f111a9dee2ce1ac2826a68b24601c4298cfeb1a587c3cb493d5ab46f52" dependencies = [ "libc", - "nix 0.26.2", + "nix 0.27.1", ] [[package]] @@ -3690,6 +3690,7 @@ dependencies = [ "hyperlocal", "kata-sys-util", "kata-types", + "nix 0.24.3", "tokio", ] @@ -3888,9 +3889,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.29.11" +version = "0.30.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +checksum = "0c385888ef380a852a16209afc8cfad22795dd8873d69c9a14d2e2088f118d18" dependencies = [ "cfg-if 1.0.0", "core-foundation-sys", @@ -3898,7 +3899,7 @@ dependencies = [ "ntapi", "once_cell", "rayon", - "winapi", + "windows 0.52.0", ] [[package]] @@ -4701,6 +4702,25 @@ dependencies = [ "windows-targets 0.48.1", ] +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.45.0" From 9b2c08935b46d6e0ebeb22eb30b9af47863ef611 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Thu, 21 Mar 2024 08:45:54 +0100 Subject: [PATCH 3/6] runtime-rs: Pass different device argument based on bus type Currently, `*-pci` is used as an argument for the device config. It is not true for a case where a different type of bus is used. s390x uses `ccw`. This commit is to make it flexible to generate the device argument based on the bus type. A structure `DeviceVhostUserFsPci` and `VhostVsockPci` is renamed to `DeviceVhostUserFs` and `VhostVsock` because the structure name is not bound to a certain bus type any more. Signed-off-by: Hyounggyu Choi --- .../hypervisor/src/qemu/cmdline_generator.rs | 66 ++++++++++++++----- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs index 1bca1fe53f..123bff82c9 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs @@ -10,6 +10,7 @@ use crate::{kernel_param::KernelParams, HypervisorConfig, NetworkConfig}; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use kata_types::config::hypervisor::NetworkInfo; +use std::fmt::Display; use std::fs::{read_to_string, File}; use std::os::fd::AsRawFd; use std::os::unix::io::RawFd; @@ -44,6 +45,27 @@ trait ToQemuParams: Send + Sync { async fn qemu_params(&self) -> Result>; } +#[derive(Debug)] +enum VirtioBusType { + Pci, + Ccw, +} + +impl VirtioBusType { + fn as_str(&self) -> &str { + match self { + VirtioBusType::Pci => "pci", + VirtioBusType::Ccw => "ccw", + } + } +} + +impl Display for VirtioBusType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + #[derive(Debug)] struct Kernel { // PathBuf would seem more appropriae but since we get the kernel path @@ -532,7 +554,8 @@ impl ToQemuParams for ChardevSocket { } #[derive(Debug)] -struct DeviceVhostUserFsPci { +struct DeviceVhostUserFs { + bus_type: VirtioBusType, chardev: String, tag: String, queue_size: u64, @@ -540,9 +563,10 @@ struct DeviceVhostUserFsPci { iommu_platform: bool, } -impl DeviceVhostUserFsPci { - fn new(chardev: &str, tag: &str) -> DeviceVhostUserFsPci { - DeviceVhostUserFsPci { +impl DeviceVhostUserFs { + fn new(chardev: &str, tag: &str, bus_type: VirtioBusType) -> DeviceVhostUserFs { + DeviceVhostUserFs { + bus_type, chardev: chardev.to_owned(), tag: tag.to_owned(), queue_size: 0, @@ -559,7 +583,9 @@ impl DeviceVhostUserFsPci { // throughout runtime-rs warn!( sl!(), - "bad vhost-user-fs-pci queue_size (must be power of two): {}, ignoring", queue_size + "bad vhost-user-fs-{} queue_size (must be power of two): {}, ignoring", + self.bus_type, + queue_size ); } self @@ -578,10 +604,10 @@ impl DeviceVhostUserFsPci { } #[async_trait] -impl ToQemuParams for DeviceVhostUserFsPci { +impl ToQemuParams for DeviceVhostUserFs { async fn qemu_params(&self) -> Result> { let mut params = Vec::new(); - params.push("vhost-user-fs-pci".to_owned()); + params.push(format!("vhost-user-fs-{}", self.bus_type)); params.push(format!("chardev={}", self.chardev)); params.push(format!("tag={}", self.tag)); if self.queue_size != 0 { @@ -625,15 +651,17 @@ impl ToQemuParams for DeviceNvdimm { } } -struct VhostVsockPci { +struct VhostVsock { + bus_type: VirtioBusType, vhostfd: RawFd, guest_cid: u32, disable_modern: bool, } -impl VhostVsockPci { - fn new(vhostfd: RawFd, guest_cid: u32) -> VhostVsockPci { - VhostVsockPci { +impl VhostVsock { + fn new(vhostfd: RawFd, guest_cid: u32, bus_type: VirtioBusType) -> VhostVsock { + VhostVsock { + bus_type, vhostfd, guest_cid, disable_modern: false, @@ -647,10 +675,10 @@ impl VhostVsockPci { } #[async_trait] -impl ToQemuParams for VhostVsockPci { +impl ToQemuParams for VhostVsock { async fn qemu_params(&self) -> Result> { let mut params = Vec::new(); - params.push("vhost-vsock-pci".to_owned()); + params.push(format!("vhost-vsock-{}", self.bus_type)); if self.disable_modern { params.push("disable-modern=true".to_owned()); } @@ -767,6 +795,14 @@ impl<'a> QemuCmdLine<'a> { }) } + fn bus_type(&self) -> VirtioBusType { + if self.config.machine_info.machine_type.contains("-ccw-") { + VirtioBusType::Ccw + } else { + VirtioBusType::Pci + } + } + pub fn add_virtiofs_share( &mut self, virtiofsd_socket_path: &str, @@ -783,7 +819,7 @@ impl<'a> QemuCmdLine<'a> { self.devices.push(Box::new(virtiofsd_socket_chardev)); - let mut virtiofs_device = DeviceVhostUserFsPci::new(chardev_name, mount_tag); + let mut virtiofs_device = DeviceVhostUserFs::new(chardev_name, mount_tag, self.bus_type()); virtiofs_device.set_queue_size(queue_size); if self.config.device_info.enable_iommu_platform { virtiofs_device.set_iommu_platform(true); @@ -806,7 +842,7 @@ impl<'a> QemuCmdLine<'a> { pub fn add_vsock(&mut self, vhostfd: RawFd, guest_cid: u32) -> Result<()> { clear_fd_flags(vhostfd).context("clear flags failed")?; - let mut vhost_vsock_pci = VhostVsockPci::new(vhostfd, guest_cid); + let mut vhost_vsock_pci = VhostVsock::new(vhostfd, guest_cid, self.bus_type()); if !self.config.disable_nesting_checks { let nested = match is_running_in_vm() { From 9bcfaad625fde401db39292bb551a5eb03c3d1c6 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Thu, 21 Mar 2024 08:38:04 +0100 Subject: [PATCH 4/6] runtime-rs: Add ccw block device for rootfs Like nvdimm for x86_64, a block device for s390x should be treated differently with `virtio-blk-ccw`. This is to generate a QEMU command line parameter for a block device by using `-blockdev` and `-device` if the `vm_rootfs_driver` is set to `virtio-blk-ccw`. Signed-off-by: Hyounggyu Choi --- .../hypervisor/src/device/device_manager.rs | 7 +- .../hypervisor/src/device/driver/mod.rs | 4 +- .../src/device/driver/virtio_blk.rs | 2 + .../hypervisor/src/qemu/cmdline_generator.rs | 156 ++++++++++++++++++ .../crates/hypervisor/src/qemu/inner.rs | 4 + 5 files changed, 169 insertions(+), 4 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index e5ec6276f5..83bed0ce3f 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -14,8 +14,8 @@ use tokio::sync::{Mutex, RwLock}; use crate::{ vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor, NetworkDevice, ShareFsDevice, VfioDevice, VhostUserConfig, VhostUserNetDevice, VsockDevice, - KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, - VIRTIO_BLOCK_PCI, VIRTIO_PMEM, + KATA_BLK_DEV_TYPE, KATA_CCW_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, + VIRTIO_BLOCK_CCW, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, }; use super::{ @@ -447,6 +447,9 @@ impl DeviceManager { VIRTIO_BLOCK_PCI => { block_config.driver_option = KATA_BLK_DEV_TYPE.to_string(); } + VIRTIO_BLOCK_CCW => { + block_config.driver_option = KATA_CCW_DEV_TYPE.to_string(); + } VIRTIO_PMEM => { block_config.driver_option = KATA_NVDIMM_DEV_TYPE.to_string(); is_pmem = true; diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs index f30070b355..6b6d23c839 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs @@ -20,8 +20,8 @@ pub use vfio::{ pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType}; pub use vhost_user_net::VhostUserNetDevice; pub use virtio_blk::{ - BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, - VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, + BlockConfig, BlockDevice, KATA_BLK_DEV_TYPE, KATA_CCW_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, + KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_CCW, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, }; pub use virtio_fs::{ ShareFsConfig, ShareFsDevice, ShareFsMountConfig, ShareFsMountOperation, ShareFsMountType, diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs index 50d8179200..fdf0f7ea2d 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_blk.rs @@ -15,9 +15,11 @@ use async_trait::async_trait; /// VIRTIO_BLOCK_PCI indicates block driver is virtio-pci based pub const VIRTIO_BLOCK_PCI: &str = "virtio-blk-pci"; pub const VIRTIO_BLOCK_MMIO: &str = "virtio-blk-mmio"; +pub const VIRTIO_BLOCK_CCW: &str = "virtio-blk-ccw"; pub const VIRTIO_PMEM: &str = "virtio-pmem"; pub const KATA_MMIO_BLK_DEV_TYPE: &str = "mmioblk"; pub const KATA_BLK_DEV_TYPE: &str = "blk"; +pub const KATA_CCW_DEV_TYPE: &str = "ccw"; pub const KATA_NVDIMM_DEV_TYPE: &str = "nvdimm"; #[derive(Debug, Clone, Default)] diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs index 123bff82c9..447049f351 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs @@ -651,6 +651,154 @@ impl ToQemuParams for DeviceNvdimm { } } +#[derive(Debug)] +struct BlockBackend { + driver: String, + id: String, + path: String, + aio: String, + cache_direct: bool, + cache_no_flush: bool, + read_only: bool, +} + +impl BlockBackend { + fn new(id: &str, path: &str) -> BlockBackend { + BlockBackend { + driver: "file".to_owned(), + id: id.to_owned(), + path: path.to_owned(), + aio: "threads".to_owned(), + cache_direct: true, + cache_no_flush: false, + read_only: true, + } + } + + #[allow(dead_code)] + fn set_driver(&mut self, driver: &str) -> &mut Self { + self.driver = driver.to_owned(); + self + } + + #[allow(dead_code)] + fn set_aio(&mut self, aio: &str) -> &mut Self { + self.aio = aio.to_owned(); + self + } + + #[allow(dead_code)] + fn set_cache_direct(&mut self, cache_direct: bool) -> &mut Self { + self.cache_direct = cache_direct; + self + } + + #[allow(dead_code)] + fn set_cache_no_flush(&mut self, cache_no_flush: bool) -> &mut Self { + self.cache_no_flush = cache_no_flush; + self + } + + #[allow(dead_code)] + fn set_read_only(&mut self, read_only: bool) -> &mut Self { + self.read_only = read_only; + self + } +} + +#[async_trait] +impl ToQemuParams for BlockBackend { + async fn qemu_params(&self) -> Result> { + let mut params = Vec::new(); + params.push(format!("driver={}", self.driver)); + params.push(format!("node-name=image-{}", self.id)); + params.push(format!("filename={}", self.path)); + params.push(format!("aio={}", self.aio)); + if self.cache_direct { + params.push("cache.direct=on".to_owned()); + } else { + params.push("cache.direct=off".to_owned()); + } + if self.cache_no_flush { + params.push("cache.no-flush=on".to_owned()); + } else { + params.push("cache.no-flush=off".to_owned()); + } + if self.read_only { + params.push("auto-read-only=on".to_owned()); + } else { + params.push("auto-read-only=off".to_owned()); + } + Ok(vec!["-blockdev".to_owned(), params.join(",")]) + } +} + +#[derive(Debug)] +struct DeviceVirtioBlk { + bus_type: VirtioBusType, + id: String, + scsi: bool, + config_wce: bool, + share_rw: bool, +} + +impl DeviceVirtioBlk { + fn new(id: &str, bus_type: VirtioBusType) -> DeviceVirtioBlk { + DeviceVirtioBlk { + bus_type, + id: id.to_owned(), + scsi: false, + config_wce: false, + share_rw: true, + } + } + + #[allow(dead_code)] + fn set_scsi(&mut self, scsi: bool) -> &mut Self { + self.scsi = scsi; + self + } + + #[allow(dead_code)] + fn set_config_wce(&mut self, config_wce: bool) -> &mut Self { + self.config_wce = config_wce; + self + } + + #[allow(dead_code)] + fn set_share_rw(&mut self, share_rw: bool) -> &mut Self { + self.share_rw = share_rw; + self + } +} + +#[async_trait] +impl ToQemuParams for DeviceVirtioBlk { + async fn qemu_params(&self) -> Result> { + let mut params = Vec::new(); + params.push(format!("virtio-blk-{}", self.bus_type)); + params.push(format!("drive=image-{}", self.id)); + if self.scsi { + params.push("scsi=on".to_owned()); + } else { + params.push("scsi=off".to_owned()); + } + if self.config_wce { + params.push("config-wce=on".to_owned()); + } else { + params.push("config-wce=off".to_owned()); + } + if self.share_rw { + params.push("share-rw=on".to_owned()); + } else { + params.push("share-rw=off".to_owned()); + } + params.push(format!("serial=image-{}", self.id)); + + Ok(vec!["-device".to_owned(), params.join(",")]) + } +} + struct VhostVsock { bus_type: VirtioBusType, vhostfd: RawFd, @@ -894,6 +1042,14 @@ impl<'a> QemuCmdLine<'a> { Ok(()) } + pub fn add_block_device(&mut self, device_id: &str, path: &str) -> Result<()> { + self.devices + .push(Box::new(BlockBackend::new(device_id, path))); + self.devices + .push(Box::new(DeviceVirtioBlk::new(device_id, self.bus_type()))); + Ok(()) + } + #[allow(dead_code)] pub fn add_serial_console(&mut self, character_device_file_path: &str) { let serial = Serial::new(character_device_file_path); diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 3d6d47526b..23c9bc7ab8 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -103,6 +103,10 @@ impl QemuInner { &block_dev.config.path_on_host, block_dev.config.is_readonly, )?, + "ccw" => cmdline.add_block_device( + block_dev.device_id.as_str(), + &block_dev.config.path_on_host, + )?, unsupported => { info!(sl!(), "unsupported block device driver: {}", unsupported) } From 2cfe745efbc20c0f15f19dc4c57bc659c23409a5 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Fri, 22 Mar 2024 14:54:45 +0100 Subject: [PATCH 5/6] runtime-rs: Enable memory backend option for Machine for s390x For s390x, it requires an additional option `memory-backend` for `-machine`. Otherwise, virtiofsd exits with HandleRequest(InvalidParam). This commit is to add a field `memory_backend` to `struct Machine` and turn it on for s390x. Signed-off-by: Hyounggyu Choi --- .../hypervisor/src/qemu/cmdline_generator.rs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs index 447049f351..950c38fafa 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs @@ -285,6 +285,7 @@ struct Machine { nvdimm: bool, is_nvdimm_supported: bool, + memory_backend: Option, } impl Machine { @@ -310,6 +311,7 @@ impl Machine { options: config.machine_info.machine_accelerators.clone(), nvdimm: false, is_nvdimm_supported, + memory_backend: None, } } @@ -320,6 +322,11 @@ impl Machine { self.nvdimm = is_on && self.is_nvdimm_supported; self } + + fn set_memory_backend(&mut self, mem_backend: &str) -> &mut Self { + self.memory_backend = Some(mem_backend.to_owned()); + self + } } #[async_trait] @@ -334,6 +341,9 @@ impl ToQemuParams for Machine { if self.nvdimm { params.push("nvdimm=on".to_owned()); } + if let Some(mem_backend) = &self.memory_backend { + params.push(format!("memory-backend={}", mem_backend)); + } Ok(vec!["-machine".to_owned(), params.join(",")]) } } @@ -983,8 +993,15 @@ impl<'a> QemuCmdLine<'a> { //self.devices.push(Box::new(mem_file)); self.memory.set_memory_backend_file(&mem_file); - self.machine.set_nvdimm(true); - self.devices.push(Box::new(NumaNode::new(&mem_file.id))); + match self.bus_type() { + VirtioBusType::Pci => { + self.machine.set_nvdimm(true); + self.devices.push(Box::new(NumaNode::new(&mem_file.id))); + } + VirtioBusType::Ccw => { + self.machine.set_memory_backend(&mem_file.id); + } + } } pub fn add_vsock(&mut self, vhostfd: RawFd, guest_cid: u32) -> Result<()> { From 81aaa34bd6ebfb92c0f7d37011b68b938f061355 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Wed, 28 Feb 2024 08:33:54 +0100 Subject: [PATCH 6/6] runtime-rs: Add DeviceVirtioSerial and DeviceVirtconsole It is observed that virtiofsd exits immediately on s390x if there is no attached console devices. This commit resolves the issue by migrating `appendConsole()` from runtime and being triggered in `start_vm()`. Signed-off-by: Hyounggyu Choi --- .../hypervisor/src/qemu/cmdline_generator.rs | 72 ++++++++++++++++++- .../crates/hypervisor/src/qemu/inner.rs | 5 ++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs index 950c38fafa..3e3a919ff9 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs @@ -528,13 +528,11 @@ impl ChardevSocket { } } - #[allow(dead_code)] fn set_server(&mut self, server: bool) -> &mut Self { self.server = server; self } - #[allow(dead_code)] fn set_wait(&mut self, wait: bool) -> &mut Self { self.wait = wait; self @@ -556,6 +554,8 @@ impl ToQemuParams for ChardevSocket { params.push("server=on".to_owned()); if self.wait { params.push("wait=on".to_owned()); + } else { + params.push("wait=off".to_owned()); } } params.append(&mut self.protocol_options.qemu_params().await?); @@ -911,6 +911,57 @@ impl ToQemuParams for NetDevice { } } +#[derive(Debug)] +struct DeviceVirtioSerial { + id: String, + bus_type: VirtioBusType, +} + +impl DeviceVirtioSerial { + fn new(id: &str, bus_type: VirtioBusType) -> DeviceVirtioSerial { + DeviceVirtioSerial { + id: id.to_owned(), + bus_type, + } + } +} + +#[async_trait] +impl ToQemuParams for DeviceVirtioSerial { + async fn qemu_params(&self) -> Result> { + let mut params = Vec::new(); + params.push(format!("virtio-serial-{}", self.bus_type)); + params.push(format!("id={}", self.id)); + Ok(vec!["-device".to_owned(), params.join(",")]) + } +} + +#[derive(Debug)] +struct DeviceVirtconsole { + id: String, + chardev: String, +} + +impl DeviceVirtconsole { + fn new(id: &str, chardev: &str) -> DeviceVirtconsole { + DeviceVirtconsole { + id: id.to_owned(), + chardev: chardev.to_owned(), + } + } +} + +#[async_trait] +impl ToQemuParams for DeviceVirtconsole { + async fn qemu_params(&self) -> Result> { + let mut params = Vec::new(); + params.push("virtconsole".to_owned()); + params.push(format!("id={}", self.id)); + params.push(format!("chardev={}", self.chardev)); + Ok(vec!["-device".to_owned(), params.join(",")]) + } +} + fn is_running_in_vm() -> Result { let res = read_to_string("/proc/cpuinfo")? .lines() @@ -1098,6 +1149,23 @@ impl<'a> QemuCmdLine<'a> { Ok(fds) } + pub fn add_console(&mut self, console_socket_path: &str) { + let serial_dev = DeviceVirtioSerial::new("serial0", self.bus_type()); + self.devices.push(Box::new(serial_dev)); + + let chardev_name = "charconsole0"; + let console_device = DeviceVirtconsole::new("console0", chardev_name); + self.devices.push(Box::new(console_device)); + + let mut console_socket_chardev = ChardevSocket::new(chardev_name); + console_socket_chardev.set_socket_opts(ProtocolOptions::Unix(UnixSocketOpts { + path: console_socket_path.to_owned(), + })); + console_socket_chardev.set_server(true); + console_socket_chardev.set_wait(false); + self.devices.push(Box::new(console_socket_chardev)); + } + pub async fn build(&self) -> Result> { let mut result = Vec::new(); diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 23c9bc7ab8..310f75c710 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -18,6 +18,7 @@ use kata_types::{ use persist::sandbox_persist::Persist; use std::collections::HashMap; use std::os::unix::io::AsRawFd; +use std::path::Path; use std::process::Stdio; use tokio::{ io::{AsyncBufReadExt, BufReader}, @@ -128,6 +129,10 @@ impl QemuInner { // `tty` in it to get its device file path and use it as the argument). //cmdline.add_serial_console("/dev/pts/23"); + // Add a console to the devices of the cmdline + let console_socket_path = Path::new(&self.get_jailer_root().await?).join("console.sock"); + cmdline.add_console(console_socket_path.to_str().unwrap()); + info!(sl!(), "qemu args: {}", cmdline.build().await?.join(" ")); let mut command = Command::new(&self.config.path); command.args(cmdline.build().await?);