From 80bd71bfcc991aea11a72cffeb959393f9bb2523 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sat, 12 Apr 2025 23:00:53 +0800 Subject: [PATCH 1/3] runtime-rs: Iterates through PCI devices to find a match with qdev_id The get_pci_path_by_qdev_id function is designed to search for a PCI device within a given list of devices based on a specified qdev_id. It tracks the device's path in the PCI topology by recording the slot values of the devices traversed during the search. If the device is located behind a PCI bridge, the function recursively explores the bridge's device list to find the target device. The function returns the matching device along with its updated path if found, otherwise, it returns None. Fixes #11143 Signed-off-by: alex.lyn --- .../crates/hypervisor/src/qemu/qmp.rs | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs b/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs index 32efa5c43e..e4069c86c6 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs @@ -3,10 +3,12 @@ // SPDX-License-Identifier: Apache-2.0 // +use crate::device::pci_path::PciPath; use crate::qemu::cmdline_generator::{DeviceVirtioNet, Netdev}; use anyhow::{anyhow, Result}; use nix::sys::socket::{sendmsg, ControlMessage, MsgFlags}; +use std::convert::TryFrom; use std::fmt::{Debug, Error, Formatter}; use std::io::BufReader; use std::os::fd::{AsRawFd, RawFd}; @@ -14,7 +16,7 @@ use std::os::unix::net::UnixStream; use std::time::Duration; use qapi::qmp; -use qapi_qmp; +use qapi_qmp::{self, PciDeviceInfo}; use qapi_spec::Dictionary; pub struct Qmp { @@ -467,8 +469,56 @@ impl Qmp { Ok(()) } + + pub fn get_device_by_qdev_id(&mut self, qdev_id: &str) -> Result { + let format_str = |vec: &Vec| -> String { + vec.iter() + .map(|num| format!("{:02x}", num)) + .collect::>() + .join("/") + }; + + let mut path = vec![]; + let pci = self.qmp.execute(&qapi_qmp::query_pci {})?; + for pci_info in pci.iter() { + if let Some(_device) = get_pci_path_by_qdev_id(&pci_info.devices, qdev_id, &mut path) { + let pci_path = format_str(&path); + return PciPath::try_from(pci_path.as_str()); + } + } + + Err(anyhow!("no target device found")) + } } fn vcpu_id_from_core_id(core_id: i64) -> String { format!("cpu-{}", core_id) } + +// The get_pci_path_by_qdev_id function searches a device list for a device matching a given qdev_id, +// tracking the device's path. It recursively explores bridge devices and returns the found device along +// with its updated path. +pub fn get_pci_path_by_qdev_id( + devices: &[PciDeviceInfo], + qdev_id: &str, + path: &mut Vec, +) -> Option { + for device in devices { + path.push(device.slot); + if device.qdev_id == qdev_id { + return Some(device.clone()); + } + + if let Some(ref bridge) = device.pci_bridge { + if let Some(ref bridge_devices) = bridge.devices { + if let Some(found_device) = get_pci_path_by_qdev_id(bridge_devices, qdev_id, path) { + return Some(found_device); + } + } + } + + // If the device not found, pop the current slot before moving to next device + path.pop(); + } + None +} From 2405301e2e6031a4b7691a65f7b07b590c2b8e90 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sun, 27 Apr 2025 11:28:44 +0800 Subject: [PATCH 2/3] runtime-rs: Support hotplugging block device via QMP This commit introduces block device hotplugging capability using QMP commands. The implementation enables attaching raw block devices to a running VM through the following steps: 1.Block Device Configuration Uses `blockdev-add` QMP command to define a raw block backend with (1) Direct I/O mode (2) Configurable read-only flag (3) Host file/block device path (`/path/to/block`) 2.PCI Device Attachment, Attaches the block device via `device_add` QMP command as a `virtio-blk-pci` device: (1) Dynamically allocates PCI slots using `find_free_slot()` (2) Binds to user-specified PCIe bus (e.g., `pcie.1`) (3) Returns PCI path for further management Fixes #11143 Signed-off-by: alex.lyn --- .../crates/hypervisor/src/qemu/qmp.rs | 109 +++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs b/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs index e4069c86c6..8b823d8161 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs @@ -6,7 +6,7 @@ use crate::device::pci_path::PciPath; use crate::qemu::cmdline_generator::{DeviceVirtioNet, Netdev}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use nix::sys::socket::{sendmsg, ControlMessage, MsgFlags}; use std::convert::TryFrom; use std::fmt::{Debug, Error, Formatter}; @@ -489,6 +489,113 @@ impl Qmp { Err(anyhow!("no target device found")) } + + /// hotplug block device: + /// { + /// "execute": "blockdev-add", + /// "arguments": { + /// "node-name": "drive-0", + /// "file": {"driver": "file", "filename": "/path/to/block"}, + /// "cache": {"direct": true}, + /// "read-only": false + /// } + /// } + /// + /// { + /// "execute": "device_add", + /// "arguments": { + /// "id": "drive-0", + /// "driver": "virtio-blk-pci", + /// "drive": "drive-0", + /// "addr":"0x0", + /// "bus": "pcie.1" + /// } + /// } + pub fn hotplug_block_device( + &mut self, + block_driver: &str, + device_id: &str, + path_on_host: &str, + is_direct: Option, + is_readonly: bool, + no_drop: bool, + ) -> Result> { + let (bus, slot) = self.find_free_slot()?; + + // `blockdev-add` + let node_name = format!("drive-{}", device_id); + self.qmp + .execute(&qmp::blockdev_add(qmp::BlockdevOptions::raw { + base: qmp::BlockdevOptionsBase { + detect_zeroes: None, + cache: None, + discard: None, + force_share: None, + auto_read_only: None, + node_name: Some(node_name.clone()), + read_only: None, + }, + raw: qmp::BlockdevOptionsRaw { + base: qmp::BlockdevOptionsGenericFormat { + file: qmp::BlockdevRef::definition(Box::new(qmp::BlockdevOptions::file { + base: qapi_qmp::BlockdevOptionsBase { + auto_read_only: None, + cache: if is_direct.is_none() { + None + } else { + Some(qapi_qmp::BlockdevCacheOptions { + direct: is_direct, + no_flush: None, + }) + }, + detect_zeroes: None, + discard: None, + force_share: None, + node_name: None, + read_only: Some(is_readonly), + }, + file: qapi_qmp::BlockdevOptionsFile { + aio: None, + aio_max_batch: None, + drop_cache: if !no_drop { None } else { Some(no_drop) }, + locking: None, + pr_manager: None, + x_check_cache_dropped: None, + filename: path_on_host.to_owned(), + }, + })), + }, + offset: None, + size: None, + }, + })) + .map_err(|e| anyhow!("blockdev_add {:?}", e)) + .map(|_| ())?; + + // `device_add` + let mut blkdev_add_args = Dictionary::new(); + blkdev_add_args.insert("addr".to_owned(), format!("{:02}", slot).into()); + blkdev_add_args.insert("drive".to_owned(), node_name.clone().into()); + self.qmp + .execute(&qmp::device_add { + bus: Some(bus), + id: Some(node_name.clone()), + driver: block_driver.to_string(), + arguments: blkdev_add_args, + }) + .map_err(|e| anyhow!("device_add {:?}", e)) + .map(|_| ())?; + + let pci_path = self + .get_device_by_qdev_id(&node_name) + .context("get device by qdev_id failed")?; + info!( + sl!(), + "hotplug_block_device return pci path: {:?}", &pci_path + ); + + Ok(Some(pci_path)) + } } fn vcpu_id_from_core_id(core_id: i64) -> String { From 378d04bdf01c9a3d69f752bf6ffda475df4ab7a8 Mon Sep 17 00:00:00 2001 From: "alex.lyn" Date: Sun, 27 Apr 2025 11:29:40 +0800 Subject: [PATCH 3/3] runtime-rs: Add hotplug block device type with QMP There's several cases that block device plays very import roles: 1. Direct Volume: In Kata cases, to achieve high-performance I/O, raw files on the host are typically passed directly to the Guest via virtio-blk, and then bond/mounted within the Guest for container usage. 2. Trusted Storage In CoCo scenarios, particularly in Guest image pull mode, images are typically pulled directly from the registry within the Guest. However, due to constrained memory resources (prioritized for containers), CoCo leverages externally attached encrypted storage to store images, requiring hot-plug capability for block devices. and as other vmms, like dragonball and cloud-hypervisor in runtime-rs or qemu in kata-runtime have already supported such capabilities, we need support block device with hot-plug method (QMP) in qemu-rs. Let's do it. Fixes #11143 Signed-off-by: alex.lyn --- src/runtime-rs/crates/hypervisor/src/qemu/inner.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index 1087e1295a..30b8f215e0 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -618,6 +618,20 @@ impl QemuInner { )?; qmp.hotplug_network_device(&netdev, &virtio_net_device)? } + DeviceType::Block(mut block_device) => { + block_device.config.pci_path = qmp + .hotplug_block_device( + &self.config.blockdev_info.block_device_driver, + &block_device.device_id, + &block_device.config.path_on_host, + block_device.config.is_direct, + block_device.config.is_readonly, + block_device.config.no_drop, + ) + .context("hotplug block device")?; + + return Ok(DeviceType::Block(block_device)); + } _ => info!(sl!(), "hotplugging of {:#?} is unsupported", device), } Ok(device)