github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2026-07-01 22:50:54 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #12832 from Apokleos/indep-iothreads

Browse Source 
runtime-rs: Add support Independent iothreads
This commit is contained in:
Fabiano Fidêncio
2026-06-12 00:24:41 +02:00
committed by GitHub
parent 8f5b898e6d 854e76fb47
commit 780c242bfd
16 changed files with 168 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/libs/kata-types/src/annotations/mod.rs
View File

@@ -99,9 +99,14 @@ pub const KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH: &str =
pub const KATA_ANNO_CFG_HYPERVISOR_JAILER_HASH: &str =
"io.katacontainers.config.hypervisor.jailer_hash";
/// A sandbox annotation to enable IO to be processed in a separate thread.
/// Supported currently for virtio-scsi driver.
/// Supported for the virtio-scsi driver, and also used for virtio-blk-pci when
/// combined with `KATA_ANNO_CFG_HYPERVISOR_INDEP_IO_THREADS`.
pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS: &str =
"io.katacontainers.config.hypervisor.enable_iothreads";
/// A sandbox annotation to specify the number of independent IO threads.
/// Used for virtio-blk-pci devices during hotplug.
pub const KATA_ANNO_CFG_HYPERVISOR_INDEP_IO_THREADS: &str =
"io.katacontainers.config.hypervisor.indep_iothreads";
/// The hash type used for assets verification
pub const KATA_ANNO_CFG_HYPERVISOR_ASSET_HASH_TYPE: &str =
"io.katacontainers.config.hypervisor.asset_hash_type";
@@ -570,6 +575,15 @@ impl Annotation {
return Err(bool_err);
}
},
KATA_ANNO_CFG_HYPERVISOR_INDEP_IO_THREADS => match self.get_value::<u32>(key) {
Ok(r) => {
let indep_iothreads = r.unwrap_or_default();
hv.indep_iothreads = indep_iothreads;
}
Err(_e) => {
return Err(u32_err);
}
},
// Hypervisor Block Device related annotations
KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER => {
hv.blockdev_info.block_device_driver = value.to_string();

11
src/libs/kata-types/src/config/hypervisor/mod.rs
View File

@@ -1720,11 +1720,20 @@ pub struct Hypervisor {
/// Enables the use of iothreads (data-plane).
///
/// This is currently implemented for SCSI devices and for virtio-blk-pci devices
/// that support hotplug when `indep_iothreads` is greater than 0.
/// When enabled, I/O operations are handled in a separate I/O thread.
/// This is currently only implemented for SCSI devices.
#[serde(default)]
pub enable_iothreads: bool,
/// Number of independent IO threads for virtio-blk-pci devices.
///
/// When set to a value greater than 0, creates independent IO threads
/// that can be attached to virtio-blk-pci devices during hotplug.
/// Requires enable_iothreads to be true for virtio-blk-pci devices to use these threads.
#[serde(default)]
pub indep_iothreads: u32,
/// Block device configuration information.
#[serde(default, flatten)]
pub blockdev_info: BlockDeviceInfo,

19
src/libs/kata-types/tests/test_config.rs
View File

@@ -14,12 +14,13 @@ mod tests {
KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS,
KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR,
KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH,
KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC,
KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_PATH,
KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM,
KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_RUNTIME_NAME,
KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_INDEP_IO_THREADS,
KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH,
KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS,
KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES,
KATA_ANNO_CFG_RUNTIME_NAME,
};
use kata_types::config::KataConfig;
use kata_types::config::{QemuConfig, TomlConfig};
@@ -122,6 +123,10 @@ mod tests {
KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS.to_string(),
"false".to_string(),
);
anno_hash.insert(
KATA_ANNO_CFG_HYPERVISOR_INDEP_IO_THREADS.to_string(),
"3".to_string(),
);
anno_hash.insert(
KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS.to_string(),
"false".to_string(),
@@ -192,7 +197,7 @@ mod tests {
assert!(!hv.memory_info.enable_guest_swap);
assert_eq!(hv.memory_info.default_memory, 100);
assert!(!hv.enable_iothreads);
assert!(!hv.enable_iothreads);
assert_eq!(hv.indep_iothreads, 3);
assert_eq!(
hv.memory_info.file_mem_backend,
"./test_file_backend_mem_root"

2
src/libs/kata-types/tests/texture/configuration-anno-0.toml
View File

@@ -19,7 +19,7 @@ default_maxvcpus = 64
machine_type = "q35"
confidential_guest = true
rootless = true
enable_annotations = ["shared_fs","path", "ctlpath","jailer_path","enable_iothreads","default_memory","memory_slots","enable_mem_prealloc","enable_hugepages","file_mem_backend","enable_virtio_mem","enable_guest_swap","default_vcpus","virtio_fs_extra_args","block_device_driver","vhost_user_store_path","kernel","guest_hook_path","block_device_cache_noflush","virtio_fs_daemon","blk_logical_sector_size","blk_physical_sector_size"]
enable_annotations = ["shared_fs","path", "ctlpath","jailer_path","enable_iothreads","indep_iothreads","default_memory","memory_slots","enable_mem_prealloc","enable_hugepages","file_mem_backend","enable_virtio_mem","enable_guest_swap","default_vcpus","virtio_fs_extra_args","block_device_driver","vhost_user_store_path","kernel","guest_hook_path","block_device_cache_noflush","virtio_fs_daemon","blk_logical_sector_size","blk_physical_sector_size"]
machine_accelerators="noapic"
default_bridges = 2
default_memory = 128

2
src/runtime-rs/Makefile
View File

@@ -204,6 +204,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024
# Make sure you quote args.
DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"-o\", \"announce_submounts\"]
DEFENABLEIOTHREADS := false
DEFINDEPIOTHREADS := 0
DEFENABLEVHOSTUSERSTORE := false
DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user
DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
@@ -710,6 +711,7 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS
USER_VARS += DEFENABLEANNOTATIONS
USER_VARS += DEFENABLEANNOTATIONS_COCO
USER_VARS += DEFENABLEIOTHREADS
USER_VARS += DEFINDEPIOTHREADS
USER_VARS += DEFSECCOMPSANDBOXPARAM
USER_VARS += DEFENABLEVHOSTUSERSTORE
USER_VARS += DEFVHOSTUSERSTOREPATH

12
src/runtime-rs/config/configuration-qemu-coco-dev-runtime-rs.toml.in
View File

@@ -271,11 +271,19 @@ block_device_logical_sector_size = 0
block_device_physical_sector_size = 0
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

12
src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in
View File

@@ -259,11 +259,19 @@ block_device_cache_direct = false
block_device_cache_noflush = false
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Virtio queue size. Size: byte. default 128
queue_size = 128

12
src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in
View File

@@ -300,11 +300,19 @@ block_device_cache_direct = false
block_device_cache_noflush = false
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Virtio queue size. Size: byte. default 128
queue_size = 128

12
src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in
View File

@@ -276,11 +276,19 @@ block_device_cache_direct = false
block_device_cache_noflush = false
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Virtio queue size. Size: byte. default 128
queue_size = 128

12
src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in
View File

@@ -256,11 +256,19 @@ block_device_logical_sector_size = 0
block_device_physical_sector_size = 0
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Virtio queue size. Size: byte. default 128
queue_size = 128

12
src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in
View File

@@ -255,11 +255,19 @@ block_device_logical_sector_size = 0
block_device_physical_sector_size = 0
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Virtio queue size. Size: byte. default 128
queue_size = 128

12
src/runtime-rs/config/configuration-qemu-snp-runtime-rs.toml.in
View File

@@ -292,11 +292,19 @@ block_device_logical_sector_size = 0
block_device_physical_sector_size = 0
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

10
src/runtime-rs/config/configuration-qemu-tdx-runtime-rs.toml.in
View File

@@ -270,10 +270,18 @@ block_device_physical_sector_size = 0
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
# for virtio-scsi. For virtio-blk-pci hotplug, use indep_iothreads instead.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Number of independent IO threads for virtio-blk-pci devices.
# When set to a value greater than 0, creates dedicated IO threads
# that are attached to virtio-blk-pci devices during hotplug.
# Requires enable_iothreads = true; has no effect otherwise.
# Default 0 (disabled).
#
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

20
src/runtime-rs/crates/hypervisor/src/qemu/cmdline_generator.rs
View File

@@ -15,7 +15,7 @@ use std::borrow::Cow;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use kata_types::config::hypervisor::VIRTIO_SCSI;
use kata_types::config::hypervisor::{VIRTIO_BLK_PCI, VIRTIO_SCSI};
use kata_types::rootless::is_rootless;
use serde::{Deserialize, Serialize};
use serde_json;
@@ -2631,6 +2631,11 @@ impl<'a> QemuCmdLine<'a> {
qemu_cmd_line.add_scsi_controller();
}
// Add independent IO threads only when hotplug uses virtio-blk-pci.
if config.blockdev_info.block_device_driver == VIRTIO_BLK_PCI {
qemu_cmd_line.add_indep_iothreads();
}
if config.device_info.reclaim_guest_freed_memory {
qemu_cmd_line.add_virtio_balloon();
}
@@ -2738,6 +2743,19 @@ impl<'a> QemuCmdLine<'a> {
self.devices.push(Box::new(virtio_scsi));
}
/// Add independent IO threads for virtio-blk-pci devices.
/// These threads can be attached to virtio-blk-pci devices during hotplug.
fn add_indep_iothreads(&mut self) {
// Only create independent IO threads if enable_iothreads is true and indep_iothreads > 0
if self.config.enable_iothreads && self.config.indep_iothreads > 0 {
for i in 0..self.config.indep_iothreads {
let iothread_id = format!("indep_iothread_{}", i);
let iothread = ObjectIoThread::new(&iothread_id);
self.devices.push(Box::new(iothread));
}
}
}
pub fn add_virtiofs_share(
&mut self,
virtiofsd_socket_path: &str,

21
src/runtime-rs/crates/hypervisor/src/qemu/inner.rs
View File

@@ -25,7 +25,7 @@ use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use kata_sys_util::netns::NetnsGuard;
use kata_types::build_path;
use kata_types::config::hypervisor::{RootlessUser, VIRTIO_BLK_CCW};
use kata_types::config::hypervisor::{RootlessUser, VIRTIO_BLK_CCW, VIRTIO_BLK_PCI};
use kata_types::rootless::is_rootless;
use kata_types::{
capabilities::{Capabilities, CapabilityBits},
@@ -949,6 +949,23 @@ impl QemuInner {
}
DeviceType::Block(mut block_device) => {
let block_driver = &self.config.blockdev_info.block_device_driver;
// Determine iothread for hotplugged virtio-blk-pci devices.
// Only attach iothread when:
// 1. enable_iothreads is true
// 2. indep_iothreads > 0
// 3. block driver is virtio-blk-pci
// 4. TODO: for more complex cases
let iothread = if self.config.enable_iothreads
&& self.config.indep_iothreads > 0
&& block_driver == VIRTIO_BLK_PCI
{
// Use the first independent iothread (indep_iothread_0)
Some("indep_iothread_0")
} else {
None
};
let (pci_path, addr_str) = qmp
.hotplug_block_device(
block_driver,
@@ -966,6 +983,7 @@ impl QemuInner {
block_device.config.logical_sector_size,
block_device.config.physical_sector_size,
&block_device.config.format,
iothread,
)
.context("hotplug block device")?;
@@ -1048,6 +1066,7 @@ impl QemuInner {
logical_sector_size,
physical_sector_size,
&BlockDeviceFormat::default(),
None,
)
.context("hotplug block device")?;

10
src/runtime-rs/crates/hypervisor/src/qemu/qmp.rs
View File

@@ -688,6 +688,7 @@ impl Qmp {
logical_block_size: u32,
physical_block_size: u32,
format: &BlockDeviceFormat,
iothread: Option<&str>,
) -> Result<(Option<PciPath>, Option<String>)> {
// `blockdev-add`
let node_name = format!("drive-{index}");
@@ -893,6 +894,15 @@ impl Qmp {
blkdev_add_args.insert("share-rw".to_string(), true.into());
}
// Add iothread parameter for virtio-blk devices if specified
if let Some(iothread_id) = iothread {
info!(
sl!(),
"hotplug_block_device(): attaching to iothread: {}", iothread_id
);
blkdev_add_args.insert("iothread".to_owned(), iothread_id.to_string().into());
}
info!(
sl!(),
"hotplug_block_device(): device_add arguments: bus: {}, id: {}, driver: {}, blkdev_add_args: {:#?}",