From 73e31ea19af10851a9c0405be9c80f8c1974d629 Mon Sep 17 00:00:00 2001 From: Fupan Li Date: Sun, 7 Sep 2025 15:15:17 +0800 Subject: [PATCH] runtime-rs: add the block devices io limit support Given that Rust-based VMMs like cloud-hypervisor, Firecracker, and Dragonball naturally offer user-level block I/O rate limiting, I/O throttling has been implemented to leverage this capability for these VMMs. This PR specifically introduces support for cloud-hypervisor. Signed-off-by: Fupan Li --- .../kata-types/src/config/hypervisor/mod.rs | 13 ++ .../configuration-cloud-hypervisor.toml.in | 30 ++++ .../crates/hypervisor/ch-config/src/lib.rs | 2 +- .../ch-config/src/virtio_devices.rs | 133 ++++++++++++++++++ .../crates/hypervisor/src/ch/inner_device.rs | 16 ++- 5 files changed, 192 insertions(+), 2 deletions(-) diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index a4fdddeb5c..6fa16e66f0 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -173,6 +173,19 @@ pub struct BlockDeviceInfo { /// The default if not set is empty (all annotations rejected.) #[serde(default)] pub valid_vhost_user_store_paths: Vec, + + /// controls disk I/O bandwidth (size in bits/sec) + #[serde(default)] + pub disk_rate_limiter_bw_max_rate: u64, + /// increases the initial max rate + #[serde(default)] + pub disk_rate_limiter_bw_one_time_burst: Option, + /// controls disk I/O bandwidth (size in ops/sec) + #[serde(default)] + pub disk_rate_limiter_ops_max_rate: u64, + /// increases the initial max rate + #[serde(default)] + pub disk_rate_limiter_ops_one_time_burst: Option, } impl BlockDeviceInfo { diff --git a/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in b/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in index 10264b14d8..4226b0f68d 100644 --- a/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in +++ b/src/runtime-rs/config/configuration-cloud-hypervisor.toml.in @@ -178,6 +178,36 @@ block_device_driver = "virtio-blk-pci" # Default false #block_device_cache_direct = true +# Bandwidth rate limiter options +# +# disk_rate_limiter_bw_max_rate controls disk I/O bandwidth (size in bits/sec +# for SB/VM). +# The same value is used for inbound and outbound bandwidth. +# Default 0-sized value means unlimited rate. +#disk_rate_limiter_bw_max_rate = 0 +# +# disk_rate_limiter_bw_one_time_burst increases the initial max rate and this +# initial extra credit does *NOT* affect the overall limit and can be used for +# an *initial* burst of data. +# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is +# set to a non zero value. +#disk_rate_limiter_bw_one_time_burst = 0 +# +# Operation rate limiter options +# +# disk_rate_limiter_ops_max_rate controls disk I/O bandwidth (size in ops/sec +# for SB/VM). +# The same value is used for inbound and outbound bandwidth. +# Default 0-sized value means unlimited rate. +#disk_rate_limiter_ops_max_rate = 0 +# +# disk_rate_limiter_ops_one_time_burst increases the initial max rate and this +# initial extra credit does *NOT* affect the overall limit and can be used for +# an *initial* burst of data. +# This is *optional* and only takes effect if disk_rate_limiter_bw_max_rate is +# set to a non zero value. +#disk_rate_limiter_ops_one_time_burst = 0 + # Enable pre allocation of VM RAM, default false # Enabling this will result in lower container density # as all of the memory will be allocated and locked diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs index 54f74648f1..6538f57d55 100644 --- a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs @@ -11,7 +11,7 @@ pub mod convert; pub mod net_util; mod virtio_devices; -use crate::virtio_devices::RateLimiterConfig; +pub use crate::virtio_devices::RateLimiterConfig; use kata_sys_util::protection::GuestProtection; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; pub use net_util::MacAddr; diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs index 02bf04bf96..0836f9b999 100644 --- a/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs @@ -4,6 +4,11 @@ use serde::{Deserialize, Serialize}; +// The DEFAULT_RATE_LIMITER_REFILL_TIME is used for calculating the rate at +// which a TokenBucket is replinished, in cases where a RateLimiter is +// applied to either network or disk I/O. +pub(crate) const DEFAULT_RATE_LIMITER_REFILL_TIME: u64 = 1000; + #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] pub struct TokenBucketConfig { pub size: u64, @@ -17,3 +22,131 @@ pub struct RateLimiterConfig { pub bandwidth: Option, pub ops: Option, } + +impl RateLimiterConfig { + /// Helper function: Creates a `TokenBucketConfig` based on the provided rate and burst. + /// Returns `None` if the `rate` is 0. + fn create_token_bucket_config( + rate: u64, + one_time_burst: Option, + ) -> Option { + if rate > 0 { + Some(TokenBucketConfig { + size: rate, + one_time_burst, + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME, + }) + } else { + None + } + } + + /// Creates a new `RateLimiterConfig` instance. + /// + /// If both `band_rate` and `ops_rate` are 0 (indicating no rate limiting configured), + /// it returns `None`. Otherwise, it returns `Some(RateLimiterConfig)` containing + /// the configured options. + pub fn new( + band_rate: u64, + ops_rate: u64, + band_onetime_burst: Option, + ops_onetime_burst: Option, + ) -> Option { + // Use the helper function to create `TokenBucketConfig` for bandwidth and ops + let bandwidth = Self::create_token_bucket_config(band_rate, band_onetime_burst); + let ops = Self::create_token_bucket_config(ops_rate, ops_onetime_burst); + + // Use pattern matching to concisely handle the final `Option` return. + // If both bandwidth and ops are `None`, the entire config is `None`. + // Otherwise, return `Some` with the actual configured options. + match (bandwidth, ops) { + (None, None) => None, + (b, o) => Some(RateLimiterConfig { + bandwidth: b, + ops: o, + }), + } + } +} + +// Unit tests +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new_all_set() { + let config = RateLimiterConfig::new(100, 50, Some(10), Some(5)).unwrap(); + assert_eq!( + config.bandwidth, + Some(TokenBucketConfig { + size: 100, + one_time_burst: Some(10), + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME + }) + ); + assert_eq!( + config.ops, + Some(TokenBucketConfig { + size: 50, + one_time_burst: Some(5), + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME + }) + ); + } + + #[test] + fn test_new_bandwidth_only() { + let config = RateLimiterConfig::new(100, 0, Some(10), None).unwrap(); + assert_eq!( + config.bandwidth, + Some(TokenBucketConfig { + size: 100, + one_time_burst: Some(10), + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME + }) + ); + assert_eq!(config.ops, None); + } + + #[test] + fn test_new_ops_only() { + let config = RateLimiterConfig::new(0, 50, None, Some(5)).unwrap(); + assert_eq!(config.bandwidth, None); + assert_eq!( + config.ops, + Some(TokenBucketConfig { + size: 50, + one_time_burst: Some(5), + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME + }) + ); + } + + #[test] + fn test_new_no_burst() { + let config = RateLimiterConfig::new(100, 50, None, None).unwrap(); + assert_eq!( + config.bandwidth, + Some(TokenBucketConfig { + size: 100, + one_time_burst: None, + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME + }) + ); + assert_eq!( + config.ops, + Some(TokenBucketConfig { + size: 50, + one_time_burst: None, + refill_time: DEFAULT_RATE_LIMITER_REFILL_TIME + }) + ); + } + + #[test] + fn test_new_none_set() { + let config = RateLimiterConfig::new(0, 0, None, None); + assert_eq!(config, None); + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs index 80c1fa8184..75009ba94d 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs @@ -24,7 +24,9 @@ use ch_config::ch_api::{ }; use ch_config::convert::{DEFAULT_DISK_QUEUES, DEFAULT_DISK_QUEUE_SIZE, DEFAULT_NUM_PCI_SEGMENTS}; use ch_config::DiskConfig; -use ch_config::{net_util::MacAddr, DeviceConfig, FsConfig, NetConfig, VsockConfig}; +use ch_config::{ + net_util::MacAddr, DeviceConfig, FsConfig, NetConfig, RateLimiterConfig, VsockConfig, +}; use safe_path::scoped_join; use std::convert::TryFrom; use std::path::PathBuf; @@ -322,6 +324,18 @@ impl CloudHypervisorInner { .is_direct .unwrap_or(self.config.blockdev_info.block_device_cache_direct); + let block_rate_limit = RateLimiterConfig::new( + self.config.blockdev_info.disk_rate_limiter_bw_max_rate, + self.config.blockdev_info.disk_rate_limiter_ops_max_rate, + self.config + .blockdev_info + .disk_rate_limiter_bw_one_time_burst, + self.config + .blockdev_info + .disk_rate_limiter_ops_one_time_burst, + ); + disk_config.rate_limiter_config = block_rate_limit; + let response = cloud_hypervisor_vm_blockdev_add( socket.try_clone().context("failed to clone socket")?, disk_config,