github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-01 03:46:34 +00:00
Code Issues Projects Releases Wiki Activity

Dragonball: add cpu resize ability

Browse Source 
Add cpu resize ability upon upcall communication channel. Runtime could
use ResizeVcpu VmmAction and pass the desired vCPU number to the
Dragonball hypervisor.
Dragonball will trigger the device manager service in guest kernel's
upcall server to do cpu resize.

Fixes: #6008
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>
This commit is contained in:
Chao Wu 2023-01-09 17:03:07 +08:00
parent 2b34f0a54f
commit 57c5e5629b
7 changed files with 178 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/dragonball/src/api/v1/vmm_action.rs
View File

@ -35,6 +35,9 @@ pub use crate::device_manager::virtio_net_dev_mgr::{
#[cfg(feature = "virtio-vsock")]
pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError};
#[cfg(feature = "hotplug")]
pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
use super::*;
/// Wrapper for all errors associated with VMM actions.
@ -44,9 +47,13 @@ pub enum VmmActionError {
#[error("the virtual machine instance ID is invalid")]
InvalidVMID,
/// VM doesn't exist and can't get VM information.
#[error("VM doesn't exist and can't get VM information")]
VmNotExist,
/// Failed to hotplug, due to Upcall not ready.
#[error("Upcall not ready, can't hotplug device.")]
UpcallNotReady,
UpcallServerNotReady,
/// The action `ConfigureBootSource` failed either because of bad user input or an internal
/// error.
@ -85,6 +92,11 @@ pub enum VmmActionError {
/// The action `InsertFsDevice` failed either because of bad user input or an internal error.
#[error("virtio-fs device error: {0}")]
FsDevice(#[source] FsDeviceError),
#[cfg(feature = "hotplug")]
/// The action `ResizeVcpu` Failed
#[error("vcpu resize error : {0}")]
ResizeVcpu(#[source] VcpuResizeError),
}
/// This enum represents the public interface of the VMM. Each action contains various
@ -156,6 +168,10 @@ pub enum VmmAction {
#[cfg(feature = "virtio-fs")]
/// Update fs rate limiter, after microVM start.
UpdateFsDevice(FsDeviceConfigUpdateInfo),
#[cfg(feature = "hotplug")]
/// Resize Vcpu number in the guest.
ResizeVcpu(VcpuResizeInfo),
}
/// The enum represents the response sent by the VMM in case of success. The response is either
@ -256,6 +272,8 @@ impl VmmService {
VmmAction::UpdateFsDevice(fs_update_cfg) => {
self.update_fs_rate_limiters(vmm, fs_update_cfg)
}
#[cfg(feature = "hotplug")]
VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg),
};
debug!("send vmm response: {:?}", response);
@ -462,8 +480,8 @@ impl VmmService {
let ctx = vm
.create_device_op_context(Some(event_mgr.epoll_manager()))
.map_err(|e| {
if let StartMicroVmError::UpcallNotReady = e {
return VmmActionError::UpcallNotReady;
if let StartMicroVmError::UpcallServerNotReady = e {
return VmmActionError::UpcallServerNotReady;
}
VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
})?;
@ -518,8 +536,8 @@ impl VmmService {
.map_err(|e| {
if let StartMicroVmError::MicroVMAlreadyRunning = e {
VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot)
} else if let StartMicroVmError::UpcallNotReady = e {
VmmActionError::UpcallNotReady
} else if let StartMicroVmError::UpcallServerNotReady = e {
VmmActionError::UpcallServerNotReady
} else {
VmmActionError::StartMicroVm(e)
}
@ -595,6 +613,37 @@ impl VmmService {
.map(|_| VmmData::Empty)
.map_err(VmmActionError::FsDevice)
}
#[cfg(feature = "hotplug")]
fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult {
if !cfg!(target_arch = "x86_64") {
// TODO: Arm need to support vcpu hotplug. issue: #6010
warn!("This arch do not support vm resize!");
return Ok(VmmData::Empty);
}
if !cfg!(feature = "dbs-upcall") {
warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature.");
return Ok(VmmData::Empty);
}
let vm = vmm.get_vm_mut().ok_or(VmmActionError::VmNotExist)?;
if !vm.is_vm_initialized() {
return Err(VmmActionError::ResizeVcpu(
VcpuResizeError::UpdateNotAllowedPreBoot,
));
}
vm.resize_vcpu(config, None).map_err(|e| {
if let VcpuResizeError::UpcallServerNotReady = e {
return VmmActionError::UpcallServerNotReady;
}
VmmActionError::ResizeVcpu(e)
})?;
Ok(VmmData::Empty)
}
}
fn handle_cpu_topology(

2
src/dragonball/src/error.rs
View File

@ -100,7 +100,7 @@ pub enum StartMicroVmError {
/// Upcall is not ready
#[error("the upcall client is not ready")]
UpcallNotReady,
UpcallServerNotReady,
/// Configuration passed in is invalidate.
#[error("invalid virtual machine configuration: {0} ")]

5
src/dragonball/src/vcpu/mod.rs
View File

@ -10,7 +10,10 @@ mod vcpu_manager;
#[cfg(target_arch = "x86_64")]
use dbs_arch::cpuid::VpmuFeatureLevel;
pub use vcpu_manager::{VcpuManager, VcpuManagerError};
pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo};
#[cfg(feature = "hotplug")]
pub use vcpu_manager::VcpuResizeError;
/// vcpu config collection
pub struct VcpuConfig {

14
src/dragonball/src/vcpu/vcpu_impl.rs
View File

@ -214,10 +214,20 @@ pub enum VcpuResponse {
CacheRevalidated,
}
#[derive(Debug, PartialEq)]
/// Vcpu Hotplug Result returned from the guest
pub enum VcpuResizeResult {
/// All vCPU hotplug / hot-unplug operations are successful
Success = 0,
/// vCPU hotplug / hot-unplug failed
Failed = 1,
}
/// List of events that the vcpu_state_sender can send.
pub enum VcpuStateEvent {
/// (result, response) for hotplug, result 0 means failure, 1 means success.
Hotplug((i32, u32)),
/// (result, response) for hotplug / hot-unplugged.
/// response records how many cpu has successfully being hotplugged / hot-unplugged.
Hotplug((VcpuResizeResult, u32)),
}
/// Wrapper over vCPU that hides the underlying interactions with the vCPU thread.

149
src/dragonball/src/vcpu/vcpu_manager.rs
View File

@ -29,7 +29,7 @@ use crate::address_space_manager::GuestAddressSpaceImpl;
use crate::api::v1::InstanceInfo;
use crate::kvm_context::KvmContext;
use crate::vcpu::vcpu_impl::{
Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResponse, VcpuStateEvent,
Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResizeResult, VcpuResponse, VcpuStateEvent,
};
use crate::vcpu::VcpuConfig;
use crate::vm::VmConfigInfo;
@ -113,11 +113,6 @@ pub enum VcpuManagerError {
#[error("vcpu internal error: {0}")]
Vcpu(#[source] VcpuError),
#[cfg(feature = "hotplug")]
/// vCPU resize error
#[error("resize vcpu error: {0}")]
VcpuResize(#[source] VcpuResizeError),
/// Kvm Ioctl Error
#[error("failure in issuing KVM ioctl command: {0}")]
Kvm(#[source] kvm_ioctls::Error),
@ -132,6 +127,10 @@ pub enum VcpuResizeError {
VcpuIsHotplugging,
/// Cannot update the configuration of the microvm pre boot.
#[error("resize vcpu operation is not allowed pre boot")]
UpdateNotAllowedPreBoot,
/// Cannot update the configuration of the microvm post boot.
#[error("resize vcpu operation is not allowed after boot")]
UpdateNotAllowedPostBoot,
@ -147,10 +146,24 @@ pub enum VcpuResizeError {
#[error("Removable vcpu not enough, removable vcpu num: {0}, number to remove: {1}, present vcpu count {2}")]
LackRemovableVcpus(u16, u16, u16),
#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
#[cfg(feature = "dbs-upcall")]
/// Cannot update the configuration by upcall channel.
#[error("cannot update the configuration by upcall channel: {0}")]
Upcall(#[source] dbs_upcall::UpcallClientError),
#[cfg(feature = "dbs-upcall")]
/// Cannot find upcall client
#[error("Cannot find upcall client")]
UpcallClientMissing,
#[cfg(feature = "dbs-upcall")]
/// Upcall server is not ready
#[error("Upcall server is not ready")]
UpcallServerNotReady,
/// Vcpu manager error
#[error("Vcpu manager error : {0}")]
Vcpu(#[source] VcpuManagerError),
}
/// Result for vCPU manager operations
@ -163,6 +176,13 @@ enum VcpuAction {
Hotunplug,
}
/// VcpuResizeInfo describes the information for vcpu hotplug / hot-unplug
#[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct VcpuResizeInfo {
/// The desired vcpu count to resize.
pub vcpu_count: Option<u8>,
}
/// Infos related to per vcpu
#[derive(Default)]
pub(crate) struct VcpuInfo {
@ -810,11 +830,9 @@ mod hotplug {
&mut self,
vcpu_count: u8,
sync_tx: Option<Sender<bool>>,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
if self.get_vcpus_action() != VcpuAction::None {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::VcpuIsHotplugging,
));
return Err(VcpuResizeError::VcpuIsHotplugging);
}
self.action_sycn_tx = sync_tx;
@ -831,9 +849,7 @@ mod hotplug {
Ordering::Less => self.do_del_vcpu(vcpu_count, upcall),
}
} else {
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::UpdateNotAllowedPostBoot,
))
Err(VcpuResizeError::UpdateNotAllowedPostBoot)
}
}
@ -841,28 +857,31 @@ mod hotplug {
&mut self,
vcpu_count: u8,
upcall_client: Arc<UpcallClient<DevMgrService>>,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
info!("resize vcpu: add");
if vcpu_count > self.vcpu_config.max_vcpu_count {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::ExpectedVcpuExceedMax,
));
return Err(VcpuResizeError::ExpectedVcpuExceedMax);
}
let created_vcpus = self.create_vcpus(vcpu_count, None, None)?;
let cpu_ids = self.activate_vcpus(vcpu_count, true).map_err(|e| {
// we need to rollback when activate vcpu error
error!("activate vcpu error, rollback! {:?}", e);
let activated_vcpus: Vec<u8> = created_vcpus
.iter()
.filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some())
.copied()
.collect();
if let Err(e) = self.exit_vcpus(&activated_vcpus) {
error!("try to rollback error, stop_vcpu: {:?}", e);
}
e
})?;
let created_vcpus = self
.create_vcpus(vcpu_count, None, None)
.map_err(VcpuResizeError::Vcpu)?;
let cpu_ids = self
.activate_vcpus(vcpu_count, true)
.map_err(|e| {
// we need to rollback when activate vcpu error
error!("activate vcpu error, rollback! {:?}", e);
let activated_vcpus: Vec<u8> = created_vcpus
.iter()
.filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some())
.copied()
.collect();
if let Err(e) = self.exit_vcpus(&activated_vcpus) {
error!("try to rollback error, stop_vcpu: {:?}", e);
}
e
})
.map_err(VcpuResizeError::Vcpu)?;
let mut cpu_ids_array = [0u8; (u8::MAX as usize) + 1];
cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
@ -882,23 +901,19 @@ mod hotplug {
&mut self,
vcpu_count: u8,
upcall_client: Arc<UpcallClient<DevMgrService>>,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
info!("resize vcpu: delete");
if vcpu_count == 0 {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::Vcpu0CanNotBeRemoved,
));
return Err(VcpuResizeError::Vcpu0CanNotBeRemoved);
}
let mut cpu_ids = self.calculate_removable_vcpus();
let cpu_num_to_be_del = (self.present_vcpus_count() - vcpu_count) as usize;
if cpu_num_to_be_del >= cpu_ids.len() {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::LackRemovableVcpus(
cpu_ids.len() as u16,
cpu_num_to_be_del as u16,
self.present_vcpus_count() as u16,
),
return Err(VcpuResizeError::LackRemovableVcpus(
cpu_ids.len() as u16,
cpu_num_to_be_del as u16,
self.present_vcpus_count() as u16,
));
}
@ -924,7 +939,7 @@ mod hotplug {
&self,
_upcall_client: Arc<UpcallClient<DevMgrService>>,
_request: DevMgrRequest,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
Ok(())
}
@ -933,7 +948,7 @@ mod hotplug {
&self,
upcall_client: Arc<UpcallClient<DevMgrService>>,
request: DevMgrRequest,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
// This is used to fix clippy warnings.
use dbs_upcall::{DevMgrResponse, UpcallClientRequest, UpcallClientResponse};
@ -946,9 +961,14 @@ mod hotplug {
Box::new(move |result| match result {
UpcallClientResponse::DevMgr(response) => {
if let DevMgrResponse::CpuDev(resp) = response {
let result: VcpuResizeResult = if resp.result == 0 {
VcpuResizeResult::Success
} else {
VcpuResizeResult::Failed
};
vcpu_state_sender
.send(VcpuStateEvent::Hotplug((
resp.result,
result,
resp.info.apic_id_index,
)))
.unwrap();
@ -957,7 +977,7 @@ mod hotplug {
}
UpcallClientResponse::UpcallReset => {
vcpu_state_sender
.send(VcpuStateEvent::Hotplug((0, 0)))
.send(VcpuStateEvent::Hotplug((VcpuResizeResult::Success, 0)))
.unwrap();
vcpu_state_event.write(1).unwrap();
}
@ -968,7 +988,6 @@ mod hotplug {
}),
)
.map_err(VcpuResizeError::Upcall)
.map_err(VcpuManagerError::VcpuResize)
}
/// Get removable vcpus.
@ -993,16 +1012,19 @@ impl VcpuEpollHandler {
while let Ok(event) = self.rx.try_recv() {
match event {
VcpuStateEvent::Hotplug((success, cpu_count)) => {
info!("get vcpu event, cpu_index {}", cpu_count);
self.process_cpu_action(success != 0, cpu_count);
info!(
"get vcpu event, cpu_index {} success {:?}",
cpu_count, success
);
self.process_cpu_action(success, cpu_count);
}
}
}
}
fn process_cpu_action(&self, success: bool, _cpu_index: u32) {
fn process_cpu_action(&self, result: VcpuResizeResult, _cpu_index: u32) {
let mut vcpu_manager = self.vcpu_manager.lock().unwrap();
if success {
if result == VcpuResizeResult::Success {
match vcpu_manager.get_vcpus_action() {
VcpuAction::Hotplug => {
// Notify hotplug success
@ -1362,12 +1384,7 @@ mod tests {
// vcpu is already in hotplug process
let res = vcpu_manager.resize_vcpu(1, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::VcpuIsHotplugging
))
));
assert!(matches!(res, Err(VcpuResizeError::VcpuIsHotplugging)));
// clear vcpus action
let cpu_ids = vec![0];
@ -1377,9 +1394,7 @@ mod tests {
let res = vcpu_manager.resize_vcpu(1, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::UpdateNotAllowedPostBoot
))
Err(VcpuResizeError::UpdateNotAllowedPostBoot)
));
// init upcall channel
@ -1397,20 +1412,10 @@ mod tests {
// exceeed max vcpu count
let res = vcpu_manager.resize_vcpu(4, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::ExpectedVcpuExceedMax
))
));
assert!(matches!(res, Err(VcpuResizeError::ExpectedVcpuExceedMax)));
// remove vcpu 0
let res = vcpu_manager.resize_vcpu(0, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::Vcpu0CanNotBeRemoved
))
));
assert!(matches!(res, Err(VcpuResizeError::Vcpu0CanNotBeRemoved)));
}
}

30
src/dragonball/src/vm/mod.rs
View File

@ -4,6 +4,7 @@
use std::io::{self, Read, Seek, SeekFrom};
use std::ops::Deref;
use std::os::unix::io::RawFd;
use std::sync::{Arc, Mutex, RwLock};
use dbs_address_space::AddressSpace;
@ -22,6 +23,8 @@ use vmm_sys_util::eventfd::EventFd;
#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
use dbs_upcall::{DevMgrService, UpcallClient};
#[cfg(feature = "hotplug")]
use std::sync::mpsc::Sender;
use crate::address_space_manager::{
AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl,
@ -35,6 +38,8 @@ use crate::event_manager::EventManager;
use crate::kvm_context::KvmContext;
use crate::resource_manager::ResourceManager;
use crate::vcpu::{VcpuManager, VcpuManagerError};
#[cfg(feature = "hotplug")]
use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
#[cfg(target_arch = "aarch64")]
use dbs_arch::gic::Error as GICError;
@ -795,7 +800,30 @@ impl Vm {
} else if self.is_upcall_client_ready() {
Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr))
} else {
Err(StartMicroVmError::UpcallNotReady)
Err(StartMicroVmError::UpcallServerNotReady)
}
}
/// Resize MicroVM vCPU number
#[cfg(feature = "hotplug")]
pub fn resize_vcpu(
&mut self,
config: VcpuResizeInfo,
sync_tx: Option<Sender<bool>>,
) -> std::result::Result<(), VcpuResizeError> {
if self.upcall_client().is_none() {
Err(VcpuResizeError::UpcallClientMissing)
} else if self.is_upcall_client_ready() {
if let Some(vcpu_count) = config.vcpu_count {
self.vcpu_manager()
.map_err(VcpuResizeError::Vcpu)?
.resize_vcpu(vcpu_count, sync_tx)?;
self.vm_config.vcpu_count = vcpu_count;
}
Ok(())
} else {
Err(VcpuResizeError::UpcallServerNotReady)
}
}

2
src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
View File

@ -314,7 +314,7 @@ impl VmmInstance {
return Ok(vmm_data);
}
Err(vmm_action_error) => {
if let VmmActionError::UpcallNotReady = vmm_action_error {
if let VmmActionError::UpcallServerNotReady = vmm_action_error {
std::thread::sleep(std::time::Duration::from_millis(10));
continue;
} else {