diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 45591e7ed1..94b4ca16d6 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -35,6 +35,9 @@ pub use crate::device_manager::virtio_net_dev_mgr::{ #[cfg(feature = "virtio-vsock")] pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; +#[cfg(feature = "hotplug")] +pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo}; + use super::*; /// Wrapper for all errors associated with VMM actions. @@ -44,9 +47,13 @@ pub enum VmmActionError { #[error("the virtual machine instance ID is invalid")] InvalidVMID, + /// VM doesn't exist and can't get VM information. + #[error("VM doesn't exist and can't get VM information")] + VmNotExist, + /// Failed to hotplug, due to Upcall not ready. #[error("Upcall not ready, can't hotplug device.")] - UpcallNotReady, + UpcallServerNotReady, /// The action `ConfigureBootSource` failed either because of bad user input or an internal /// error. @@ -85,6 +92,11 @@ pub enum VmmActionError { /// The action `InsertFsDevice` failed either because of bad user input or an internal error. #[error("virtio-fs device error: {0}")] FsDevice(#[source] FsDeviceError), + + #[cfg(feature = "hotplug")] + /// The action `ResizeVcpu` Failed + #[error("vcpu resize error : {0}")] + ResizeVcpu(#[source] VcpuResizeError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -156,6 +168,10 @@ pub enum VmmAction { #[cfg(feature = "virtio-fs")] /// Update fs rate limiter, after microVM start. UpdateFsDevice(FsDeviceConfigUpdateInfo), + + #[cfg(feature = "hotplug")] + /// Resize Vcpu number in the guest. + ResizeVcpu(VcpuResizeInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -256,6 +272,8 @@ impl VmmService { VmmAction::UpdateFsDevice(fs_update_cfg) => { self.update_fs_rate_limiters(vmm, fs_update_cfg) } + #[cfg(feature = "hotplug")] + VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg), }; debug!("send vmm response: {:?}", response); @@ -462,8 +480,8 @@ impl VmmService { let ctx = vm .create_device_op_context(Some(event_mgr.epoll_manager())) .map_err(|e| { - if let StartMicroVmError::UpcallNotReady = e { - return VmmActionError::UpcallNotReady; + if let StartMicroVmError::UpcallServerNotReady = e { + return VmmActionError::UpcallServerNotReady; } VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot) })?; @@ -518,8 +536,8 @@ impl VmmService { .map_err(|e| { if let StartMicroVmError::MicroVMAlreadyRunning = e { VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot) - } else if let StartMicroVmError::UpcallNotReady = e { - VmmActionError::UpcallNotReady + } else if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady } else { VmmActionError::StartMicroVm(e) } @@ -595,6 +613,37 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::FsDevice) } + + #[cfg(feature = "hotplug")] + fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult { + if !cfg!(target_arch = "x86_64") { + // TODO: Arm need to support vcpu hotplug. issue: #6010 + warn!("This arch do not support vm resize!"); + return Ok(VmmData::Empty); + } + + if !cfg!(feature = "dbs-upcall") { + warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature."); + return Ok(VmmData::Empty); + } + + let vm = vmm.get_vm_mut().ok_or(VmmActionError::VmNotExist)?; + + if !vm.is_vm_initialized() { + return Err(VmmActionError::ResizeVcpu( + VcpuResizeError::UpdateNotAllowedPreBoot, + )); + } + + vm.resize_vcpu(config, None).map_err(|e| { + if let VcpuResizeError::UpcallServerNotReady = e { + return VmmActionError::UpcallServerNotReady; + } + VmmActionError::ResizeVcpu(e) + })?; + + Ok(VmmData::Empty) + } } fn handle_cpu_topology( diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 9ad0b07923..35f092a501 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -100,7 +100,7 @@ pub enum StartMicroVmError { /// Upcall is not ready #[error("the upcall client is not ready")] - UpcallNotReady, + UpcallServerNotReady, /// Configuration passed in is invalidate. #[error("invalid virtual machine configuration: {0} ")] diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs index 5b8ed397ff..b04baf29f4 100644 --- a/src/dragonball/src/vcpu/mod.rs +++ b/src/dragonball/src/vcpu/mod.rs @@ -10,7 +10,10 @@ mod vcpu_manager; #[cfg(target_arch = "x86_64")] use dbs_arch::cpuid::VpmuFeatureLevel; -pub use vcpu_manager::{VcpuManager, VcpuManagerError}; +pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo}; + +#[cfg(feature = "hotplug")] +pub use vcpu_manager::VcpuResizeError; /// vcpu config collection pub struct VcpuConfig { diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index f6c1c2d4c4..1c21ea38b8 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -214,10 +214,20 @@ pub enum VcpuResponse { CacheRevalidated, } +#[derive(Debug, PartialEq)] +/// Vcpu Hotplug Result returned from the guest +pub enum VcpuResizeResult { + /// All vCPU hotplug / hot-unplug operations are successful + Success = 0, + /// vCPU hotplug / hot-unplug failed + Failed = 1, +} + /// List of events that the vcpu_state_sender can send. pub enum VcpuStateEvent { - /// (result, response) for hotplug, result 0 means failure, 1 means success. - Hotplug((i32, u32)), + /// (result, response) for hotplug / hot-unplugged. + /// response records how many cpu has successfully being hotplugged / hot-unplugged. + Hotplug((VcpuResizeResult, u32)), } /// Wrapper over vCPU that hides the underlying interactions with the vCPU thread. diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 2b076cd5ba..383f1f0a7a 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -29,7 +29,7 @@ use crate::address_space_manager::GuestAddressSpaceImpl; use crate::api::v1::InstanceInfo; use crate::kvm_context::KvmContext; use crate::vcpu::vcpu_impl::{ - Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResponse, VcpuStateEvent, + Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResizeResult, VcpuResponse, VcpuStateEvent, }; use crate::vcpu::VcpuConfig; use crate::vm::VmConfigInfo; @@ -113,11 +113,6 @@ pub enum VcpuManagerError { #[error("vcpu internal error: {0}")] Vcpu(#[source] VcpuError), - #[cfg(feature = "hotplug")] - /// vCPU resize error - #[error("resize vcpu error: {0}")] - VcpuResize(#[source] VcpuResizeError), - /// Kvm Ioctl Error #[error("failure in issuing KVM ioctl command: {0}")] Kvm(#[source] kvm_ioctls::Error), @@ -132,6 +127,10 @@ pub enum VcpuResizeError { VcpuIsHotplugging, /// Cannot update the configuration of the microvm pre boot. + #[error("resize vcpu operation is not allowed pre boot")] + UpdateNotAllowedPreBoot, + + /// Cannot update the configuration of the microvm post boot. #[error("resize vcpu operation is not allowed after boot")] UpdateNotAllowedPostBoot, @@ -147,10 +146,24 @@ pub enum VcpuResizeError { #[error("Removable vcpu not enough, removable vcpu num: {0}, number to remove: {1}, present vcpu count {2}")] LackRemovableVcpus(u16, u16, u16), - #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] + #[cfg(feature = "dbs-upcall")] /// Cannot update the configuration by upcall channel. #[error("cannot update the configuration by upcall channel: {0}")] Upcall(#[source] dbs_upcall::UpcallClientError), + + #[cfg(feature = "dbs-upcall")] + /// Cannot find upcall client + #[error("Cannot find upcall client")] + UpcallClientMissing, + + #[cfg(feature = "dbs-upcall")] + /// Upcall server is not ready + #[error("Upcall server is not ready")] + UpcallServerNotReady, + + /// Vcpu manager error + #[error("Vcpu manager error : {0}")] + Vcpu(#[source] VcpuManagerError), } /// Result for vCPU manager operations @@ -163,6 +176,13 @@ enum VcpuAction { Hotunplug, } +/// VcpuResizeInfo describes the information for vcpu hotplug / hot-unplug +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct VcpuResizeInfo { + /// The desired vcpu count to resize. + pub vcpu_count: Option, +} + /// Infos related to per vcpu #[derive(Default)] pub(crate) struct VcpuInfo { @@ -810,11 +830,9 @@ mod hotplug { &mut self, vcpu_count: u8, sync_tx: Option>, - ) -> std::result::Result<(), VcpuManagerError> { + ) -> std::result::Result<(), VcpuResizeError> { if self.get_vcpus_action() != VcpuAction::None { - return Err(VcpuManagerError::VcpuResize( - VcpuResizeError::VcpuIsHotplugging, - )); + return Err(VcpuResizeError::VcpuIsHotplugging); } self.action_sycn_tx = sync_tx; @@ -831,9 +849,7 @@ mod hotplug { Ordering::Less => self.do_del_vcpu(vcpu_count, upcall), } } else { - Err(VcpuManagerError::VcpuResize( - VcpuResizeError::UpdateNotAllowedPostBoot, - )) + Err(VcpuResizeError::UpdateNotAllowedPostBoot) } } @@ -841,28 +857,31 @@ mod hotplug { &mut self, vcpu_count: u8, upcall_client: Arc>, - ) -> std::result::Result<(), VcpuManagerError> { + ) -> std::result::Result<(), VcpuResizeError> { info!("resize vcpu: add"); if vcpu_count > self.vcpu_config.max_vcpu_count { - return Err(VcpuManagerError::VcpuResize( - VcpuResizeError::ExpectedVcpuExceedMax, - )); + return Err(VcpuResizeError::ExpectedVcpuExceedMax); } - let created_vcpus = self.create_vcpus(vcpu_count, None, None)?; - let cpu_ids = self.activate_vcpus(vcpu_count, true).map_err(|e| { - // we need to rollback when activate vcpu error - error!("activate vcpu error, rollback! {:?}", e); - let activated_vcpus: Vec = created_vcpus - .iter() - .filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some()) - .copied() - .collect(); - if let Err(e) = self.exit_vcpus(&activated_vcpus) { - error!("try to rollback error, stop_vcpu: {:?}", e); - } - e - })?; + let created_vcpus = self + .create_vcpus(vcpu_count, None, None) + .map_err(VcpuResizeError::Vcpu)?; + let cpu_ids = self + .activate_vcpus(vcpu_count, true) + .map_err(|e| { + // we need to rollback when activate vcpu error + error!("activate vcpu error, rollback! {:?}", e); + let activated_vcpus: Vec = created_vcpus + .iter() + .filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some()) + .copied() + .collect(); + if let Err(e) = self.exit_vcpus(&activated_vcpus) { + error!("try to rollback error, stop_vcpu: {:?}", e); + } + e + }) + .map_err(VcpuResizeError::Vcpu)?; let mut cpu_ids_array = [0u8; (u8::MAX as usize) + 1]; cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]); @@ -882,23 +901,19 @@ mod hotplug { &mut self, vcpu_count: u8, upcall_client: Arc>, - ) -> std::result::Result<(), VcpuManagerError> { + ) -> std::result::Result<(), VcpuResizeError> { info!("resize vcpu: delete"); if vcpu_count == 0 { - return Err(VcpuManagerError::VcpuResize( - VcpuResizeError::Vcpu0CanNotBeRemoved, - )); + return Err(VcpuResizeError::Vcpu0CanNotBeRemoved); } let mut cpu_ids = self.calculate_removable_vcpus(); let cpu_num_to_be_del = (self.present_vcpus_count() - vcpu_count) as usize; if cpu_num_to_be_del >= cpu_ids.len() { - return Err(VcpuManagerError::VcpuResize( - VcpuResizeError::LackRemovableVcpus( - cpu_ids.len() as u16, - cpu_num_to_be_del as u16, - self.present_vcpus_count() as u16, - ), + return Err(VcpuResizeError::LackRemovableVcpus( + cpu_ids.len() as u16, + cpu_num_to_be_del as u16, + self.present_vcpus_count() as u16, )); } @@ -924,7 +939,7 @@ mod hotplug { &self, _upcall_client: Arc>, _request: DevMgrRequest, - ) -> std::result::Result<(), VcpuManagerError> { + ) -> std::result::Result<(), VcpuResizeError> { Ok(()) } @@ -933,7 +948,7 @@ mod hotplug { &self, upcall_client: Arc>, request: DevMgrRequest, - ) -> std::result::Result<(), VcpuManagerError> { + ) -> std::result::Result<(), VcpuResizeError> { // This is used to fix clippy warnings. use dbs_upcall::{DevMgrResponse, UpcallClientRequest, UpcallClientResponse}; @@ -946,9 +961,14 @@ mod hotplug { Box::new(move |result| match result { UpcallClientResponse::DevMgr(response) => { if let DevMgrResponse::CpuDev(resp) = response { + let result: VcpuResizeResult = if resp.result == 0 { + VcpuResizeResult::Success + } else { + VcpuResizeResult::Failed + }; vcpu_state_sender .send(VcpuStateEvent::Hotplug(( - resp.result, + result, resp.info.apic_id_index, ))) .unwrap(); @@ -957,7 +977,7 @@ mod hotplug { } UpcallClientResponse::UpcallReset => { vcpu_state_sender - .send(VcpuStateEvent::Hotplug((0, 0))) + .send(VcpuStateEvent::Hotplug((VcpuResizeResult::Success, 0))) .unwrap(); vcpu_state_event.write(1).unwrap(); } @@ -968,7 +988,6 @@ mod hotplug { }), ) .map_err(VcpuResizeError::Upcall) - .map_err(VcpuManagerError::VcpuResize) } /// Get removable vcpus. @@ -993,16 +1012,19 @@ impl VcpuEpollHandler { while let Ok(event) = self.rx.try_recv() { match event { VcpuStateEvent::Hotplug((success, cpu_count)) => { - info!("get vcpu event, cpu_index {}", cpu_count); - self.process_cpu_action(success != 0, cpu_count); + info!( + "get vcpu event, cpu_index {} success {:?}", + cpu_count, success + ); + self.process_cpu_action(success, cpu_count); } } } } - fn process_cpu_action(&self, success: bool, _cpu_index: u32) { + fn process_cpu_action(&self, result: VcpuResizeResult, _cpu_index: u32) { let mut vcpu_manager = self.vcpu_manager.lock().unwrap(); - if success { + if result == VcpuResizeResult::Success { match vcpu_manager.get_vcpus_action() { VcpuAction::Hotplug => { // Notify hotplug success @@ -1362,12 +1384,7 @@ mod tests { // vcpu is already in hotplug process let res = vcpu_manager.resize_vcpu(1, None); - assert!(matches!( - res, - Err(VcpuManagerError::VcpuResize( - VcpuResizeError::VcpuIsHotplugging - )) - )); + assert!(matches!(res, Err(VcpuResizeError::VcpuIsHotplugging))); // clear vcpus action let cpu_ids = vec![0]; @@ -1377,9 +1394,7 @@ mod tests { let res = vcpu_manager.resize_vcpu(1, None); assert!(matches!( res, - Err(VcpuManagerError::VcpuResize( - VcpuResizeError::UpdateNotAllowedPostBoot - )) + Err(VcpuResizeError::UpdateNotAllowedPostBoot) )); // init upcall channel @@ -1397,20 +1412,10 @@ mod tests { // exceeed max vcpu count let res = vcpu_manager.resize_vcpu(4, None); - assert!(matches!( - res, - Err(VcpuManagerError::VcpuResize( - VcpuResizeError::ExpectedVcpuExceedMax - )) - )); + assert!(matches!(res, Err(VcpuResizeError::ExpectedVcpuExceedMax))); // remove vcpu 0 let res = vcpu_manager.resize_vcpu(0, None); - assert!(matches!( - res, - Err(VcpuManagerError::VcpuResize( - VcpuResizeError::Vcpu0CanNotBeRemoved - )) - )); + assert!(matches!(res, Err(VcpuResizeError::Vcpu0CanNotBeRemoved))); } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index d573080ae8..9a39a3d533 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -4,6 +4,7 @@ use std::io::{self, Read, Seek, SeekFrom}; use std::ops::Deref; use std::os::unix::io::RawFd; + use std::sync::{Arc, Mutex, RwLock}; use dbs_address_space::AddressSpace; @@ -22,6 +23,8 @@ use vmm_sys_util::eventfd::EventFd; #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] use dbs_upcall::{DevMgrService, UpcallClient}; +#[cfg(feature = "hotplug")] +use std::sync::mpsc::Sender; use crate::address_space_manager::{ AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl, @@ -35,6 +38,8 @@ use crate::event_manager::EventManager; use crate::kvm_context::KvmContext; use crate::resource_manager::ResourceManager; use crate::vcpu::{VcpuManager, VcpuManagerError}; +#[cfg(feature = "hotplug")] +use crate::vcpu::{VcpuResizeError, VcpuResizeInfo}; #[cfg(target_arch = "aarch64")] use dbs_arch::gic::Error as GICError; @@ -795,7 +800,30 @@ impl Vm { } else if self.is_upcall_client_ready() { Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) } else { - Err(StartMicroVmError::UpcallNotReady) + Err(StartMicroVmError::UpcallServerNotReady) + } + } + + /// Resize MicroVM vCPU number + #[cfg(feature = "hotplug")] + pub fn resize_vcpu( + &mut self, + config: VcpuResizeInfo, + sync_tx: Option>, + ) -> std::result::Result<(), VcpuResizeError> { + if self.upcall_client().is_none() { + Err(VcpuResizeError::UpcallClientMissing) + } else if self.is_upcall_client_ready() { + if let Some(vcpu_count) = config.vcpu_count { + self.vcpu_manager() + .map_err(VcpuResizeError::Vcpu)? + .resize_vcpu(vcpu_count, sync_tx)?; + + self.vm_config.vcpu_count = vcpu_count; + } + Ok(()) + } else { + Err(VcpuResizeError::UpcallServerNotReady) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs index 00829ad4c9..9837ea6677 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs @@ -314,7 +314,7 @@ impl VmmInstance { return Ok(vmm_data); } Err(vmm_action_error) => { - if let VmmActionError::UpcallNotReady = vmm_action_error { + if let VmmActionError::UpcallServerNotReady = vmm_action_error { std::thread::sleep(std::time::Duration::from_millis(10)); continue; } else {