github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-09-17 23:07:55 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6009 from openanolis/dragonball/add_cpu_resize

Browse Source 
Dragonball: add cpu resize ability
This commit is contained in:
Bin Liu
2023-02-05 19:54:08 +08:00
committed by GitHub
parent 7fc35f19eb 57c5e5629b
commit df93439c3b
7 changed files with 178 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
src/dragonball/src/api/v1/vmm_action.rs
View File

@@ -35,6 +35,9 @@ pub use crate::device_manager::virtio_net_dev_mgr::{
#[cfg(feature = "virtio-vsock")]
pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError};
#[cfg(feature = "hotplug")]
pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
use super::*;
/// Wrapper for all errors associated with VMM actions.
@@ -44,9 +47,13 @@ pub enum VmmActionError {
#[error("the virtual machine instance ID is invalid")]
InvalidVMID,
/// VM doesn't exist and can't get VM information.
#[error("VM doesn't exist and can't get VM information")]
VmNotExist,
/// Failed to hotplug, due to Upcall not ready.
#[error("Upcall not ready, can't hotplug device.")]
UpcallNotReady,
UpcallServerNotReady,
/// The action `ConfigureBootSource` failed either because of bad user input or an internal
/// error.
@@ -85,6 +92,11 @@ pub enum VmmActionError {
/// The action `InsertFsDevice` failed either because of bad user input or an internal error.
#[error("virtio-fs device error: {0}")]
FsDevice(#[source] FsDeviceError),
#[cfg(feature = "hotplug")]
/// The action `ResizeVcpu` Failed
#[error("vcpu resize error : {0}")]
ResizeVcpu(#[source] VcpuResizeError),
}
/// This enum represents the public interface of the VMM. Each action contains various
@@ -156,6 +168,10 @@ pub enum VmmAction {
#[cfg(feature = "virtio-fs")]
/// Update fs rate limiter, after microVM start.
UpdateFsDevice(FsDeviceConfigUpdateInfo),
#[cfg(feature = "hotplug")]
/// Resize Vcpu number in the guest.
ResizeVcpu(VcpuResizeInfo),
}
/// The enum represents the response sent by the VMM in case of success. The response is either
@@ -256,6 +272,8 @@ impl VmmService {
VmmAction::UpdateFsDevice(fs_update_cfg) => {
self.update_fs_rate_limiters(vmm, fs_update_cfg)
}
#[cfg(feature = "hotplug")]
VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg),
};
debug!("send vmm response: {:?}", response);
@@ -462,8 +480,8 @@ impl VmmService {
let ctx = vm
.create_device_op_context(Some(event_mgr.epoll_manager()))
.map_err(|e| {
if let StartMicroVmError::UpcallNotReady = e {
return VmmActionError::UpcallNotReady;
if let StartMicroVmError::UpcallServerNotReady = e {
return VmmActionError::UpcallServerNotReady;
}
VmmActionError::Block(BlockDeviceError::UpdateNotAllowedPostBoot)
})?;
@@ -518,8 +536,8 @@ impl VmmService {
.map_err(|e| {
if let StartMicroVmError::MicroVMAlreadyRunning = e {
VmmActionError::VirtioNet(VirtioNetDeviceError::UpdateNotAllowedPostBoot)
} else if let StartMicroVmError::UpcallNotReady = e {
VmmActionError::UpcallNotReady
} else if let StartMicroVmError::UpcallServerNotReady = e {
VmmActionError::UpcallServerNotReady
} else {
VmmActionError::StartMicroVm(e)
}
@@ -595,6 +613,37 @@ impl VmmService {
.map(|_| VmmData::Empty)
.map_err(VmmActionError::FsDevice)
}
#[cfg(feature = "hotplug")]
fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult {
if !cfg!(target_arch = "x86_64") {
// TODO: Arm need to support vcpu hotplug. issue: #6010
warn!("This arch do not support vm resize!");
return Ok(VmmData::Empty);
}
if !cfg!(feature = "dbs-upcall") {
warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature.");
return Ok(VmmData::Empty);
}
let vm = vmm.get_vm_mut().ok_or(VmmActionError::VmNotExist)?;
if !vm.is_vm_initialized() {
return Err(VmmActionError::ResizeVcpu(
VcpuResizeError::UpdateNotAllowedPreBoot,
));
}
vm.resize_vcpu(config, None).map_err(|e| {
if let VcpuResizeError::UpcallServerNotReady = e {
return VmmActionError::UpcallServerNotReady;
}
VmmActionError::ResizeVcpu(e)
})?;
Ok(VmmData::Empty)
}
}
fn handle_cpu_topology(

2
src/dragonball/src/error.rs
View File

@@ -100,7 +100,7 @@ pub enum StartMicroVmError {
/// Upcall is not ready
#[error("the upcall client is not ready")]
UpcallNotReady,
UpcallServerNotReady,
/// Configuration passed in is invalidate.
#[error("invalid virtual machine configuration: {0} ")]

5
src/dragonball/src/vcpu/mod.rs
View File

@@ -10,7 +10,10 @@ mod vcpu_manager;
#[cfg(target_arch = "x86_64")]
use dbs_arch::cpuid::VpmuFeatureLevel;
pub use vcpu_manager::{VcpuManager, VcpuManagerError};
pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo};
#[cfg(feature = "hotplug")]
pub use vcpu_manager::VcpuResizeError;
/// vcpu config collection
pub struct VcpuConfig {

14
src/dragonball/src/vcpu/vcpu_impl.rs
View File

@@ -214,10 +214,20 @@ pub enum VcpuResponse {
CacheRevalidated,
}
#[derive(Debug, PartialEq)]
/// Vcpu Hotplug Result returned from the guest
pub enum VcpuResizeResult {
/// All vCPU hotplug / hot-unplug operations are successful
Success = 0,
/// vCPU hotplug / hot-unplug failed
Failed = 1,
}
/// List of events that the vcpu_state_sender can send.
pub enum VcpuStateEvent {
/// (result, response) for hotplug, result 0 means failure, 1 means success.
Hotplug((i32, u32)),
/// (result, response) for hotplug / hot-unplugged.
/// response records how many cpu has successfully being hotplugged / hot-unplugged.
Hotplug((VcpuResizeResult, u32)),
}
/// Wrapper over vCPU that hides the underlying interactions with the vCPU thread.

149
src/dragonball/src/vcpu/vcpu_manager.rs
View File

@@ -29,7 +29,7 @@ use crate::address_space_manager::GuestAddressSpaceImpl;
use crate::api::v1::InstanceInfo;
use crate::kvm_context::KvmContext;
use crate::vcpu::vcpu_impl::{
Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResponse, VcpuStateEvent,
Vcpu, VcpuError, VcpuEvent, VcpuHandle, VcpuResizeResult, VcpuResponse, VcpuStateEvent,
};
use crate::vcpu::VcpuConfig;
use crate::vm::VmConfigInfo;
@@ -113,11 +113,6 @@ pub enum VcpuManagerError {
#[error("vcpu internal error: {0}")]
Vcpu(#[source] VcpuError),
#[cfg(feature = "hotplug")]
/// vCPU resize error
#[error("resize vcpu error: {0}")]
VcpuResize(#[source] VcpuResizeError),
/// Kvm Ioctl Error
#[error("failure in issuing KVM ioctl command: {0}")]
Kvm(#[source] kvm_ioctls::Error),
@@ -132,6 +127,10 @@ pub enum VcpuResizeError {
VcpuIsHotplugging,
/// Cannot update the configuration of the microvm pre boot.
#[error("resize vcpu operation is not allowed pre boot")]
UpdateNotAllowedPreBoot,
/// Cannot update the configuration of the microvm post boot.
#[error("resize vcpu operation is not allowed after boot")]
UpdateNotAllowedPostBoot,
@@ -147,10 +146,24 @@ pub enum VcpuResizeError {
#[error("Removable vcpu not enough, removable vcpu num: {0}, number to remove: {1}, present vcpu count {2}")]
LackRemovableVcpus(u16, u16, u16),
#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
#[cfg(feature = "dbs-upcall")]
/// Cannot update the configuration by upcall channel.
#[error("cannot update the configuration by upcall channel: {0}")]
Upcall(#[source] dbs_upcall::UpcallClientError),
#[cfg(feature = "dbs-upcall")]
/// Cannot find upcall client
#[error("Cannot find upcall client")]
UpcallClientMissing,
#[cfg(feature = "dbs-upcall")]
/// Upcall server is not ready
#[error("Upcall server is not ready")]
UpcallServerNotReady,
/// Vcpu manager error
#[error("Vcpu manager error : {0}")]
Vcpu(#[source] VcpuManagerError),
}
/// Result for vCPU manager operations
@@ -163,6 +176,13 @@ enum VcpuAction {
Hotunplug,
}
/// VcpuResizeInfo describes the information for vcpu hotplug / hot-unplug
#[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct VcpuResizeInfo {
/// The desired vcpu count to resize.
pub vcpu_count: Option<u8>,
}
/// Infos related to per vcpu
#[derive(Default)]
pub(crate) struct VcpuInfo {
@@ -810,11 +830,9 @@ mod hotplug {
&mut self,
vcpu_count: u8,
sync_tx: Option<Sender<bool>>,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
if self.get_vcpus_action() != VcpuAction::None {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::VcpuIsHotplugging,
));
return Err(VcpuResizeError::VcpuIsHotplugging);
}
self.action_sycn_tx = sync_tx;
@@ -831,9 +849,7 @@ mod hotplug {
Ordering::Less => self.do_del_vcpu(vcpu_count, upcall),
}
} else {
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::UpdateNotAllowedPostBoot,
))
Err(VcpuResizeError::UpdateNotAllowedPostBoot)
}
}
@@ -841,28 +857,31 @@ mod hotplug {
&mut self,
vcpu_count: u8,
upcall_client: Arc<UpcallClient<DevMgrService>>,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
info!("resize vcpu: add");
if vcpu_count > self.vcpu_config.max_vcpu_count {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::ExpectedVcpuExceedMax,
));
return Err(VcpuResizeError::ExpectedVcpuExceedMax);
}
let created_vcpus = self.create_vcpus(vcpu_count, None, None)?;
let cpu_ids = self.activate_vcpus(vcpu_count, true).map_err(|e| {
// we need to rollback when activate vcpu error
error!("activate vcpu error, rollback! {:?}", e);
let activated_vcpus: Vec<u8> = created_vcpus
.iter()
.filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some())
.copied()
.collect();
if let Err(e) = self.exit_vcpus(&activated_vcpus) {
error!("try to rollback error, stop_vcpu: {:?}", e);
}
e
})?;
let created_vcpus = self
.create_vcpus(vcpu_count, None, None)
.map_err(VcpuResizeError::Vcpu)?;
let cpu_ids = self
.activate_vcpus(vcpu_count, true)
.map_err(|e| {
// we need to rollback when activate vcpu error
error!("activate vcpu error, rollback! {:?}", e);
let activated_vcpus: Vec<u8> = created_vcpus
.iter()
.filter(|&cpu_id| self.vcpu_infos[*cpu_id as usize].handle.is_some())
.copied()
.collect();
if let Err(e) = self.exit_vcpus(&activated_vcpus) {
error!("try to rollback error, stop_vcpu: {:?}", e);
}
e
})
.map_err(VcpuResizeError::Vcpu)?;
let mut cpu_ids_array = [0u8; (u8::MAX as usize) + 1];
cpu_ids_array[..cpu_ids.len()].copy_from_slice(&cpu_ids[..cpu_ids.len()]);
@@ -882,23 +901,19 @@ mod hotplug {
&mut self,
vcpu_count: u8,
upcall_client: Arc<UpcallClient<DevMgrService>>,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
info!("resize vcpu: delete");
if vcpu_count == 0 {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::Vcpu0CanNotBeRemoved,
));
return Err(VcpuResizeError::Vcpu0CanNotBeRemoved);
}
let mut cpu_ids = self.calculate_removable_vcpus();
let cpu_num_to_be_del = (self.present_vcpus_count() - vcpu_count) as usize;
if cpu_num_to_be_del >= cpu_ids.len() {
return Err(VcpuManagerError::VcpuResize(
VcpuResizeError::LackRemovableVcpus(
cpu_ids.len() as u16,
cpu_num_to_be_del as u16,
self.present_vcpus_count() as u16,
),
return Err(VcpuResizeError::LackRemovableVcpus(
cpu_ids.len() as u16,
cpu_num_to_be_del as u16,
self.present_vcpus_count() as u16,
));
}
@@ -924,7 +939,7 @@ mod hotplug {
&self,
_upcall_client: Arc<UpcallClient<DevMgrService>>,
_request: DevMgrRequest,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
Ok(())
}
@@ -933,7 +948,7 @@ mod hotplug {
&self,
upcall_client: Arc<UpcallClient<DevMgrService>>,
request: DevMgrRequest,
) -> std::result::Result<(), VcpuManagerError> {
) -> std::result::Result<(), VcpuResizeError> {
// This is used to fix clippy warnings.
use dbs_upcall::{DevMgrResponse, UpcallClientRequest, UpcallClientResponse};
@@ -946,9 +961,14 @@ mod hotplug {
Box::new(move |result| match result {
UpcallClientResponse::DevMgr(response) => {
if let DevMgrResponse::CpuDev(resp) = response {
let result: VcpuResizeResult = if resp.result == 0 {
VcpuResizeResult::Success
} else {
VcpuResizeResult::Failed
};
vcpu_state_sender
.send(VcpuStateEvent::Hotplug((
resp.result,
result,
resp.info.apic_id_index,
)))
.unwrap();
@@ -957,7 +977,7 @@ mod hotplug {
}
UpcallClientResponse::UpcallReset => {
vcpu_state_sender
.send(VcpuStateEvent::Hotplug((0, 0)))
.send(VcpuStateEvent::Hotplug((VcpuResizeResult::Success, 0)))
.unwrap();
vcpu_state_event.write(1).unwrap();
}
@@ -968,7 +988,6 @@ mod hotplug {
}),
)
.map_err(VcpuResizeError::Upcall)
.map_err(VcpuManagerError::VcpuResize)
}
/// Get removable vcpus.
@@ -993,16 +1012,19 @@ impl VcpuEpollHandler {
while let Ok(event) = self.rx.try_recv() {
match event {
VcpuStateEvent::Hotplug((success, cpu_count)) => {
info!("get vcpu event, cpu_index {}", cpu_count);
self.process_cpu_action(success != 0, cpu_count);
info!(
"get vcpu event, cpu_index {} success {:?}",
cpu_count, success
);
self.process_cpu_action(success, cpu_count);
}
}
}
}
fn process_cpu_action(&self, success: bool, _cpu_index: u32) {
fn process_cpu_action(&self, result: VcpuResizeResult, _cpu_index: u32) {
let mut vcpu_manager = self.vcpu_manager.lock().unwrap();
if success {
if result == VcpuResizeResult::Success {
match vcpu_manager.get_vcpus_action() {
VcpuAction::Hotplug => {
// Notify hotplug success
@@ -1362,12 +1384,7 @@ mod tests {
// vcpu is already in hotplug process
let res = vcpu_manager.resize_vcpu(1, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::VcpuIsHotplugging
))
));
assert!(matches!(res, Err(VcpuResizeError::VcpuIsHotplugging)));
// clear vcpus action
let cpu_ids = vec![0];
@@ -1377,9 +1394,7 @@ mod tests {
let res = vcpu_manager.resize_vcpu(1, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::UpdateNotAllowedPostBoot
))
Err(VcpuResizeError::UpdateNotAllowedPostBoot)
));
// init upcall channel
@@ -1397,20 +1412,10 @@ mod tests {
// exceeed max vcpu count
let res = vcpu_manager.resize_vcpu(4, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::ExpectedVcpuExceedMax
))
));
assert!(matches!(res, Err(VcpuResizeError::ExpectedVcpuExceedMax)));
// remove vcpu 0
let res = vcpu_manager.resize_vcpu(0, None);
assert!(matches!(
res,
Err(VcpuManagerError::VcpuResize(
VcpuResizeError::Vcpu0CanNotBeRemoved
))
));
assert!(matches!(res, Err(VcpuResizeError::Vcpu0CanNotBeRemoved)));
}
}

30
src/dragonball/src/vm/mod.rs
View File

@@ -4,6 +4,7 @@
use std::io::{self, Read, Seek, SeekFrom};
use std::ops::Deref;
use std::os::unix::io::RawFd;
use std::sync::{Arc, Mutex, RwLock};
use dbs_address_space::AddressSpace;
@@ -22,6 +23,8 @@ use vmm_sys_util::eventfd::EventFd;
#[cfg(all(feature = "hotplug", feature = "dbs-upcall"))]
use dbs_upcall::{DevMgrService, UpcallClient};
#[cfg(feature = "hotplug")]
use std::sync::mpsc::Sender;
use crate::address_space_manager::{
AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl,
@@ -35,6 +38,8 @@ use crate::event_manager::EventManager;
use crate::kvm_context::KvmContext;
use crate::resource_manager::ResourceManager;
use crate::vcpu::{VcpuManager, VcpuManagerError};
#[cfg(feature = "hotplug")]
use crate::vcpu::{VcpuResizeError, VcpuResizeInfo};
#[cfg(target_arch = "aarch64")]
use dbs_arch::gic::Error as GICError;
@@ -795,7 +800,30 @@ impl Vm {
} else if self.is_upcall_client_ready() {
Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr))
} else {
Err(StartMicroVmError::UpcallNotReady)
Err(StartMicroVmError::UpcallServerNotReady)
}
}
/// Resize MicroVM vCPU number
#[cfg(feature = "hotplug")]
pub fn resize_vcpu(
&mut self,
config: VcpuResizeInfo,
sync_tx: Option<Sender<bool>>,
) -> std::result::Result<(), VcpuResizeError> {
if self.upcall_client().is_none() {
Err(VcpuResizeError::UpcallClientMissing)
} else if self.is_upcall_client_ready() {
if let Some(vcpu_count) = config.vcpu_count {
self.vcpu_manager()
.map_err(VcpuResizeError::Vcpu)?
.resize_vcpu(vcpu_count, sync_tx)?;
self.vm_config.vcpu_count = vcpu_count;
}
Ok(())
} else {
Err(VcpuResizeError::UpcallServerNotReady)
}
}

2
src/runtime-rs/crates/hypervisor/src/dragonball/vmm_instance.rs
View File

@@ -314,7 +314,7 @@ impl VmmInstance {
return Ok(vmm_data);
}
Err(vmm_action_error) => {
if let VmmActionError::UpcallNotReady = vmm_action_error {
if let VmmActionError::UpcallServerNotReady = vmm_action_error {
std::thread::sleep(std::time::Duration::from_millis(10));
continue;
} else {