dragonball: add support resize_vcpu on aarch64

This commit add support of resize_vcpu on aarch64. As kvm will check
whether vgic is initialized when calling KVM_CREATE_VCPU ioctl, all the
vcpu fds should be created before vm is booted.

To support resizing vcpu scenario, we use max_vcpu_count for
create_vcpus and setup_interrupt_controller interfaces. The
SetVmConfiguration API will ensure max_vcpu_count >= boot_vcpu_count.

Fixes: #6010

Signed-off-by: xuejun-xj <jiyunxue@linux.alibaba.com>

src/dragonball/src/api/v1/vmm_action.rs

@@ -626,12 +626,6 @@ impl VmmService {
 
     #[cfg(feature = "hotplug")]
     fn resize_vcpu(&mut self, vmm: &mut Vmm, config: VcpuResizeInfo) -> VmmRequestResult {
-        if !cfg!(target_arch = "x86_64") {
-            // TODO: Arm need to support vcpu hotplug. issue: #6010
-            warn!("This arch do not support vm resize!");
-            return Ok(VmmData::Empty);
-        }
-
         if !cfg!(feature = "dbs-upcall") {
             warn!("We only support cpu resize through upcall server in the guest kernel now, please enable dbs-upcall feature.");
             return Ok(VmmData::Empty);

src/dragonball/src/vcpu/vcpu_manager.rs

@@ -374,11 +374,17 @@ impl VcpuManager {
         entry_addr: GuestAddress,
     ) -> Result<()> {
         info!("create boot vcpus");
-        self.create_vcpus(
+        let boot_vcpu_count = if cfg!(target_arch = "aarch64") {
-            self.vcpu_config.boot_vcpu_count,
+            // On aarch64, kvm doesn't allow to call KVM_CREATE_VCPU ioctl after vm has been booted
-            Some(request_ts),
+            // because of vgic check. To support vcpu hotplug/hotunplug feature, we should create
-            Some(entry_addr),
+            // all the vcpufd at booting procedure.
-        )?;
+            // SetVmConfiguration API will ensure max_vcpu_count >= boot_vcpu_count, so it is safe
+            // to directly use max_vcpu_count here.
+            self.vcpu_config.max_vcpu_count
+        } else {
+            self.vcpu_config.boot_vcpu_count
+        };
+        self.create_vcpus(boot_vcpu_count, Some(request_ts), Some(entry_addr))?;
 
         Ok(())
     }
@@ -1213,7 +1219,10 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         // test start boot vcpus
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
@@ -1267,8 +1276,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
+
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // invalid cpuid for pause
         let cpu_indexes = vec![2];
@@ -1304,9 +1319,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // invalid cpuid for exit
         let cpu_indexes = vec![2];
@@ -1330,9 +1350,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // exit all success
         assert!(vcpu_manager.exit_all_vcpus().is_ok());
@@ -1351,9 +1376,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // invalid cpuid for exit
         let cpu_indexes = vec![2];
@@ -1377,9 +1407,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // revalidate all success
         assert!(vcpu_manager.revalidate_all_vcpus_cache().is_ok());
@@ -1395,9 +1430,14 @@ mod tests {
         assert!(vcpu_manager
             .create_boot_vcpus(TimestampUs::default(), GuestAddress(0))
             .is_ok());
+        #[cfg(target_arch = "x86_64")]
         assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 1);
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 3);
 
         assert!(vcpu_manager.start_boot_vcpus(BpfProgram::default()).is_ok());
+        #[cfg(target_arch = "aarch64")]
+        assert_eq!(get_present_unstart_vcpus(&vcpu_manager), 2);
 
         // set vcpus in hotplug action
         let cpu_ids = vec![0];

src/dragonball/src/vm/aarch64.rs

@@ -34,7 +34,7 @@ impl Vm {
 
     /// Creates the irq chip in-kernel device model.
     pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> {
-        let vcpu_count = self.vm_config.vcpu_count;
+        let vcpu_count = self.vm_config.max_vcpu_count;
 
         self.irqchip_handle = Some(
             dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into())