dragonball: add legacy device support for aarch64

Implement RTC device for aarch64.

Fixes: #4544

Signed-off-by: xuejun-xj <jiyunxue@alibaba.linux.com>
Signed-off-by: jingshan <jingshan@linux.alibaba.com>

src/dragonball/src/device_manager/legacy.rs

@@ -13,6 +13,8 @@ use std::sync::{Arc, Mutex};
 
 use dbs_device::device_manager::Error as IoManagerError;
 use dbs_legacy_devices::SerialDevice;
+#[cfg(target_arch = "aarch64")]
+use dbs_legacy_devices::RTCDevice;
 use vmm_sys_util::eventfd::EventFd;
 
 // The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself.
@@ -42,6 +44,10 @@ pub enum Error {
 pub struct LegacyDeviceManager {
     #[cfg(target_arch = "x86_64")]
     i8042_reset_eventfd: EventFd,
+    #[cfg(target_arch = "aarch64")]
+    pub(crate) _rtc_device: Arc<Mutex<RTCDevice>>,
+    #[cfg(target_arch = "aarch64")]
+    _rtc_eventfd: EventFd,
     pub(crate) com1_device: Arc<Mutex<SerialDevice>>,
     _com1_eventfd: EventFd,
     pub(crate) com2_device: Arc<Mutex<SerialDevice>>,
@@ -140,6 +146,83 @@ pub(crate) mod x86_64 {
     }
 }
 
+#[cfg(target_arch = "aarch64")]
+pub(crate) mod aarch64 {
+    use super::*;
+    use dbs_device::device_manager::{IoManager};
+    use dbs_device::resources::DeviceResources;
+    use std::collections::HashMap;
+    use kvm_ioctls::VmFd;
+
+    type Result<T> = ::std::result::Result<T, Error>;
+
+    impl LegacyDeviceManager {
+        pub fn create_manager(
+            bus: &mut IoManager,
+            vm_fd: Option<Arc<VmFd>>,
+            resources: &HashMap<String, DeviceResources>,
+        ) -> Result<Self> {
+            let (com1_device, com1_eventfd) =
+                Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com1").unwrap())?;
+            let (com2_device, com2_eventfd) =
+                Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com2").unwrap())?;
+            let (rtc_device, rtc_eventfd) =
+                Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get("rtc").unwrap())?;
+
+            Ok(LegacyDeviceManager {
+                _rtc_device: rtc_device,
+                _rtc_eventfd: rtc_eventfd,
+                com1_device,
+                _com1_eventfd: com1_eventfd,
+                com2_device,
+                _com2_eventfd: com2_eventfd,
+            })
+        }
+
+        fn create_com_device(
+            bus: &mut IoManager,
+            vm_fd: Option<&Arc<VmFd>>,
+            resources: &DeviceResources,
+        ) -> Result<(Arc<Mutex<SerialDevice>>, EventFd)> {
+            let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?;
+            let device = Arc::new(Mutex::new(SerialDevice::new(
+                eventfd.try_clone().map_err(Error::EventFd)?
+            )));
+
+            bus.register_device_io(device.clone(), resources.get_all_resources())
+                .map_err(Error::BusError)?;
+
+            if let Some(fd) = vm_fd {
+                let irq = resources.get_legacy_irq().unwrap();
+                fd.register_irqfd(&eventfd, irq)
+                    .map_err(Error::IrqManager)?;
+            }
+
+            Ok((device, eventfd))
+        }
+
+        fn create_rtc_device(
+            bus: &mut IoManager,
+            vm_fd: Option<&Arc<VmFd>>,
+            resources: &DeviceResources,
+        ) -> Result<(Arc<Mutex<RTCDevice>>, EventFd)> {
+            let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?;
+            let device = Arc::new(Mutex::new(RTCDevice::new()));
+
+            bus.register_device_io(device.clone(), resources.get_all_resources())
+                .map_err(Error::BusError)?;
+
+            if let Some(fd) = vm_fd {
+                let irq = resources.get_legacy_irq().unwrap();
+                fd.register_irqfd(&eventfd, irq)
+                    .map_err(Error::IrqManager)?;
+            }
+
+            Ok((device, eventfd))
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     #[cfg(target_arch = "x86_64")]

src/dragonball/src/device_manager/mod.rs

@@ -13,6 +13,8 @@ use dbs_address_space::AddressSpace;
 use dbs_arch::{DeviceType, MMIODeviceInfo};
 use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext};
 use dbs_device::resources::Resource;
+#[cfg(target_arch = "aarch64")]
+use dbs_device::resources::DeviceResources;
 use dbs_device::DeviceIo;
 use dbs_interrupt::KvmIrqManager;
 use dbs_legacy_devices::ConsoleHandler;
@@ -532,11 +534,31 @@ impl DeviceManager {
         &mut self,
         ctx: &mut DeviceOpContext,
     ) -> std::result::Result<(), StartMicroVmError> {
-        #[cfg(target_arch = "x86_64")]
+        #[cfg(
+            any(target_arch = "x86_64",
+                all(target_arch = "aarch64", feature = "dbs-virtio-devices")
+            )
+        )]
         {
             let mut tx = ctx.io_context.begin_tx();
-            let legacy_manager =
+            let legacy_manager;
-                LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone()));
+
+            #[cfg(target_arch = "x86_64")]
+            {
+                let legacy_manager =
+                    LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone()));
+            }
+
+            #[cfg(target_arch = "aarch64")]
+            #[cfg(feature = "dbs-virtio-devices")]
+            {
+                let resources = self.get_legacy_resources()?;
+                legacy_manager = LegacyDeviceManager::create_manager(
+                    &mut tx.io_manager,
+                    Some(self.vm_fd.clone()),
+                    &resources,
+                );
+            }
 
             match legacy_manager {
                 Ok(v) => {
@@ -658,7 +680,7 @@ impl DeviceManager {
         {
             let dev_info = ctx
                 .generate_virtio_device_info()
-                .map_err(StartMicrovmError::DeviceManager)?;
+                .map_err(StartMicroVmError::DeviceManager)?;
             self.mmio_device_info.extend(dev_info);
         }
 
@@ -716,6 +738,78 @@ impl DeviceManager {
         Some(&self.mmio_device_info)
     }
 
+    #[cfg(feature = "dbs-virtio-devices")]
+    fn get_legacy_resources(
+        &mut self,
+    ) -> std::result::Result<HashMap<String, DeviceResources>, StartMicroVmError> {
+        let mut resources = HashMap::new();
+        let legacy_devices = vec![
+            (DeviceType::Serial, String::from("com1")),
+            (DeviceType::Serial, String::from("com2")),
+            (DeviceType::RTC, String::from("rtc")),
+        ];
+
+        for (device_type, device_id) in legacy_devices {
+            let res = self.allocate_mmio_device_resource()?;
+            self.add_mmio_device_info(&res, device_type, device_id.clone(), None);
+            resources.insert(device_id.clone(), res);
+        }
+
+        Ok(resources)
+    }
+
+    fn mmio_device_info_to_resources(
+        &self,
+        key: &(DeviceType, String),
+    ) -> std::result::Result<DeviceResources, StartMicroVmError> {
+        self.mmio_device_info
+            .get(key)
+            .map(|info| {
+                let mut resources = DeviceResources::new();
+                resources.append(Resource::LegacyIrq(info.irqs[0]));
+                resources.append(Resource::MmioAddressRange {
+                    base: info.base,
+                    size: info.size,
+                });
+                resources
+            })
+            .ok_or(StartMicroVmError::DeviceManager(
+                DeviceMgrError::GetDeviceResource,
+            ))
+    }
+
+    #[cfg(feature = "dbs-virtio-devices")]
+    fn allocate_mmio_device_resource(
+        &self,
+    ) -> std::result::Result<DeviceResources, StartMicroVmError> {
+        let mut requests = Vec::new();
+        requests.push(ResourceConstraint::MmioAddress {
+            range: None,
+            align: MMIO_DEFAULT_CFG_SIZE,
+            size: MMIO_DEFAULT_CFG_SIZE,
+        });
+        requests.push(ResourceConstraint::LegacyIrq { irq: None });
+
+        self.res_manager
+            .allocate_device_resources(&requests, false)
+            .map_err(StartMicroVmError::AllocateResource)
+    }
+
+    fn add_mmio_device_info(
+        &mut self,
+        resource: &DeviceResources,
+        device_type: DeviceType,
+        device_id: String,
+        msi_device_id: Option<u32>,
+    ) {
+        let (base, size) = resource.get_mmio_address_ranges()[0];
+        let irq = resource.get_legacy_irq().unwrap();
+        self.mmio_device_info.insert(
+            (device_type, device_id),
+            MMIODeviceInfo::new(base, size, vec![irq], msi_device_id),
+        );
+    }
+
     #[cfg(feature = "dbs-virtio-devices")]
     fn get_virtio_mmio_device_info(device: &Arc<DbsMmioV2Device>) -> Result<(u64, u64, u32)> {
         let resources = device.get_assigned_resources();

src/dragonball/src/error.rs

@@ -12,7 +12,7 @@
 #[cfg(feature = "dbs-virtio-devices")]
 use dbs_virtio_devices::Error as VirtIoError;
 
-use crate::{address_space_manager, device_manager, vcpu, vm};
+use crate::{address_space_manager, device_manager, vcpu, vm, resource_manager};
 
 /// Shorthand result type for internal VMM commands.
 pub type Result<T> = std::result::Result<T, Error>;
@@ -73,6 +73,10 @@ pub enum Error {
 /// Errors associated with starting the instance.
 #[derive(Debug, thiserror::Error)]
 pub enum StartMicroVmError {
+    /// Failed to allocate resources.
+    #[error("cannot allocate resources")]
+    AllocateResource(#[source] resource_manager::ResourceError),
+
     /// Cannot read from an Event file descriptor.
     #[error("failure while reading from EventFd file descriptor")]
     EventFd,

src/dragonball/src/vm/aarch64.rs

@@ -21,7 +21,7 @@ use vmm_sys_util::eventfd::EventFd;
 
 use super::{Vm, VmError};
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
-use crate::error::{Error, StartMicrovmError};
+use crate::error::{Error, StartMicroVmError};
 use crate::event_manager::EventManager;
 
 /// Configures the system and should be called once per vm before starting vcpu threads.
@@ -63,12 +63,12 @@ impl Vm {
     }
 
     /// Creates the irq chip in-kernel device model.
-    pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> {
+    pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> {
         let vcpu_count = self.vm_config.vcpu_count;
 
         self.irqchip_handle = Some(
             dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into())
-                .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?,
+                .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupGIC(e)))?,
         );
 
         Ok(())
@@ -88,16 +88,16 @@ impl Vm {
         epoll_mgr: EpollManager,
         vm_as: GuestAddressSpaceImpl,
         request_ts: TimestampUs,
-    ) -> Result<(), StartMicrovmError> {
+    ) -> Result<(), StartMicroVmError> {
         let reset_eventfd =
-            EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?;
+            EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicroVmError::EventFd)?;
         self.reset_eventfd = Some(
             reset_eventfd
                 .try_clone()
-                .map_err(|_| StartMicrovmError::EventFd)?,
+                .map_err(|_| StartMicroVmError::EventFd)?,
         );
         self.vcpu_manager()
-            .map_err(StartMicrovmError::Vcpu)?
+            .map_err(StartMicroVmError::Vcpu)?
             .set_reset_event_fd(reset_eventfd);
 
         // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before
@@ -106,9 +106,9 @@ impl Vm {
         // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c.
         let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?;
         self.vcpu_manager()
-            .map_err(StartMicrovmError::Vcpu)?
+            .map_err(StartMicroVmError::Vcpu)?
             .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load)
-            .map_err(StartMicrovmError::Vcpu)?;
+            .map_err(StartMicroVmError::Vcpu)?;
         self.setup_interrupt_controller()?;
         self.init_devices(epoll_mgr)?;
 
@@ -124,8 +124,8 @@ impl Vm {
         vm_memory: &GuestMemoryImpl,
         cmdline: &Cmdline,
         initrd: Option<InitrdConfig>,
-    ) -> std::result::Result<(), StartMicrovmError> {
+    ) -> std::result::Result<(), StartMicroVmError> {
-        let vcpu_manager = self.vcpu_manager().map_err(StartMicrovmError::Vcpu)?;
+        let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?;
         let vcpu_mpidr = vcpu_manager
             .vcpus()
             .into_iter()
@@ -141,17 +141,17 @@ impl Vm {
             self.get_irqchip(),
             &initrd,
         )
-        .map_err(StartMicrovmError::ConfigureSystem)
+        .map_err(StartMicroVmError::ConfigureSystem)
     }
 
     pub(crate) fn register_events(
         &mut self,
         event_mgr: &mut EventManager,
-    ) -> std::result::Result<(), StartMicrovmError> {
+    ) -> std::result::Result<(), StartMicroVmError> {
-        let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?;
+        let reset_evt = self.get_reset_eventfd().ok_or(StartMicroVmError::EventFd)?;
         event_mgr
             .register_exit_eventfd(reset_evt)
-            .map_err(|_| StartMicrovmError::RegisterEvent)?;
+            .map_err(|_| StartMicroVmError::RegisterEvent)?;
 
         Ok(())
     }