From f6f96b8fee9c4bbfd0a28ae826b34f8510b5e8ed Mon Sep 17 00:00:00 2001 From: xuejun-xj Date: Sun, 10 Jul 2022 17:35:30 +0800 Subject: [PATCH] dragonball: add legacy device support for aarch64 Implement RTC device for aarch64. Fixes: #4544 Signed-off-by: xuejun-xj Signed-off-by: jingshan --- src/dragonball/src/device_manager/legacy.rs | 83 ++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 102 +++++++++++++++++++- src/dragonball/src/error.rs | 6 +- src/dragonball/src/vm/aarch64.rs | 30 +++--- 4 files changed, 201 insertions(+), 20 deletions(-) diff --git a/src/dragonball/src/device_manager/legacy.rs b/src/dragonball/src/device_manager/legacy.rs index 9dbb300d4b..b3e33a7f22 100644 --- a/src/dragonball/src/device_manager/legacy.rs +++ b/src/dragonball/src/device_manager/legacy.rs @@ -13,6 +13,8 @@ use std::sync::{Arc, Mutex}; use dbs_device::device_manager::Error as IoManagerError; use dbs_legacy_devices::SerialDevice; +#[cfg(target_arch = "aarch64")] +use dbs_legacy_devices::RTCDevice; use vmm_sys_util::eventfd::EventFd; // The I8042 Data Port (IO Port 0x60) is used for reading data that was received from a I8042 device or from the I8042 controller itself and writing data to a I8042 device or to the I8042 controller itself. @@ -42,6 +44,10 @@ pub enum Error { pub struct LegacyDeviceManager { #[cfg(target_arch = "x86_64")] i8042_reset_eventfd: EventFd, + #[cfg(target_arch = "aarch64")] + pub(crate) _rtc_device: Arc>, + #[cfg(target_arch = "aarch64")] + _rtc_eventfd: EventFd, pub(crate) com1_device: Arc>, _com1_eventfd: EventFd, pub(crate) com2_device: Arc>, @@ -140,6 +146,83 @@ pub(crate) mod x86_64 { } } +#[cfg(target_arch = "aarch64")] +pub(crate) mod aarch64 { + use super::*; + use dbs_device::device_manager::{IoManager}; + use dbs_device::resources::DeviceResources; + use std::collections::HashMap; + use kvm_ioctls::VmFd; + + type Result = ::std::result::Result; + + impl LegacyDeviceManager { + pub fn create_manager( + bus: &mut IoManager, + vm_fd: Option>, + resources: &HashMap, + ) -> Result { + let (com1_device, com1_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com1").unwrap())?; + let (com2_device, com2_eventfd) = + Self::create_com_device(bus, vm_fd.as_ref(), resources.get("com2").unwrap())?; + let (rtc_device, rtc_eventfd) = + Self::create_rtc_device(bus, vm_fd.as_ref(), resources.get("rtc").unwrap())?; + + Ok(LegacyDeviceManager { + _rtc_device: rtc_device, + _rtc_eventfd: rtc_eventfd, + com1_device, + _com1_eventfd: com1_eventfd, + com2_device, + _com2_eventfd: com2_eventfd, + }) + } + + fn create_com_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + resources: &DeviceResources, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(SerialDevice::new( + eventfd.try_clone().map_err(Error::EventFd)? + ))); + + bus.register_device_io(device.clone(), resources.get_all_resources()) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + let irq = resources.get_legacy_irq().unwrap(); + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + + fn create_rtc_device( + bus: &mut IoManager, + vm_fd: Option<&Arc>, + resources: &DeviceResources, + ) -> Result<(Arc>, EventFd)> { + let eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(Error::EventFd)?; + let device = Arc::new(Mutex::new(RTCDevice::new())); + + bus.register_device_io(device.clone(), resources.get_all_resources()) + .map_err(Error::BusError)?; + + if let Some(fd) = vm_fd { + let irq = resources.get_legacy_irq().unwrap(); + fd.register_irqfd(&eventfd, irq) + .map_err(Error::IrqManager)?; + } + + Ok((device, eventfd)) + } + } +} + #[cfg(test)] mod tests { #[cfg(target_arch = "x86_64")] diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 0ce6af6062..1e52600e5d 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -13,6 +13,8 @@ use dbs_address_space::AddressSpace; use dbs_arch::{DeviceType, MMIODeviceInfo}; use dbs_device::device_manager::{Error as IoManagerError, IoManager, IoManagerContext}; use dbs_device::resources::Resource; +#[cfg(target_arch = "aarch64")] +use dbs_device::resources::DeviceResources; use dbs_device::DeviceIo; use dbs_interrupt::KvmIrqManager; use dbs_legacy_devices::ConsoleHandler; @@ -532,11 +534,31 @@ impl DeviceManager { &mut self, ctx: &mut DeviceOpContext, ) -> std::result::Result<(), StartMicroVmError> { - #[cfg(target_arch = "x86_64")] + #[cfg( + any(target_arch = "x86_64", + all(target_arch = "aarch64", feature = "dbs-virtio-devices") + ) + )] { let mut tx = ctx.io_context.begin_tx(); - let legacy_manager = - LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + let legacy_manager; + + #[cfg(target_arch = "x86_64")] + { + let legacy_manager = + LegacyDeviceManager::create_manager(&mut tx.io_manager, Some(self.vm_fd.clone())); + } + + #[cfg(target_arch = "aarch64")] + #[cfg(feature = "dbs-virtio-devices")] + { + let resources = self.get_legacy_resources()?; + legacy_manager = LegacyDeviceManager::create_manager( + &mut tx.io_manager, + Some(self.vm_fd.clone()), + &resources, + ); + } match legacy_manager { Ok(v) => { @@ -658,7 +680,7 @@ impl DeviceManager { { let dev_info = ctx .generate_virtio_device_info() - .map_err(StartMicrovmError::DeviceManager)?; + .map_err(StartMicroVmError::DeviceManager)?; self.mmio_device_info.extend(dev_info); } @@ -716,6 +738,78 @@ impl DeviceManager { Some(&self.mmio_device_info) } + #[cfg(feature = "dbs-virtio-devices")] + fn get_legacy_resources( + &mut self, + ) -> std::result::Result, StartMicroVmError> { + let mut resources = HashMap::new(); + let legacy_devices = vec![ + (DeviceType::Serial, String::from("com1")), + (DeviceType::Serial, String::from("com2")), + (DeviceType::RTC, String::from("rtc")), + ]; + + for (device_type, device_id) in legacy_devices { + let res = self.allocate_mmio_device_resource()?; + self.add_mmio_device_info(&res, device_type, device_id.clone(), None); + resources.insert(device_id.clone(), res); + } + + Ok(resources) + } + + fn mmio_device_info_to_resources( + &self, + key: &(DeviceType, String), + ) -> std::result::Result { + self.mmio_device_info + .get(key) + .map(|info| { + let mut resources = DeviceResources::new(); + resources.append(Resource::LegacyIrq(info.irqs[0])); + resources.append(Resource::MmioAddressRange { + base: info.base, + size: info.size, + }); + resources + }) + .ok_or(StartMicroVmError::DeviceManager( + DeviceMgrError::GetDeviceResource, + )) + } + + #[cfg(feature = "dbs-virtio-devices")] + fn allocate_mmio_device_resource( + &self, + ) -> std::result::Result { + let mut requests = Vec::new(); + requests.push(ResourceConstraint::MmioAddress { + range: None, + align: MMIO_DEFAULT_CFG_SIZE, + size: MMIO_DEFAULT_CFG_SIZE, + }); + requests.push(ResourceConstraint::LegacyIrq { irq: None }); + + self.res_manager + .allocate_device_resources(&requests, false) + .map_err(StartMicroVmError::AllocateResource) + } + + fn add_mmio_device_info( + &mut self, + resource: &DeviceResources, + device_type: DeviceType, + device_id: String, + msi_device_id: Option, + ) { + let (base, size) = resource.get_mmio_address_ranges()[0]; + let irq = resource.get_legacy_irq().unwrap(); + self.mmio_device_info.insert( + (device_type, device_id), + MMIODeviceInfo::new(base, size, vec![irq], msi_device_id), + ); + } + #[cfg(feature = "dbs-virtio-devices")] fn get_virtio_mmio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { let resources = device.get_assigned_resources(); diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index b4ee401194..454ad45b59 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,7 +12,7 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; -use crate::{address_space_manager, device_manager, vcpu, vm}; +use crate::{address_space_manager, device_manager, vcpu, vm, resource_manager}; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; @@ -73,6 +73,10 @@ pub enum Error { /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] pub enum StartMicroVmError { + /// Failed to allocate resources. + #[error("cannot allocate resources")] + AllocateResource(#[source] resource_manager::ResourceError), + /// Cannot read from an Event file descriptor. #[error("failure while reading from EventFd file descriptor")] EventFd, diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 25450f380d..ffecaf356f 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -21,7 +21,7 @@ use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; -use crate::error::{Error, StartMicrovmError}; +use crate::error::{Error, StartMicroVmError}; use crate::event_manager::EventManager; /// Configures the system and should be called once per vm before starting vcpu threads. @@ -63,12 +63,12 @@ impl Vm { } /// Creates the irq chip in-kernel device model. - pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicroVmError> { let vcpu_count = self.vm_config.vcpu_count; self.irqchip_handle = Some( dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupGIC(e)))?, ); Ok(()) @@ -88,16 +88,16 @@ impl Vm { epoll_mgr: EpollManager, vm_as: GuestAddressSpaceImpl, request_ts: TimestampUs, - ) -> Result<(), StartMicrovmError> { + ) -> Result<(), StartMicroVmError> { let reset_eventfd = - EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?; + EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicroVmError::EventFd)?; self.reset_eventfd = Some( reset_eventfd .try_clone() - .map_err(|_| StartMicrovmError::EventFd)?, + .map_err(|_| StartMicroVmError::EventFd)?, ); self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .set_reset_event_fd(reset_eventfd); // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before @@ -106,9 +106,9 @@ impl Vm { // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; self.vcpu_manager() - .map_err(StartMicrovmError::Vcpu)? + .map_err(StartMicroVmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) - .map_err(StartMicrovmError::Vcpu)?; + .map_err(StartMicroVmError::Vcpu)?; self.setup_interrupt_controller()?; self.init_devices(epoll_mgr)?; @@ -124,8 +124,8 @@ impl Vm { vm_memory: &GuestMemoryImpl, cmdline: &Cmdline, initrd: Option, - ) -> std::result::Result<(), StartMicrovmError> { - let vcpu_manager = self.vcpu_manager().map_err(StartMicrovmError::Vcpu)?; + ) -> std::result::Result<(), StartMicroVmError> { + let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; let vcpu_mpidr = vcpu_manager .vcpus() .into_iter() @@ -141,17 +141,17 @@ impl Vm { self.get_irqchip(), &initrd, ) - .map_err(StartMicrovmError::ConfigureSystem) + .map_err(StartMicroVmError::ConfigureSystem) } pub(crate) fn register_events( &mut self, event_mgr: &mut EventManager, - ) -> std::result::Result<(), StartMicrovmError> { - let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?; + ) -> std::result::Result<(), StartMicroVmError> { + let reset_evt = self.get_reset_eventfd().ok_or(StartMicroVmError::EventFd)?; event_mgr .register_exit_eventfd(reset_evt) - .map_err(|_| StartMicrovmError::RegisterEvent)?; + .map_err(|_| StartMicroVmError::RegisterEvent)?; Ok(()) }