diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 35f092a501..66a24f5629 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -9,6 +9,8 @@ //! Error codes for the virtual machine monitor subsystem. +#[cfg(target_arch = "aarch64")] +use dbs_arch::pmu::PmuError; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; @@ -61,6 +63,11 @@ pub enum Error { #[error("failed to write MP table to guest memory: {0}")] MpTableSetup(#[source] dbs_boot::mptable::Error), + /// Create pmu device error + #[cfg(target_arch = "aarch64")] + #[error("Create pmu device error: {0}")] + PmuDeviceError(#[source] PmuError), + /// Fail to boot system #[error("failed to boot system: {0}")] BootSystem(#[source] dbs_boot::Error), diff --git a/src/dragonball/src/vcpu/aarch64.rs b/src/dragonball/src/vcpu/aarch64.rs index dc4b9c61a6..ae45cd99d9 100644 --- a/src/dragonball/src/vcpu/aarch64.rs +++ b/src/dragonball/src/vcpu/aarch64.rs @@ -11,7 +11,7 @@ use std::sync::mpsc::{channel, Sender}; use std::sync::Arc; use crate::IoManagerCached; -use dbs_arch::regs; +use dbs_arch::{regs, VpmuFeatureLevel}; use dbs_boot::get_fdt_addr; use dbs_utils::time::TimestampUs; use kvm_ioctls::{VcpuFd, VmFd}; @@ -81,7 +81,7 @@ impl Vcpu { /// * `_pgtable_addr` - pgtable address for ap vcpu (not used in aarch64) pub fn configure( &mut self, - _vcpu_config: &VcpuConfig, + vcpu_config: &VcpuConfig, vm_fd: &VmFd, vm_as: &GuestAddressSpaceImpl, kernel_load_addr: Option, @@ -99,6 +99,9 @@ impl Vcpu { if self.id > 0 { kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF; } + if vcpu_config.vpmu_feature == VpmuFeatureLevel::FullyEnabled { + kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3; + } self.fd.vcpu_init(&kvi).map_err(VcpuError::VcpuArmInit)?; diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs index b04baf29f4..0011d81bde 100644 --- a/src/dragonball/src/vcpu/mod.rs +++ b/src/dragonball/src/vcpu/mod.rs @@ -7,9 +7,7 @@ mod sm; mod vcpu_impl; mod vcpu_manager; -#[cfg(target_arch = "x86_64")] -use dbs_arch::cpuid::VpmuFeatureLevel; - +use dbs_arch::VpmuFeatureLevel; pub use vcpu_manager::{VcpuManager, VcpuManagerError, VcpuResizeInfo}; #[cfg(feature = "hotplug")] @@ -32,6 +30,6 @@ pub struct VcpuConfig { /// if vpmu feature is Disabled, it means vpmu feature is off (by default) /// if vpmu feature is LimitedlyEnabled, it means minimal vpmu counters are supported (cycles and instructions) /// if vpmu feature is FullyEnabled, it means all vpmu counters are supported - #[cfg(target_arch = "x86_64")] + /// For aarch64, VpmuFeatureLevel only supports Disabled and FullyEnabled. pub vpmu_feature: VpmuFeatureLevel, } diff --git a/src/dragonball/src/vcpu/vcpu_impl.rs b/src/dragonball/src/vcpu/vcpu_impl.rs index 26f48c6d1d..3dffd579f1 100644 --- a/src/dragonball/src/vcpu/vcpu_impl.rs +++ b/src/dragonball/src/vcpu/vcpu_impl.rs @@ -760,6 +760,11 @@ impl Vcpu { // State machine reached its end. StateMachine::finish(Self::exited) } + + /// Get vcpu file descriptor. + pub fn vcpu_fd(&self) -> &VcpuFd { + self.fd.as_ref() + } } impl Drop for Vcpu { diff --git a/src/dragonball/src/vcpu/vcpu_manager.rs b/src/dragonball/src/vcpu/vcpu_manager.rs index 7b93c1a825..8f39af5194 100644 --- a/src/dragonball/src/vcpu/vcpu_manager.rs +++ b/src/dragonball/src/vcpu/vcpu_manager.rs @@ -15,6 +15,7 @@ use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender}; use std::sync::{Arc, Barrier, Mutex, RwLock}; use std::time::Duration; +use dbs_arch::VpmuFeatureLevel; #[cfg(all(feature = "hotplug", feature = "dbs-upcall"))] use dbs_upcall::{DevMgrService, UpcallClient}; use dbs_utils::epoll_manager::{EpollManager, EventOps, EventSet, Events, MutEventSubscriber}; @@ -281,11 +282,20 @@ impl VcpuManager { let supported_cpuid = kvm_context .supported_cpuid(kvm_bindings::KVM_MAX_CPUID_ENTRIES) .map_err(VcpuManagerError::Kvm)?; - #[cfg(target_arch = "x86_64")] + #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] let vpmu_feature_level = match vm_config_info.vpmu_feature { - 1 => dbs_arch::cpuid::VpmuFeatureLevel::LimitedlyEnabled, - 2 => dbs_arch::cpuid::VpmuFeatureLevel::FullyEnabled, - _ => dbs_arch::cpuid::VpmuFeatureLevel::Disabled, + #[cfg(target_arch = "x86_64")] + 1 => VpmuFeatureLevel::LimitedlyEnabled, + #[cfg(target_arch = "aarch64")] + 1 => { + log::warn!( + "Limitedly enabled vpmu feature isn't supported on aarch64 for now.\ + This will be supported in the future. The vpmu_feature will be set disabled!" + ); + VpmuFeatureLevel::Disabled + } + 2 => VpmuFeatureLevel::FullyEnabled, + _ => VpmuFeatureLevel::Disabled, }; let vcpu_manager = Arc::new(Mutex::new(VcpuManager { @@ -297,7 +307,6 @@ impl VcpuManager { cores_per_die: vm_config_info.cpu_topology.cores_per_die, dies_per_socket: vm_config_info.cpu_topology.dies_per_socket, sockets: vm_config_info.cpu_topology.sockets, - #[cfg(target_arch = "x86_64")] vpmu_feature: vpmu_feature_level, }, vcpu_seccomp_filter, @@ -799,6 +808,11 @@ impl VcpuManager { ) .map_err(VcpuManagerError::Vcpu) } + + /// get vpmu_feature config + pub fn vpmu_feature(&self) -> VpmuFeatureLevel { + self.vcpu_config.vpmu_feature + } } #[cfg(feature = "hotplug")] diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 0ec8251f45..edc7532457 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -11,6 +11,7 @@ use std::fmt::Debug; use std::ops::Deref; use dbs_arch::gic::GICDevice; +use dbs_arch::pmu::initialize_pmu; use dbs_arch::{DeviceInfoForFDT, DeviceType, VpmuFeatureLevel}; use dbs_boot::InitrdConfig; use dbs_utils::epoll_manager::EpollManager; @@ -43,6 +44,7 @@ fn configure_system( device_info: Option<&HashMap<(DeviceType, String), T>>, gic_device: &Box, initrd: &Option, + vpmu_feature: &VpmuFeatureLevel, ) -> super::Result<()> { dbs_boot::fdt::create_fdt( guest_mem, @@ -51,8 +53,7 @@ fn configure_system( device_info, gic_device, initrd, - // We will add vpmu feature support in the future PRs. issue: #6168 - &VpmuFeatureLevel::Disabled, + vpmu_feature, ) .map_err(Error::BootSystem)?; Ok(()) @@ -78,6 +79,23 @@ impl Vm { Ok(()) } + /// Setup pmu devices for guest vm. + pub fn setup_pmu_devices(&mut self) -> std::result::Result<(), StartMicroVmError> { + let vm = self.vm_fd(); + let mut vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; + let vpmu_feature = vcpu_manager.vpmu_feature(); + if vpmu_feature == VpmuFeatureLevel::Disabled { + return Ok(()); + } + + for vcpu in vcpu_manager.vcpus_mut() { + initialize_pmu(vm, vcpu.vcpu_fd()) + .map_err(|e| StartMicroVmError::ConfigureVm(VmError::SetupPmu(e)))?; + } + + Ok(()) + } + /// Initialize the virtual machine instance. /// /// It initialize the virtual machine instance by: @@ -115,6 +133,7 @@ impl Vm { .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) .map_err(StartMicroVmError::Vcpu)?; self.setup_interrupt_controller()?; + self.setup_pmu_devices()?; self.init_devices(epoll_mgr)?; Ok(()) @@ -131,6 +150,7 @@ impl Vm { initrd: Option, ) -> std::result::Result<(), StartMicroVmError> { let vcpu_manager = self.vcpu_manager().map_err(StartMicroVmError::Vcpu)?; + let vpmu_feature = vcpu_manager.vpmu_feature(); let vcpu_mpidr = vcpu_manager .vcpus() .into_iter() @@ -145,6 +165,7 @@ impl Vm { self.device_manager.get_mmio_device_info(), self.get_irqchip(), &initrd, + &vpmu_feature, ) .map_err(StartMicroVmError::ConfigureSystem) } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 13e6584b8d..a9ccb02ea3 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -10,6 +10,8 @@ use std::sync::{Arc, Mutex, RwLock}; use dbs_address_space::AddressSpace; #[cfg(target_arch = "aarch64")] use dbs_arch::gic::GICDevice; +#[cfg(target_arch = "aarch64")] +use dbs_arch::pmu::PmuError; use dbs_boot::InitrdConfig; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; @@ -69,6 +71,11 @@ pub enum VmError { #[cfg(target_arch = "aarch64")] #[error("failed to configure GIC")] SetupGIC(GICError), + + /// Cannot setup pmu device + #[cfg(target_arch = "aarch64")] + #[error("failed to setup pmu device")] + SetupPmu(#[source] PmuError), } /// Configuration information for user defined NUMA nodes.