From 4d234f5742e437cfa9bf4e1a0aaa6250df9f9904 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 22 May 2022 14:42:32 +0800 Subject: [PATCH] dragonball: refactor code layout Refactored some code layout. Signed-off-by: Jiang Liu --- .../src/device_manager/console_manager.rs | 2 +- src/dragonball/src/device_manager/mod.rs | 43 +- src/dragonball/src/error.rs | 5 +- src/dragonball/src/vcpu/mod.rs | 7 +- src/dragonball/src/vm/aarch64.rs | 51 +-- src/dragonball/src/vm/kernel_config.rs | 7 +- src/dragonball/src/vm/mod.rs | 417 +++++++++--------- src/dragonball/src/vm/x86_64.rs | 22 +- 8 files changed, 281 insertions(+), 273 deletions(-) diff --git a/src/dragonball/src/device_manager/console_manager.rs b/src/dragonball/src/device_manager/console_manager.rs index 388eea78b8..4ebad58167 100644 --- a/src/dragonball/src/device_manager/console_manager.rs +++ b/src/dragonball/src/device_manager/console_manager.rs @@ -353,7 +353,7 @@ pub struct DmesgWriter { impl DmesgWriter { /// Creates a new instance. - pub fn new(logger: slog::Logger) -> Self { + pub fn new(logger: &slog::Logger) -> Self { Self { buf: BytesMut::with_capacity(1024), logger: logger.new(slog::o!("subsystem" => "dmesg")), diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 011a1607dc..07698d4b88 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -22,8 +22,6 @@ use kvm_ioctls::VmFd; use dbs_device::resources::ResourceConstraint; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices as virtio; -#[cfg(feature = "virtio-vsock")] -use dbs_virtio_devices::vsock::backend::VsockInnerConnector; #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::{ mmio::{ @@ -38,6 +36,8 @@ use dbs_upcall::{ DevMgrRequest, DevMgrService, MmioDevRequest, UpcallClient, UpcallClientError, UpcallClientRequest, UpcallClientResponse, }; +#[cfg(feature = "hotplug")] +use dbs_virtio_devices::vsock::backend::VsockInnerConnector; use crate::address_space_manager::GuestAddressSpaceImpl; use crate::error::StartMicrovmError; @@ -99,6 +99,10 @@ pub enum DeviceMgrError { /// Failed to hotplug the device. #[error("failed to hotplug virtual device")] HotplugDevice(#[source] UpcallClientError), + + /// Failed to free device resource. + #[error("failed to free device resources: {0}")] + ResourceError(#[source] crate::resource_manager::ResourceError), } /// Specialized version of `std::result::Result` for device manager operations. @@ -315,8 +319,6 @@ impl DeviceOpContext { pub(crate) fn create_hotplug_ctx(vm: &Vm, epoll_mgr: Option) -> Self { let vm_as = vm.vm_as().expect("VM should have memory ready").clone(); - let vm_config = vm.vm_config().clone(); - let mut ctx = Self::new( epoll_mgr, vm.device_manager(), @@ -325,7 +327,6 @@ impl DeviceOpContext { true, ); ctx.upcall_client = vm.upcall_client().clone(); - ctx } @@ -404,10 +405,10 @@ pub struct DeviceManager { pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, + #[cfg(target_arch = "aarch64")] + pub(crate) mmio_device_info: HashMap<(DeviceType, String), MMIODeviceInfo>, #[cfg(feature = "virtio-vsock")] pub(crate) vsock_manager: VsockDeviceMgr, - #[cfg(target_arch = "aarch64")] - mmio_device_info: HashMap<(DeviceType, String), MMIODeviceInfo>, } impl DeviceManager { @@ -425,12 +426,13 @@ impl DeviceManager { res_manager, vm_fd, logger: logger.new(slog::o!()), + con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, - #[cfg(feature = "virtio-vsock")] - vsock_manager: VsockDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), + #[cfg(feature = "virtio-vsock")] + vsock_manager: VsockDeviceMgr::default(), } } @@ -452,6 +454,7 @@ impl DeviceManager { } /// Create legacy devices associted virtual machine + #[allow(unused_variables)] pub fn create_legacy_devices( &mut self, ctx: &mut DeviceOpContext, @@ -529,12 +532,6 @@ impl DeviceManager { self.con_manager.reset_console() } - #[cfg(target_arch = "aarch64")] - /// Return mmio device info for FDT build. - pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { - Some(&self.mmio_device_info) - } - /// Create all registered devices when booting the associated virtual machine. pub fn create_devices( &mut self, @@ -579,8 +576,10 @@ impl DeviceManager { ) -> Result<()> { Ok(()) } +} - #[cfg(target_arch = "x86_64")] +#[cfg(target_arch = "x86_64")] +impl DeviceManager { /// Get the underlying eventfd for vm exit notification. pub fn get_reset_eventfd(&self) -> Result { if let Some(legacy) = self.legacy_manager.as_ref() { @@ -595,6 +594,14 @@ impl DeviceManager { } } +#[cfg(target_arch = "aarch64")] +impl DeviceManager { + /// Return mmio device info for FDT build. + pub fn get_mmio_device_info(&self) -> Option<&HashMap<(DeviceType, String), MMIODeviceInfo>> { + Some(&self.mmio_device_info) + } +} + #[cfg(feature = "dbs-virtio-devices")] impl DeviceManager { fn get_virtio_device_info(device: &Arc) -> Result<(u64, u64, u32)> { @@ -694,7 +701,9 @@ impl DeviceManager { // unregister Resource manager let resources = device.get_assigned_resources(); - ctx.res_manager.free_device_resources(&resources); + ctx.res_manager + .free_device_resources(&resources) + .map_err(DeviceMgrError::ResourceError)?; Ok(()) } diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index b201c1d779..c50e50b256 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -12,10 +12,7 @@ #[cfg(feature = "dbs-virtio-devices")] use dbs_virtio_devices::Error as VirtIoError; -use crate::address_space_manager; -use crate::device_manager; -use crate::vcpu; -use crate::vm; +use crate::{address_space_manager, device_manager, vcpu, vm}; /// Shorthand result type for internal VMM commands. pub type Result = std::result::Result; diff --git a/src/dragonball/src/vcpu/mod.rs b/src/dragonball/src/vcpu/mod.rs index fe66883245..5b8ed397ff 100644 --- a/src/dragonball/src/vcpu/mod.rs +++ b/src/dragonball/src/vcpu/mod.rs @@ -4,13 +4,14 @@ // SPDX-License-Identifier: Apache-2.0 mod sm; -pub mod vcpu_impl; -pub mod vcpu_manager; -pub use vcpu_manager::{VcpuManager, VcpuManagerError}; +mod vcpu_impl; +mod vcpu_manager; #[cfg(target_arch = "x86_64")] use dbs_arch::cpuid::VpmuFeatureLevel; +pub use vcpu_manager::{VcpuManager, VcpuManagerError}; + /// vcpu config collection pub struct VcpuConfig { /// initial vcpu count diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 02af4a6917..739b1515c6 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -7,6 +7,7 @@ // found in the THIRD-PARTY file. use std::collections::HashMap; +use std::fmt::Debug; use std::ops::Deref; use dbs_arch::gic::GICDevice; @@ -14,15 +15,13 @@ use dbs_arch::{DeviceInfoForFDT, DeviceType}; use dbs_boot::InitrdConfig; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; -use std::fmt::Debug; +use linux_loader::loader::Cmdline; use vm_memory::{GuestAddressSpace, GuestMemory}; use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; -use crate::error::Error; -use crate::StartMicrovmError; -use linux_loader::loader::Cmdline; +use crate::error::{Error, StartMicrovmError}; /// Configures the system and should be called once per vm before starting vcpu threads. /// For aarch64, we only setup the FDT. @@ -35,7 +34,7 @@ use linux_loader::loader::Cmdline; /// * `device_info` - A hashmap containing the attached devices for building FDT device nodes. /// * `gic_device` - The GIC device. /// * `initrd` - Information about an optional initrd. -pub fn configure_system( +fn configure_system( guest_mem: &M, cmdline: &str, vcpu_mpidr: Vec, @@ -62,6 +61,18 @@ impl Vm { &self.irqchip_handle.as_ref().unwrap() } + /// Creates the irq chip in-kernel device model. + pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { + let vcpu_count = self.vm_config.vcpu_count; + + self.irqchip_handle = Some( + dbs_arch::gic::create_gic(&self.vm_fd, vcpu_count.into()) + .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, + ); + + Ok(()) + } + /// Initialize the virtual machine instance. /// /// It initialize the virtual machine instance by: @@ -76,13 +87,7 @@ impl Vm { epoll_mgr: EpollManager, vm_as: GuestAddressSpaceImpl, request_ts: TimestampUs, - ) -> std::result::Result<(), StartMicrovmError> { - let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; - // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before - // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP - // was already initialized. - // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. - + ) -> Result<(), StartMicrovmError> { let reset_eventfd = EventFd::new(libc::EFD_NONBLOCK).map_err(|_| StartMicrovmError::EventFd)?; self.reset_eventfd = Some( @@ -90,33 +95,25 @@ impl Vm { .try_clone() .map_err(|_| StartMicrovmError::EventFd)?, ); - self.vcpu_manager() .map_err(StartMicrovmError::Vcpu)? .set_reset_event_fd(reset_eventfd); + + // On aarch64, the vCPUs need to be created (i.e call KVM_CREATE_VCPU) and configured before + // setting up the IRQ chip because the `KVM_CREATE_VCPU` ioctl will return error if the IRQCHIP + // was already initialized. + // Search for `kvm_arch_vcpu_create` in arch/arm/kvm/arm.c. + let kernel_loader_result = self.load_kernel(vm_as.memory().deref())?; self.vcpu_manager() .map_err(StartMicrovmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) .map_err(StartMicrovmError::Vcpu)?; - self.setup_interrupt_controller()?; self.init_devices(epoll_mgr)?; Ok(()) } - /// Creates the irq chip in-kernel device model. - pub fn setup_interrupt_controller(&mut self) -> std::result::Result<(), StartMicrovmError> { - let vcpu_count = self.vm_config.vcpu_count; - - self.irqchip_handle = Some( - dbs_arch::gic::create_gic(&self.fd, vcpu_count.into()) - .map_err(|e| StartMicrovmError::ConfigureVm(VmError::SetupGIC(e)))?, - ); - - Ok(()) - } - /// Execute system architecture specific configurations. /// /// 1) set guest kernel boot parameters @@ -133,8 +130,8 @@ impl Vm { .into_iter() .map(|cpu| cpu.get_mpidr()) .collect(); - let guest_memory = vm_memory.memory(); + configure_system( guest_memory, cmdline.as_str(), diff --git a/src/dragonball/src/vm/kernel_config.rs b/src/dragonball/src/vm/kernel_config.rs index 466fd8849c..4798d8da3d 100644 --- a/src/dragonball/src/vm/kernel_config.rs +++ b/src/dragonball/src/vm/kernel_config.rs @@ -8,7 +8,7 @@ pub struct KernelConfigInfo { /// The descriptor to the kernel file. kernel_file: File, /// The descriptor to the initrd file, if there is one - pub initrd_file: Option, + initrd_file: Option, /// The commandline for guest kernel. cmdline: linux_loader::cmdline::Cmdline, } @@ -32,6 +32,11 @@ impl KernelConfigInfo { &mut self.kernel_file } + /// Get an immutable reference to the initrd file. + pub fn initrd_file(&self) -> Option<&File> { + self.initrd_file.as_ref() + } + /// Get a mutable reference to the initrd file. pub fn initrd_file_mut(&mut self) -> Option<&mut File> { self.initrd_file.as_mut() diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index db10e593e6..fff044e3aa 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -10,8 +10,6 @@ use dbs_address_space::AddressSpace; #[cfg(target_arch = "aarch64")] use dbs_arch::gic::GICDevice; use dbs_boot::InitrdConfig; -#[cfg(feature = "hotplug")] -use dbs_upcall::{DevMgrService, UpcallClient}; use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; use kvm_ioctls::VmFd; @@ -22,6 +20,9 @@ use slog::{error, info}; use vm_memory::{Bytes, GuestAddress, GuestAddressSpace}; use vmm_sys_util::eventfd::EventFd; +#[cfg(feature = "hotplug")] +use dbs_upcall::{DevMgrService, UpcallClient}; + use crate::address_space_manager::{ AddressManagerError, AddressSpaceMgr, AddressSpaceMgrBuilder, GuestAddressSpaceImpl, GuestMemoryImpl, @@ -61,7 +62,7 @@ pub enum VmError { /// Cannot setup GIC #[cfg(target_arch = "aarch64")] #[error("failed to configure GIC")] - SetupGIC(GICError), + SetupGIC(dbs_arch::gic::Error), } /// Configuration information for user defined NUMA nodes. @@ -169,21 +170,21 @@ impl Default for VmConfigInfo { /// | ^---1:N-> Vcpu /// |---<-1:N-> Event Manager pub struct Vm { - fd: Arc, + epoll_manager: EpollManager, kvm: KvmContext, + shared_info: Arc>, address_space: AddressSpaceMgr, device_manager: DeviceManager, - epoll_manager: EpollManager, + dmesg_fifo: Option>, + kernel_config: Option, + logger: slog::Logger, + reset_eventfd: Option, resource_manager: Arc, vcpu_manager: Option>>, - logger: slog::Logger, - /// Config of virtual machine vm_config: VmConfigInfo, - kernel_config: Option, - shared_info: Arc>, - reset_eventfd: Option, - dmesg_fifo: Option>, + vm_fd: Arc, + start_instance_request_ts: u64, start_instance_request_cpu_ts: u64, start_instance_downtime: u64, @@ -191,7 +192,7 @@ pub struct Vm { // Arm specific fields. // On aarch64 we need to keep around the fd obtained by creating the VGIC device. #[cfg(target_arch = "aarch64")] - irqchip_handle: Option>, + irqchip_handle: Option>, #[cfg(feature = "hotplug")] upcall_client: Option>>, @@ -206,36 +207,36 @@ impl Vm { ) -> Result { let id = api_shared_info.read().unwrap().id.clone(); let logger = slog_scope::logger().new(slog::o!("id" => id)); - let kvm = KvmContext::new(kvm_fd)?; - let fd = Arc::new(kvm.create_vm()?); - + let vm_fd = Arc::new(kvm.create_vm()?); let resource_manager = Arc::new(ResourceManager::new(Some(kvm.max_memslots()))); - let device_manager = DeviceManager::new( - fd.clone(), + vm_fd.clone(), resource_manager.clone(), epoll_manager.clone(), &logger, ); Ok(Vm { - fd, + epoll_manager, kvm, + shared_info: api_shared_info, + address_space: AddressSpaceMgr::default(), device_manager, - epoll_manager, + dmesg_fifo: None, + kernel_config: None, + logger, + reset_eventfd: None, resource_manager, vcpu_manager: None, - logger, vm_config: Default::default(), - kernel_config: None, - shared_info: api_shared_info, - reset_eventfd: None, - dmesg_fifo: None, + vm_fd, + start_instance_request_ts: 0, start_instance_request_cpu_ts: 0, start_instance_downtime: 0, + #[cfg(target_arch = "aarch64")] irqchip_handle: None, #[cfg(feature = "hotplug")] @@ -243,9 +244,29 @@ impl Vm { }) } - /// Gets a reference to the kvm file descriptor owned by this VM. - pub fn vm_fd(&self) -> &VmFd { - &self.fd + /// Gets a reference to the device manager by this VM. + pub fn device_manager(&self) -> &DeviceManager { + &self.device_manager + } + + /// Get a reference to EpollManager. + pub fn epoll_manager(&self) -> &EpollManager { + &self.epoll_manager + } + + /// Get eventfd for exit notification. + pub fn get_reset_eventfd(&self) -> Option<&EventFd> { + self.reset_eventfd.as_ref() + } + + /// Set guest kernel boot configurations. + pub fn set_kernel_config(&mut self, kernel_config: KernelConfigInfo) { + self.kernel_config = Some(kernel_config); + } + + /// Get virtual machine shared instance information. + pub fn shared_info(&self) -> &Arc> { + &self.shared_info } /// Gets a reference to the address_space.address_space for guest memory owned by this VM. @@ -253,11 +274,6 @@ impl Vm { self.address_space.get_address_space() } - /// Gets a reference to the device manager by this VM. - pub fn device_manager(&self) -> &DeviceManager { - &self.device_manager - } - /// Gets a reference to the address space for guest memory owned by this VM. /// /// Note that `GuestMemory` does not include any device memory that may have been added after @@ -276,24 +292,21 @@ impl Vm { self.vm_config = config; } - /// Set guest kernel boot configurations. - pub fn set_kernel_config(&mut self, kernel_config: KernelConfigInfo) { - self.kernel_config = Some(kernel_config); + /// Gets a reference to the kvm file descriptor owned by this VM. + pub fn vm_fd(&self) -> &VmFd { + &self.vm_fd } - /// Get virtual machine shared instance information. - pub fn shared_info(&self) -> &Arc> { - &self.shared_info - } + /// returns true if system upcall service is ready + pub fn is_upcall_client_ready(&self) -> bool { + #[cfg(feature = "hotplug")] + { + if let Some(upcall_client) = self.upcall_client() { + return upcall_client.is_ready(); + } + } - /// Get a reference to EpollManager. - pub fn epoll_manager(&self) -> &EpollManager { - &self.epoll_manager - } - - /// Get eventfd for exit notification. - pub fn get_reset_eventfd(&self) -> Option<&EventFd> { - self.reset_eventfd.as_ref() + false } /// Check whether the VM has been initialized. @@ -318,16 +331,16 @@ impl Vm { instance_state == InstanceState::Running } - /// returns true if system upcall service is ready - pub fn is_upcall_client_ready(&self) -> bool { - #[cfg(feature = "hotplug")] - { - if let Some(upcall_client) = self.upcall_client() { - return upcall_client.is_ready(); - } + /// Save VM instance exit state + pub fn vm_exit(&self, exit_code: i32) { + if let Ok(mut info) = self.shared_info.write() { + info.state = InstanceState::Exited(exit_code); + } else { + error!( + self.logger, + "Failed to save exit state, couldn't be written due to poisoned lock" + ); } - - false } /// Create device operation context. @@ -359,41 +372,6 @@ impl Vm { } } - /// Save VM instance exit state - pub fn vm_exit(&self, exit_code: i32) { - if let Ok(mut info) = self.shared_info.write() { - info.state = InstanceState::Exited(exit_code); - } else { - error!( - self.logger, - "Failed to save exit state, couldn't be written due to poisoned lock" - ); - } - } - - /// Reset the console into canonical mode. - pub fn reset_console(&self) -> std::result::Result<(), DeviceMgrError> { - self.device_manager.reset_console() - } - - fn get_dragonball_info(&self) -> (String, String) { - let guard = self.shared_info.read().unwrap(); - let instance_id = guard.id.clone(); - let dragonball_version = guard.vmm_version.clone(); - - (dragonball_version, instance_id) - } - - fn init_dmesg_logger(&mut self) { - let writer = self.dmesg_logger(); - self.dmesg_fifo = Some(writer); - } - - /// dmesg write to logger - pub fn dmesg_logger(&self) -> Box { - Box::new(DmesgWriter::new(self.logger.clone())) - } - pub(crate) fn check_health(&self) -> std::result::Result<(), StartMicrovmError> { if self.kernel_config.is_none() { return Err(StartMicrovmError::MissingKernelConfig); @@ -401,13 +379,23 @@ impl Vm { Ok(()) } + pub(crate) fn get_dragonball_info(&self) -> (String, String) { + let guard = self.shared_info.read().unwrap(); + let instance_id = guard.id.clone(); + let dragonball_version = guard.vmm_version.clone(); + + (dragonball_version, instance_id) + } +} + +impl Vm { pub(crate) fn init_vcpu_manager( &mut self, vm_as: GuestAddressSpaceImpl, vcpu_seccomp_filter: BpfProgram, ) -> std::result::Result<(), VcpuManagerError> { let vcpu_manager = VcpuManager::new( - self.fd.clone(), + self.vm_fd.clone(), &self.kvm, &self.vm_config, vm_as, @@ -422,7 +410,7 @@ impl Vm { } /// get the cpu manager's reference - pub fn vcpu_manager( + pub(crate) fn vcpu_manager( &self, ) -> std::result::Result, VcpuManagerError> { self.vcpu_manager @@ -448,69 +436,7 @@ impl Vm { Ok(()) } - pub(crate) fn init_guest_memory(&mut self) -> std::result::Result<(), StartMicrovmError> { - info!(self.logger, "VM: initializing guest memory..."); - - // We are not allowing reinitialization of vm guest memory. - if self.address_space.is_initialized() { - return Ok(()); - } - // vcpu boot up require local memory. reserve 100 MiB memory - let mem_size = (self.vm_config.mem_size_mib as u64) << 20; - let reserve_memory_bytes = self.vm_config.reserve_memory_bytes; - if reserve_memory_bytes > (mem_size >> 1) as u64 { - return Err(StartMicrovmError::ConfigureInvalid(String::from( - "invalid reserve_memory_bytes", - ))); - } - - let mem_type = self.vm_config.mem_type.clone(); - let mut mem_file_path = String::from(""); - if mem_type == "hugetlbfs" { - let shared_info = self.shared_info.read() - .expect("Failed to determine if instance is initialized because shared info couldn't be read due to poisoned lock"); - mem_file_path.push_str("/dragonball/"); - mem_file_path.push_str(shared_info.id.as_str()); - } - - // init default regions. - let mut numa_regions = Vec::with_capacity(1); - let mut vcpu_ids: Vec = Vec::new(); - - for i in 0..self.vm_config().max_vcpu_count { - vcpu_ids.push(i as u32); - } - let numa_node = NumaRegionInfo { - size: self.vm_config.mem_size_mib as u64, - host_numa_node_id: None, - guest_numa_node_id: Some(0), - vcpu_ids, - }; - numa_regions.push(numa_node); - - info!( - self.logger, - "VM: mem_type:{} mem_file_path:{}, mem_size:{}, reserve_memory_bytes:{}, \ - numa_regions:{:?}", - mem_type, - mem_file_path, - mem_size, - reserve_memory_bytes, - numa_regions, - ); - - let mut address_space_param = AddressSpaceMgrBuilder::new(&mem_type, &mem_file_path) - .map_err(StartMicrovmError::AddressManagerError)?; - address_space_param.set_kvm_vm_fd(self.fd.clone()); - self.address_space - .create_address_space(&self.resource_manager, &numa_regions, address_space_param) - .map_err(StartMicrovmError::AddressManagerError)?; - - info!(self.logger, "VM: initializing guest memory done"); - Ok(()) - } - - fn init_devices( + pub(crate) fn init_devices( &mut self, epoll_manager: EpollManager, ) -> std::result::Result<(), StartMicrovmError> { @@ -567,36 +493,108 @@ impl Vm { .map_err(StopMicrovmError::DeviceManager) } - fn load_kernel( - &mut self, - vm_memory: &GuestMemoryImpl, - ) -> std::result::Result { - // This is the easy way out of consuming the value of the kernel_cmdline. + /// Reset the console into canonical mode. + pub fn reset_console(&self) -> std::result::Result<(), DeviceMgrError> { + self.device_manager.reset_console() + } + pub(crate) fn init_dmesg_logger(&mut self) { + let writer = self.dmesg_logger(); + self.dmesg_fifo = Some(writer); + } + + /// dmesg write to logger + fn dmesg_logger(&self) -> Box { + Box::new(DmesgWriter::new(&self.logger)) + } + + pub(crate) fn init_guest_memory(&mut self) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: initializing guest memory..."); + // We are not allowing reinitialization of vm guest memory. + if self.address_space.is_initialized() { + return Ok(()); + } + + // vcpu boot up require local memory. reserve 100 MiB memory + let mem_size = (self.vm_config.mem_size_mib as u64) << 20; + let reserve_memory_bytes = self.vm_config.reserve_memory_bytes; + if reserve_memory_bytes > (mem_size >> 1) as u64 { + return Err(StartMicrovmError::ConfigureInvalid(String::from( + "invalid reserve_memory_bytes", + ))); + } + + let mem_type = self.vm_config.mem_type.clone(); + let mut mem_file_path = String::from(""); + if mem_type == "hugetlbfs" { + let shared_info = self.shared_info.read() + .expect("Failed to determine if instance is initialized because shared info couldn't be read due to poisoned lock"); + mem_file_path.push_str("/dragonball/"); + mem_file_path.push_str(shared_info.id.as_str()); + } + + let mut vcpu_ids: Vec = Vec::new(); + for i in 0..self.vm_config().max_vcpu_count { + vcpu_ids.push(i as u32); + } + + // init default regions. + let mut numa_regions = Vec::with_capacity(1); + let numa_node = NumaRegionInfo { + size: self.vm_config.mem_size_mib as u64, + host_numa_node_id: None, + guest_numa_node_id: Some(0), + vcpu_ids, + }; + numa_regions.push(numa_node); + + info!( + self.logger, + "VM: mem_type:{} mem_file_path:{}, mem_size:{}, reserve_memory_bytes:{}, \ + numa_regions:{:?}", + mem_type, + mem_file_path, + mem_size, + reserve_memory_bytes, + numa_regions, + ); + + let mut address_space_param = AddressSpaceMgrBuilder::new(&mem_type, &mem_file_path) + .map_err(StartMicrovmError::AddressManagerError)?; + address_space_param.set_kvm_vm_fd(self.vm_fd.clone()); + self.address_space + .create_address_space(&self.resource_manager, &numa_regions, address_space_param) + .map_err(StartMicrovmError::AddressManagerError)?; + + info!(self.logger, "VM: initializing guest memory done"); + Ok(()) + } + + fn init_configure_system( + &mut self, + vm_as: &GuestAddressSpaceImpl, + ) -> std::result::Result<(), StartMicrovmError> { + let vm_memory = vm_as.memory(); let kernel_config = self .kernel_config - .as_mut() + .as_ref() .ok_or(StartMicrovmError::MissingKernelConfig)?; + //let cmdline = kernel_config.cmdline.clone(); + let initrd: Option = match kernel_config.initrd_file() { + Some(f) => { + let initrd_file = f.try_clone(); + if initrd_file.is_err() { + return Err(StartMicrovmError::InitrdLoader( + LoadInitrdError::ReadInitrd(io::Error::from(io::ErrorKind::InvalidData)), + )); + } + let res = self.load_initrd(vm_memory.deref(), &mut initrd_file.unwrap())?; + Some(res) + } + None => None, + }; - let high_mem_addr = GuestAddress(dbs_boot::get_kernel_start()); - - #[cfg(target_arch = "x86_64")] - return linux_loader::loader::elf::Elf::load( - vm_memory, - None, - kernel_config.kernel_file_mut(), - Some(high_mem_addr), - ) - .map_err(StartMicrovmError::KernelLoader); - - #[cfg(target_arch = "aarch64")] - return linux_loader::loader::pe::PE::load( - vm_memory, - Some(GuestAddress(dbs_boot::get_kernel_start())), - kernel_config.kernel_file_mut(), - Some(high_mem_addr), - ) - .map_err(StartMicrovmError::KernelLoader); + self.configure_system_arch(vm_memory.deref(), kernel_config.kernel_cmdline(), initrd) } /// Loads the initrd from a file into the given memory slice. @@ -644,49 +642,46 @@ impl Vm { }) } - fn init_configure_system( + fn load_kernel( &mut self, - vm_as: &GuestAddressSpaceImpl, - ) -> std::result::Result<(), StartMicrovmError> { - let vm_memory = vm_as.memory(); + vm_memory: &GuestMemoryImpl, + ) -> std::result::Result { + // This is the easy way out of consuming the value of the kernel_cmdline. let kernel_config = self .kernel_config - .as_ref() + .as_mut() .ok_or(StartMicrovmError::MissingKernelConfig)?; - //let cmdline = kernel_config.cmdline.clone(); - let initrd: Option = match &kernel_config.initrd_file { - Some(f) => { - let initrd_file = f.try_clone(); - if initrd_file.is_err() { - return Err(StartMicrovmError::InitrdLoader( - LoadInitrdError::ReadInitrd(io::Error::from(io::ErrorKind::InvalidData)), - )); - } - let res = self.load_initrd(vm_memory.deref(), &mut initrd_file.unwrap())?; - Some(res) - } - None => None, - }; + let high_mem_addr = GuestAddress(dbs_boot::get_kernel_start()); - self.configure_system_arch(vm_memory.deref(), kernel_config.kernel_cmdline(), initrd) + #[cfg(target_arch = "x86_64")] + return linux_loader::loader::elf::Elf::load( + vm_memory, + None, + kernel_config.kernel_file_mut(), + Some(high_mem_addr), + ) + .map_err(StartMicrovmError::KernelLoader); + + #[cfg(target_arch = "aarch64")] + return linux_loader::loader::pe::PE::load( + vm_memory, + Some(GuestAddress(dbs_boot::get_kernel_start())), + kernel_config.kernel_file_mut(), + Some(high_mem_addr), + ) + .map_err(StartMicrovmError::KernelLoader); } } #[cfg(feature = "hotplug")] impl Vm { - /// Get upcall client. - pub fn upcall_client(&self) -> &Option>> { - &self.upcall_client - } - /// initialize upcall client for guest os - fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + pub(crate) fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { // get vsock inner connector for upcall let inner_connector = self .device_manager .get_vsock_inner_connector() .ok_or(StartMicrovmError::UpcallMissVsock)?; - let mut upcall_client = UpcallClient::new( inner_connector, self.epoll_manager.clone(), @@ -697,10 +692,14 @@ impl Vm { upcall_client .connect() .map_err(StartMicrovmError::UpcallConnectError)?; - self.upcall_client = Some(Arc::new(upcall_client)); info!(self.logger, "upcall client init success"); Ok(()) } + + /// Get upcall client. + pub fn upcall_client(&self) -> &Option>> { + &self.upcall_client + } } diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index a559962821..74336b6da0 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -15,6 +15,7 @@ use dbs_boot::{add_e820_entry, bootparam, layout, mptable, BootParamsWrapper, In use dbs_utils::epoll_manager::EpollManager; use dbs_utils::time::TimestampUs; use kvm_bindings::{kvm_irqchip, kvm_pit_config, kvm_pit_state2, KVM_PIT_SPEAKER_DUMMY}; +use linux_loader::cmdline::Cmdline; use slog::info; use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; @@ -22,8 +23,6 @@ use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, Result, StartMicrovmError}; use crate::vm::{Vm, VmError}; -use linux_loader::cmdline::Cmdline; - /// Configures the system and should be called once per vm before starting vcpu /// threads. /// @@ -40,7 +39,7 @@ use linux_loader::cmdline::Cmdline; /// * `max_cpus` - Max number of virtual CPUs the guest will have. /// * `rsv_mem_bytes` - Reserve memory from microVM.. #[allow(clippy::too_many_arguments)] -pub fn configure_system( +fn configure_system( guest_mem: &M, address_space: Option<&AddressSpace>, cmdline_addr: GuestAddress, @@ -136,12 +135,14 @@ pub fn configure_system( impl Vm { /// Get the status of in-kernel PIT. pub fn get_pit_state(&self) -> Result { - self.fd.get_pit2().map_err(|e| Error::Vm(VmError::Irq(e))) + self.vm_fd + .get_pit2() + .map_err(|e| Error::Vm(VmError::Irq(e))) } /// Set the status of in-kernel PIT. pub fn set_pit_state(&self, pit_state: &kvm_pit_state2) -> Result<()> { - self.fd + self.vm_fd .set_pit2(pit_state) .map_err(|e| Error::Vm(VmError::Irq(e))) } @@ -152,7 +153,7 @@ impl Vm { chip_id, ..kvm_irqchip::default() }; - self.fd + self.vm_fd .get_irqchip(&mut irqchip) .map(|_| irqchip) .map_err(|e| Error::Vm(VmError::Irq(e))) @@ -160,7 +161,7 @@ impl Vm { /// Set the status of in-kernel ioapic. pub fn set_irqchip_state(&self, irqchip: &kvm_irqchip) -> Result<()> { - self.fd + self.vm_fd .set_irqchip(irqchip) .map_err(|e| Error::Vm(VmError::Irq(e))) } @@ -204,7 +205,6 @@ impl Vm { let vm_memory = vm_as.memory(); let kernel_loader_result = self.load_kernel(vm_memory.deref())?; - self.vcpu_manager() .map_err(StartMicrovmError::Vcpu)? .create_boot_vcpus(request_ts, kernel_loader_result.kernel_load) @@ -243,7 +243,7 @@ impl Vm { /// Initializes the guest memory. pub(crate) fn init_tss(&mut self) -> std::result::Result<(), StartMicrovmError> { - self.fd + self.vm_fd .set_tss_address(dbs_boot::layout::KVM_TSS_ADDRESS.try_into().unwrap()) .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } @@ -252,7 +252,7 @@ impl Vm { pub(crate) fn setup_interrupt_controller( &mut self, ) -> std::result::Result<(), StartMicrovmError> { - self.fd + self.vm_fd .create_irq_chip() .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } @@ -269,7 +269,7 @@ impl Vm { // Safe because we know that our file is a VM fd, we know the kernel will only read the // correct amount of memory from our pointer, and we verify the return result. - self.fd + self.vm_fd .create_pit2(pit_config) .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) }