diff --git a/src/dragonball/src/api/v1/boot_source.rs b/src/dragonball/src/api/v1/boot_source.rs index e7de030438..0094c8d7c7 100644 --- a/src/dragonball/src/api/v1/boot_source.rs +++ b/src/dragonball/src/api/v1/boot_source.rs @@ -35,10 +35,6 @@ pub struct BootSourceConfig { /// Errors associated with actions on `BootSourceConfig`. #[derive(Debug, thiserror::Error)] pub enum BootSourceConfigError { - /// The virutal machine instance ID is invalid. - #[error("the virtual machine instance ID is invalid")] - InvalidVMID, - /// The kernel file cannot be opened. #[error( "the kernel file cannot be opened due to invalid kernel path or invalid permissions: {0}" diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 1d7ac7e630..7f8a8b98db 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -12,7 +12,7 @@ use std::sync::mpsc::{Receiver, Sender, TryRecvError}; use log::{debug, error, warn}; use vmm_sys_util::eventfd::EventFd; -use crate::error::Result; +use crate::error::{Result, StartMicrovmError, StopMicrovmError}; use crate::event_manager::EventManager; use crate::vm::{KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; @@ -22,19 +22,39 @@ use super::*; /// Wrapper for all errors associated with VMM actions. #[derive(Debug, thiserror::Error)] pub enum VmmActionError { + /// Invalid virtual machine instance ID. + #[error("the virtual machine instance ID is invalid")] + InvalidVMID, + /// The action `ConfigureBootSource` failed either because of bad user input or an internal /// error. #[error("failed to configure boot source for VM: {0}")] BootSource(#[source] BootSourceConfigError), + + /// The action `StartMicroVm` failed either because of bad user input or an internal error. + #[error("failed to boot the VM: {0}")] + StartMicroVm(#[source] StartMicroVmError), + + /// The action `StopMicroVm` failed either because of bad user input or an internal error. + #[error("failed to shutdown the VM: {0}")] + StopMicrovm(#[source] StopMicrovmError), } /// This enum represents the public interface of the VMM. Each action contains various /// bits of information (ids, paths, etc.). #[derive(Clone, Debug, PartialEq)] pub enum VmmAction { - /// Configure the boot source of the microVM using as input the `ConfigureBootSource`. This - /// action can only be called before the microVM has booted. + /// Configure the boot source of the microVM using `BootSourceConfig`. + /// This action can only be called before the microVM has booted. ConfigureBootSource(BootSourceConfig), + + /// Launch the microVM. This action can only be called before the microVM has booted. + StartMicroVm, + + /// Shutdown the vmicroVM. This action can only be called after the microVM has booted. + /// When vmm is used as the crate by the other process, which is need to + /// shutdown the vcpu threads and destory all of the object. + ShutdownMicroVm, } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -75,7 +95,7 @@ impl VmmService { } /// Handle requests from the HTTP API Server and send back replies. - pub fn run_vmm_action(&mut self, vmm: &mut Vmm, _event_mgr: &mut EventManager) -> Result<()> { + pub fn run_vmm_action(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> Result<()> { let request = match self.from_api.try_recv() { Ok(t) => *t, Err(TryRecvError::Empty) => { @@ -92,6 +112,8 @@ impl VmmService { VmmAction::ConfigureBootSource(boot_source_body) => { self.configure_boot_source(vmm, boot_source_body) } + VmmAction::StartMicroVm => self.start_microvm(vmm, event_mgr), + VmmAction::ShutdownMicroVm => self.shutdown_microvm(vmm), }; debug!("send vmm response: {:?}", response); @@ -113,12 +135,14 @@ impl VmmService { boot_source_config: BootSourceConfig, ) -> VmmRequestResult { use super::BootSourceConfigError::{ - InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath, InvalidVMID, + InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath, UpdateNotAllowedPostBoot, }; use super::VmmActionError::BootSource; - let vm = vmm.get_vm_by_id_mut("").ok_or(BootSource(InvalidVMID))?; + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; if vm.is_vm_initialized() { return Err(BootSource(UpdateNotAllowedPostBoot)); } @@ -145,4 +169,28 @@ impl VmmService { Ok(VmmData::Empty) } + + fn start_microvm(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> VmmRequestResult { + use self::StartMicrovmError::MicroVMAlreadyRunning; + use self::VmmActionError::StartMicrovm; + + let vmm_seccomp_filter = vmm.vmm_seccomp_filter(); + let vcpu_seccomp_filter = vmm.vcpu_seccomp_filter(); + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + if vm.is_vm_initialized() { + return Err(StartMicrovm(MicroVMAlreadyRunning)); + } + + vm.start_microvm(event_mgr, vmm_seccomp_filter, vcpu_seccomp_filter) + .map(|_| VmmData::Empty) + .map_err(StartMicrovm) + } + + fn shutdown_microvm(&mut self, vmm: &mut Vmm) -> VmmRequestResult { + vmm.event_ctx.exit_evt_triggered = true; + + Ok(VmmData::Empty) + } } diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 9cb27fdd73..f6b37c867d 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -72,11 +72,15 @@ pub enum Error { /// Errors associated with starting the instance. #[derive(Debug, thiserror::Error)] -pub enum StartMicrovmError { +pub enum StartMicroVmError { /// Cannot read from an Event file descriptor. #[error("failure while reading from EventFd file descriptor")] EventFd, + /// Cannot add event to Epoll. + #[error("failure while registering epoll event for file descriptor")] + RegisterEvent, + /// The start command was issued more than once. #[error("the virtual machine is already running")] MicroVMAlreadyRunning, diff --git a/src/dragonball/src/event_manager.rs b/src/dragonball/src/event_manager.rs index cd0da10c87..9a2ad9d1c7 100644 --- a/src/dragonball/src/event_manager.rs +++ b/src/dragonball/src/event_manager.rs @@ -27,8 +27,8 @@ pub(crate) const EPOLL_EVENT_API_REQUEST: u32 = 1; /// Shared information between vmm::vmm_thread_event_loop() and VmmEpollHandler. pub(crate) struct EventContext { pub api_event_fd: EventFd, - pub api_event_flag: bool, - pub exit_evt_flag: bool, + pub api_event_triggered: bool, + pub exit_evt_triggered: bool, } impl EventContext { @@ -36,8 +36,8 @@ impl EventContext { pub fn new(api_event_fd: EventFd) -> Result { Ok(EventContext { api_event_fd, - api_event_flag: false, - exit_evt_flag: false, + api_event_triggered: false, + exit_evt_triggered: false, }) } } @@ -131,7 +131,7 @@ impl MutEventSubscriber for VmmEpollHandler { if let Err(e) = vmm.event_ctx.api_event_fd.read() { error!("event_manager: failed to read API eventfd, {:?}", e); } - vmm.event_ctx.api_event_flag = true; + vmm.event_ctx.api_event_triggered = true; self.vmm_event_count.fetch_add(1, Ordering::AcqRel); } EPOLL_EVENT_EXIT => { @@ -144,7 +144,7 @@ impl MutEventSubscriber for VmmEpollHandler { } None => warn!("event_manager: leftover exit event in epoll context!"), } - vmm.event_ctx.exit_evt_flag = true; + vmm.event_ctx.exit_evt_triggered = true; self.vmm_event_count.fetch_add(1, Ordering::AcqRel); } _ => error!("event_manager: unknown epoll slot number {}", events.data()), diff --git a/src/dragonball/src/vm/aarch64.rs b/src/dragonball/src/vm/aarch64.rs index 739b1515c6..25450f380d 100644 --- a/src/dragonball/src/vm/aarch64.rs +++ b/src/dragonball/src/vm/aarch64.rs @@ -22,6 +22,7 @@ use vmm_sys_util::eventfd::EventFd; use super::{Vm, VmError}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, StartMicrovmError}; +use crate::event_manager::EventManager; /// Configures the system and should be called once per vm before starting vcpu threads. /// For aarch64, we only setup the FDT. @@ -142,4 +143,16 @@ impl Vm { ) .map_err(StartMicrovmError::ConfigureSystem) } + + pub(crate) fn register_events( + &mut self, + event_mgr: &mut EventManager, + ) -> std::result::Result<(), StartMicrovmError> { + let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?; + event_mgr + .register_exit_eventfd(reset_evt) + .map_err(|_| StartMicrovmError::RegisterEvent)?; + + Ok(()) + } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index fff044e3aa..1089c99bf2 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -31,6 +31,7 @@ use crate::api::v1::{InstanceInfo, InstanceState}; use crate::device_manager::console_manager::DmesgWriter; use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; use crate::error::{LoadInitrdError, Result, StartMicrovmError, StopMicrovmError}; +use crate::event_manager::EventManager; use crate::kvm_context::KvmContext; use crate::resource_manager::ResourceManager; use crate::vcpu::{VcpuManager, VcpuManagerError}; @@ -355,20 +356,7 @@ impl Vm { if !self.is_vm_initialized() { Ok(DeviceOpContext::create_boot_ctx(self, epoll_mgr)) } else { - #[cfg(feature = "hotplug")] - { - if self.upcall_client().is_none() { - Err(StartMicrovmError::UpcallMissVsock) - } else if self.is_upcall_client_ready() { - Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) - } else { - Err(StartMicrovmError::UpcallNotReady) - } - } - #[cfg(not(feature = "hotplug"))] - { - Err(StartMicrovmError::MicroVMAlreadyRunning) - } + self.create_device_hotplug_context(epoll_mgr) } } @@ -671,12 +659,70 @@ impl Vm { ) .map_err(StartMicrovmError::KernelLoader); } + + /// Set up the initial microVM state and start the vCPU threads. + /// + /// This is the main entrance of the Vm object, to bring up the virtual machine instance into + /// running state. + pub fn start_microvm( + &mut self, + event_mgr: &mut EventManager, + vmm_seccomp_filter: BpfProgram, + vcpu_seccomp_filter: BpfProgram, + ) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM: received instance start command"); + if self.is_vm_initialized() { + return Err(StartMicrovmError::MicroVMAlreadyRunning); + } + + let request_ts = TimestampUs::default(); + self.start_instance_request_ts = request_ts.time_us; + self.start_instance_request_cpu_ts = request_ts.cputime_us; + + self.init_dmesg_logger(); + self.check_health()?; + + // Use expect() to crash if the other thread poisoned this lock. + self.shared_info + .write() + .expect("Failed to start microVM because shared info couldn't be written due to poisoned lock") + .state = InstanceState::Starting; + + self.init_guest_memory()?; + let vm_as = self.vm_as().cloned().ok_or(StartMicrovmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + ))?; + + self.init_vcpu_manager(vm_as.clone(), vcpu_seccomp_filter) + .map_err(StartMicrovmError::Vcpu)?; + self.init_microvm(event_mgr.epoll_manager(), vm_as.clone(), request_ts)?; + self.init_configure_system(&vm_as)?; + self.init_upcall()?; + + info!(self.logger, "VM: register events"); + self.register_events(event_mgr)?; + + info!(self.logger, "VM: start vcpus"); + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .start_boot_vcpus(vmm_seccomp_filter) + .map_err(StartMicrovmError::Vcpu)?; + + // Use expect() to crash if the other thread poisoned this lock. + self.shared_info + .write() + .expect("Failed to start microVM because shared info couldn't be written due to poisoned lock") + .state = InstanceState::Running; + + info!(self.logger, "VM started"); + Ok(()) + } } #[cfg(feature = "hotplug")] impl Vm { /// initialize upcall client for guest os - pub(crate) fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + fn new_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { // get vsock inner connector for upcall let inner_connector = self .device_manager @@ -698,8 +744,51 @@ impl Vm { Ok(()) } + fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + info!(self.logger, "VM upcall init"); + if let Err(e) = self.new_upcall() { + info!( + self.logger, + "VM upcall init failed, no support hotplug: {}", e + ); + Err(e) + } else { + self.vcpu_manager() + .map_err(StartMicrovmError::Vcpu)? + .set_upcall_channel(self.upcall_client().clone()); + Ok(()) + } + } + /// Get upcall client. pub fn upcall_client(&self) -> &Option>> { &self.upcall_client } + + fn create_device_hotplug_context( + &self, + epoll_mgr: Option, + ) -> std::result::Result { + if self.upcall_client().is_none() { + Err(StartMicrovmError::UpcallMissVsock) + } else if self.is_upcall_client_ready() { + Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr)) + } else { + Err(StartMicrovmError::UpcallNotReady) + } + } +} + +#[cfg(not(feature = "hotplug"))] +impl Vm { + fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> { + Ok(()) + } + + fn create_device_hotplug_context( + &self, + _epoll_mgr: Option, + ) -> std::result::Result { + Err(StartMicrovmError::MicroVMAlreadyRunning) + } } diff --git a/src/dragonball/src/vm/x86_64.rs b/src/dragonball/src/vm/x86_64.rs index 74336b6da0..6bd74acc09 100644 --- a/src/dragonball/src/vm/x86_64.rs +++ b/src/dragonball/src/vm/x86_64.rs @@ -21,6 +21,7 @@ use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory}; use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl}; use crate::error::{Error, Result, StartMicrovmError}; +use crate::event_manager::EventManager; use crate::vm::{Vm, VmError}; /// Configures the system and should be called once per vm before starting vcpu @@ -273,4 +274,20 @@ impl Vm { .create_pit2(pit_config) .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e))) } + + pub(crate) fn register_events( + &mut self, + event_mgr: &mut EventManager, + ) -> std::result::Result<(), StartMicrovmError> { + let reset_evt = self + .device_manager + .get_reset_eventfd() + .map_err(StartMicrovmError::DeviceManager)?; + event_mgr + .register_exit_eventfd(&reset_evt) + .map_err(|_| StartMicrovmError::RegisterEvent)?; + self.reset_eventfd = Some(reset_evt); + + Ok(()) + } } diff --git a/src/dragonball/src/vmm.rs b/src/dragonball/src/vmm.rs index a2c75ff716..362415dbd9 100644 --- a/src/dragonball/src/vmm.rs +++ b/src/dragonball/src/vmm.rs @@ -124,18 +124,18 @@ impl Vmm { } let mut v = vmm.lock().unwrap(); - if v.event_ctx.api_event_flag { + if v.event_ctx.api_event_triggered { // The run_vmm_action() needs to access event_mgr, so it could // not be handled in EpollHandler::handle_events(). It has been // delayed to the main loop. - v.event_ctx.api_event_flag = false; + v.event_ctx.api_event_triggered = false; service .run_vmm_action(&mut v, &mut event_mgr) .unwrap_or_else(|_| { warn!("got spurious notification from api thread"); }); } - if v.event_ctx.exit_evt_flag { + if v.event_ctx.exit_evt_triggered { info!("Gracefully terminated VMM control loop"); return v.stop(EXIT_CODE_OK as i32); }