dragonball: add start microvm support

We add microvm start related support in thie pull request.

Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: wllenyj <wllenyj@linux.alibaba.com>
Signed-off-by: jingshan <jingshan@linux.alibaba.com>
Signed-off-by: Chao Wu <chaowu@linux.alibaba.com>

src/dragonball/src/api/v1/boot_source.rs

@@ -35,10 +35,6 @@ pub struct BootSourceConfig {
 /// Errors associated with actions on `BootSourceConfig`.
 #[derive(Debug, thiserror::Error)]
 pub enum BootSourceConfigError {
-    /// The virutal machine instance ID is invalid.
-    #[error("the virtual machine instance ID is invalid")]
-    InvalidVMID,
-
     /// The kernel file cannot be opened.
     #[error(
         "the kernel file cannot be opened due to invalid kernel path or invalid permissions: {0}"

src/dragonball/src/api/v1/vmm_action.rs

@@ -12,7 +12,7 @@ use std::sync::mpsc::{Receiver, Sender, TryRecvError};
 use log::{debug, error, warn};
 use vmm_sys_util::eventfd::EventFd;
 
-use crate::error::Result;
+use crate::error::{Result, StartMicrovmError, StopMicrovmError};
 use crate::event_manager::EventManager;
 use crate::vm::{KernelConfigInfo, VmConfigInfo};
 use crate::vmm::Vmm;
@@ -22,19 +22,39 @@ use super::*;
 /// Wrapper for all errors associated with VMM actions.
 #[derive(Debug, thiserror::Error)]
 pub enum VmmActionError {
+    /// Invalid virtual machine instance ID.
+    #[error("the virtual machine instance ID is invalid")]
+    InvalidVMID,
+
     /// The action `ConfigureBootSource` failed either because of bad user input or an internal
     /// error.
     #[error("failed to configure boot source for VM: {0}")]
     BootSource(#[source] BootSourceConfigError),
+
+    /// The action `StartMicroVm` failed either because of bad user input or an internal error.
+    #[error("failed to boot the VM: {0}")]
+    StartMicroVm(#[source] StartMicroVmError),
+
+    /// The action `StopMicroVm` failed either because of bad user input or an internal error.
+    #[error("failed to shutdown the VM: {0}")]
+    StopMicrovm(#[source] StopMicrovmError),
 }
 
 /// This enum represents the public interface of the VMM. Each action contains various
 /// bits of information (ids, paths, etc.).
 #[derive(Clone, Debug, PartialEq)]
 pub enum VmmAction {
-    /// Configure the boot source of the microVM using as input the `ConfigureBootSource`. This
+    /// Configure the boot source of the microVM using `BootSourceConfig`.
-    /// action can only be called before the microVM has booted.
+    /// This action can only be called before the microVM has booted.
     ConfigureBootSource(BootSourceConfig),
+
+    /// Launch the microVM. This action can only be called before the microVM has booted.
+    StartMicroVm,
+
+    /// Shutdown the vmicroVM. This action can only be called after the microVM has booted.
+    /// When vmm is used as the crate by the other process, which is need to
+    /// shutdown the vcpu threads and destory all of the object.
+    ShutdownMicroVm,
 }
 
 /// The enum represents the response sent by the VMM in case of success. The response is either
@@ -75,7 +95,7 @@ impl VmmService {
     }
 
     /// Handle requests from the HTTP API Server and send back replies.
-    pub fn run_vmm_action(&mut self, vmm: &mut Vmm, _event_mgr: &mut EventManager) -> Result<()> {
+    pub fn run_vmm_action(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> Result<()> {
         let request = match self.from_api.try_recv() {
             Ok(t) => *t,
             Err(TryRecvError::Empty) => {
@@ -92,6 +112,8 @@ impl VmmService {
             VmmAction::ConfigureBootSource(boot_source_body) => {
                 self.configure_boot_source(vmm, boot_source_body)
             }
+            VmmAction::StartMicroVm => self.start_microvm(vmm, event_mgr),
+            VmmAction::ShutdownMicroVm => self.shutdown_microvm(vmm),
         };
 
         debug!("send vmm response: {:?}", response);
@@ -113,12 +135,14 @@ impl VmmService {
         boot_source_config: BootSourceConfig,
     ) -> VmmRequestResult {
         use super::BootSourceConfigError::{
-            InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath, InvalidVMID,
+            InvalidInitrdPath, InvalidKernelCommandLine, InvalidKernelPath,
             UpdateNotAllowedPostBoot,
         };
         use super::VmmActionError::BootSource;
 
-        let vm = vmm.get_vm_by_id_mut("").ok_or(BootSource(InvalidVMID))?;
+        let vm = vmm
+            .get_vm_by_id_mut("")
+            .ok_or(VmmActionError::InvalidVMID)?;
         if vm.is_vm_initialized() {
             return Err(BootSource(UpdateNotAllowedPostBoot));
         }
@@ -145,4 +169,28 @@ impl VmmService {
 
         Ok(VmmData::Empty)
     }
+
+    fn start_microvm(&mut self, vmm: &mut Vmm, event_mgr: &mut EventManager) -> VmmRequestResult {
+        use self::StartMicrovmError::MicroVMAlreadyRunning;
+        use self::VmmActionError::StartMicrovm;
+
+        let vmm_seccomp_filter = vmm.vmm_seccomp_filter();
+        let vcpu_seccomp_filter = vmm.vcpu_seccomp_filter();
+        let vm = vmm
+            .get_vm_by_id_mut("")
+            .ok_or(VmmActionError::InvalidVMID)?;
+        if vm.is_vm_initialized() {
+            return Err(StartMicrovm(MicroVMAlreadyRunning));
+        }
+
+        vm.start_microvm(event_mgr, vmm_seccomp_filter, vcpu_seccomp_filter)
+            .map(|_| VmmData::Empty)
+            .map_err(StartMicrovm)
+    }
+
+    fn shutdown_microvm(&mut self, vmm: &mut Vmm) -> VmmRequestResult {
+        vmm.event_ctx.exit_evt_triggered = true;
+
+        Ok(VmmData::Empty)
+    }
 }

src/dragonball/src/error.rs

@@ -72,11 +72,15 @@ pub enum Error {
 
 /// Errors associated with starting the instance.
 #[derive(Debug, thiserror::Error)]
-pub enum StartMicrovmError {
+pub enum StartMicroVmError {
     /// Cannot read from an Event file descriptor.
     #[error("failure while reading from EventFd file descriptor")]
     EventFd,
 
+    /// Cannot add event to Epoll.
+    #[error("failure while registering epoll event for file descriptor")]
+    RegisterEvent,
+
     /// The start command was issued more than once.
     #[error("the virtual machine is already running")]
     MicroVMAlreadyRunning,

src/dragonball/src/event_manager.rs

@@ -27,8 +27,8 @@ pub(crate) const EPOLL_EVENT_API_REQUEST: u32 = 1;
 /// Shared information between vmm::vmm_thread_event_loop() and VmmEpollHandler.
 pub(crate) struct EventContext {
     pub api_event_fd: EventFd,
-    pub api_event_flag: bool,
+    pub api_event_triggered: bool,
-    pub exit_evt_flag: bool,
+    pub exit_evt_triggered: bool,
 }
 
 impl EventContext {
@@ -36,8 +36,8 @@ impl EventContext {
     pub fn new(api_event_fd: EventFd) -> Result<Self> {
         Ok(EventContext {
             api_event_fd,
-            api_event_flag: false,
+            api_event_triggered: false,
-            exit_evt_flag: false,
+            exit_evt_triggered: false,
         })
     }
 }
@@ -131,7 +131,7 @@ impl MutEventSubscriber for VmmEpollHandler {
                 if let Err(e) = vmm.event_ctx.api_event_fd.read() {
                     error!("event_manager: failed to read API eventfd, {:?}", e);
                 }
-                vmm.event_ctx.api_event_flag = true;
+                vmm.event_ctx.api_event_triggered = true;
                 self.vmm_event_count.fetch_add(1, Ordering::AcqRel);
             }
             EPOLL_EVENT_EXIT => {
@@ -144,7 +144,7 @@ impl MutEventSubscriber for VmmEpollHandler {
                     }
                     None => warn!("event_manager: leftover exit event in epoll context!"),
                 }
-                vmm.event_ctx.exit_evt_flag = true;
+                vmm.event_ctx.exit_evt_triggered = true;
                 self.vmm_event_count.fetch_add(1, Ordering::AcqRel);
             }
             _ => error!("event_manager: unknown epoll slot number {}", events.data()),

src/dragonball/src/vm/aarch64.rs

@@ -22,6 +22,7 @@ use vmm_sys_util::eventfd::EventFd;
 use super::{Vm, VmError};
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::error::{Error, StartMicrovmError};
+use crate::event_manager::EventManager;
 
 /// Configures the system and should be called once per vm before starting vcpu threads.
 /// For aarch64, we only setup the FDT.
@@ -142,4 +143,16 @@ impl Vm {
         )
         .map_err(StartMicrovmError::ConfigureSystem)
     }
+
+    pub(crate) fn register_events(
+        &mut self,
+        event_mgr: &mut EventManager,
+    ) -> std::result::Result<(), StartMicrovmError> {
+        let reset_evt = self.get_reset_eventfd().ok_or(StartMicrovmError::EventFd)?;
+        event_mgr
+            .register_exit_eventfd(reset_evt)
+            .map_err(|_| StartMicrovmError::RegisterEvent)?;
+
+        Ok(())
+    }
 }

src/dragonball/src/vm/mod.rs

@@ -31,6 +31,7 @@ use crate::api::v1::{InstanceInfo, InstanceState};
 use crate::device_manager::console_manager::DmesgWriter;
 use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext};
 use crate::error::{LoadInitrdError, Result, StartMicrovmError, StopMicrovmError};
+use crate::event_manager::EventManager;
 use crate::kvm_context::KvmContext;
 use crate::resource_manager::ResourceManager;
 use crate::vcpu::{VcpuManager, VcpuManagerError};
@@ -355,20 +356,7 @@ impl Vm {
         if !self.is_vm_initialized() {
             Ok(DeviceOpContext::create_boot_ctx(self, epoll_mgr))
         } else {
-            #[cfg(feature = "hotplug")]
+            self.create_device_hotplug_context(epoll_mgr)
-            {
-                if self.upcall_client().is_none() {
-                    Err(StartMicrovmError::UpcallMissVsock)
-                } else if self.is_upcall_client_ready() {
-                    Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr))
-                } else {
-                    Err(StartMicrovmError::UpcallNotReady)
-                }
-            }
-            #[cfg(not(feature = "hotplug"))]
-            {
-                Err(StartMicrovmError::MicroVMAlreadyRunning)
-            }
         }
     }
 
@@ -671,12 +659,70 @@ impl Vm {
         )
         .map_err(StartMicrovmError::KernelLoader);
     }
+
+    /// Set up the initial microVM state and start the vCPU threads.
+    ///
+    /// This is the main entrance of the Vm object, to bring up the virtual machine instance into
+    /// running state.
+    pub fn start_microvm(
+        &mut self,
+        event_mgr: &mut EventManager,
+        vmm_seccomp_filter: BpfProgram,
+        vcpu_seccomp_filter: BpfProgram,
+    ) -> std::result::Result<(), StartMicrovmError> {
+        info!(self.logger, "VM: received instance start command");
+        if self.is_vm_initialized() {
+            return Err(StartMicrovmError::MicroVMAlreadyRunning);
+        }
+
+        let request_ts = TimestampUs::default();
+        self.start_instance_request_ts = request_ts.time_us;
+        self.start_instance_request_cpu_ts = request_ts.cputime_us;
+
+        self.init_dmesg_logger();
+        self.check_health()?;
+
+        // Use expect() to crash if the other thread poisoned this lock.
+        self.shared_info
+            .write()
+            .expect("Failed to start microVM because shared info couldn't be written due to poisoned lock")
+            .state = InstanceState::Starting;
+
+        self.init_guest_memory()?;
+        let vm_as = self.vm_as().cloned().ok_or(StartMicrovmError::AddressManagerError(
+            AddressManagerError::GuestMemoryNotInitialized,
+        ))?;
+
+        self.init_vcpu_manager(vm_as.clone(), vcpu_seccomp_filter)
+            .map_err(StartMicrovmError::Vcpu)?;
+        self.init_microvm(event_mgr.epoll_manager(), vm_as.clone(), request_ts)?;
+        self.init_configure_system(&vm_as)?;
+        self.init_upcall()?;
+
+        info!(self.logger, "VM: register events");
+        self.register_events(event_mgr)?;
+
+        info!(self.logger, "VM: start vcpus");
+        self.vcpu_manager()
+            .map_err(StartMicrovmError::Vcpu)?
+            .start_boot_vcpus(vmm_seccomp_filter)
+            .map_err(StartMicrovmError::Vcpu)?;
+
+        // Use expect() to crash if the other thread poisoned this lock.
+        self.shared_info
+            .write()
+            .expect("Failed to start microVM because shared info couldn't be written due to poisoned lock")
+            .state = InstanceState::Running;
+
+        info!(self.logger, "VM started");
+        Ok(())
+    }
 }
 
 #[cfg(feature = "hotplug")]
 impl Vm {
     /// initialize upcall client for guest os
-    pub(crate) fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> {
+    fn new_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> {
         // get vsock inner connector for upcall
         let inner_connector = self
             .device_manager
@@ -698,8 +744,51 @@ impl Vm {
         Ok(())
     }
 
+    fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> {
+        info!(self.logger, "VM upcall init");
+        if let Err(e) = self.new_upcall() {
+            info!(
+                self.logger,
+                "VM upcall init failed, no support hotplug: {}", e
+            );
+            Err(e)
+        } else {
+            self.vcpu_manager()
+                .map_err(StartMicrovmError::Vcpu)?
+                .set_upcall_channel(self.upcall_client().clone());
+            Ok(())
+        }
+    }
+
     /// Get upcall client.
     pub fn upcall_client(&self) -> &Option<Arc<UpcallClient<DevMgrService>>> {
         &self.upcall_client
     }
+
+    fn create_device_hotplug_context(
+        &self,
+        epoll_mgr: Option<EpollManager>,
+    ) -> std::result::Result<DeviceOpContext, StartMicrovmError> {
+        if self.upcall_client().is_none() {
+            Err(StartMicrovmError::UpcallMissVsock)
+        } else if self.is_upcall_client_ready() {
+            Ok(DeviceOpContext::create_hotplug_ctx(self, epoll_mgr))
+        } else {
+            Err(StartMicrovmError::UpcallNotReady)
+        }
+    }
+}
+
+#[cfg(not(feature = "hotplug"))]
+impl Vm {
+    fn init_upcall(&mut self) -> std::result::Result<(), StartMicrovmError> {
+        Ok(())
+    }
+
+    fn create_device_hotplug_context(
+        &self,
+        _epoll_mgr: Option<EpollManager>,
+    ) -> std::result::Result<DeviceOpContext, StartMicrovmError> {
+        Err(StartMicrovmError::MicroVMAlreadyRunning)
+    }
 }

src/dragonball/src/vm/x86_64.rs

@@ -21,6 +21,7 @@ use vm_memory::{Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory};
 
 use crate::address_space_manager::{GuestAddressSpaceImpl, GuestMemoryImpl};
 use crate::error::{Error, Result, StartMicrovmError};
+use crate::event_manager::EventManager;
 use crate::vm::{Vm, VmError};
 
 /// Configures the system and should be called once per vm before starting vcpu
@@ -273,4 +274,20 @@ impl Vm {
             .create_pit2(pit_config)
             .map_err(|e| StartMicrovmError::ConfigureVm(VmError::VmSetup(e)))
     }
+
+    pub(crate) fn register_events(
+        &mut self,
+        event_mgr: &mut EventManager,
+    ) -> std::result::Result<(), StartMicrovmError> {
+        let reset_evt = self
+            .device_manager
+            .get_reset_eventfd()
+            .map_err(StartMicrovmError::DeviceManager)?;
+        event_mgr
+            .register_exit_eventfd(&reset_evt)
+            .map_err(|_| StartMicrovmError::RegisterEvent)?;
+        self.reset_eventfd = Some(reset_evt);
+
+        Ok(())
+    }
 }

src/dragonball/src/vmm.rs

@@ -124,18 +124,18 @@ impl Vmm {
                     }
 
                     let mut v = vmm.lock().unwrap();
-                    if v.event_ctx.api_event_flag {
+                    if v.event_ctx.api_event_triggered {
                         // The run_vmm_action() needs to access event_mgr, so it could
                         // not be handled in EpollHandler::handle_events(). It has been
                         // delayed to the main loop.
-                        v.event_ctx.api_event_flag = false;
+                        v.event_ctx.api_event_triggered = false;
                         service
                             .run_vmm_action(&mut v, &mut event_mgr)
                             .unwrap_or_else(|_| {
                                 warn!("got spurious notification from api thread");
                             });
                     }
-                    if v.event_ctx.exit_evt_flag {
+                    if v.event_ctx.exit_evt_triggered {
                         info!("Gracefully terminated VMM control loop");
                         return v.stop(EXIT_CODE_OK as i32);
                     }