From 89b9ba86032691f92edfc8b50d6b22067a836228 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Mon, 16 May 2022 01:42:19 +0800 Subject: [PATCH] dragonball: add set_vm_configuration api Set virtual machine configuration configurations. Signed-off-by: wllenyj --- src/dragonball/src/api/v1/machine_config.rs | 86 +++++++++++ src/dragonball/src/api/v1/mod.rs | 4 + src/dragonball/src/api/v1/vmm_action.rs | 154 +++++++++++++++++++- src/dragonball/src/vm/mod.rs | 9 +- 4 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 src/dragonball/src/api/v1/machine_config.rs diff --git a/src/dragonball/src/api/v1/machine_config.rs b/src/dragonball/src/api/v1/machine_config.rs new file mode 100644 index 0000000000..e4ae228679 --- /dev/null +++ b/src/dragonball/src/api/v1/machine_config.rs @@ -0,0 +1,86 @@ +// Copyright (C) 2022 Alibaba Cloud. All rights reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/// We only support this number of vcpus for now. Mostly because we have set all vcpu related metrics as u8 +/// and breaking u8 will take extra efforts. +pub const MAX_SUPPORTED_VCPUS: u8 = 254; + +/// Memory hotplug value should have alignment in this size (unit: MiB) +pub const MEMORY_HOTPLUG_ALIGHMENT: u8 = 64; + +/// Errors associated with configuring the microVM. +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum VmConfigError { + /// Cannot update the configuration of the microvm post boot. + #[error("update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// The max vcpu count is invalid. + #[error("the vCPU number shouldn't large than {}", MAX_SUPPORTED_VCPUS)] + VcpuCountExceedsMaximum, + + /// The vcpu count is invalid. When hyperthreading is enabled, the `cpu_count` must be either + /// 1 or an even number. + #[error( + "the vCPU number '{0}' can only be 1 or an even number when hyperthreading is enabled" + )] + InvalidVcpuCount(u8), + + /// The threads_per_core is invalid. It should be either 1 or 2. + #[error("the threads_per_core number '{0}' can only be 1 or 2")] + InvalidThreadsPerCore(u8), + + /// The cores_per_die is invalid. It should be larger than 0. + #[error("the cores_per_die number '{0}' can only be larger than 0")] + InvalidCoresPerDie(u8), + + /// The dies_per_socket is invalid. It should be larger than 0. + #[error("the dies_per_socket number '{0}' can only be larger than 0")] + InvalidDiesPerSocket(u8), + + /// The socket number is invalid. It should be either 1 or 2. + #[error("the socket number '{0}' can only be 1 or 2")] + InvalidSocket(u8), + + /// max vcpu count inferred from cpu topology(threads_per_core * cores_per_die * dies_per_socket * sockets) should be larger or equal to vcpu_count + #[error("the max vcpu count inferred from cpu topology '{0}' (threads_per_core * cores_per_die * dies_per_socket * sockets) should be larger or equal to vcpu_count")] + InvalidCpuTopology(u8), + + /// The max vcpu count is invalid. + #[error( + "the max vCPU number '{0}' shouldn't less than vCPU count and can only be 1 or an even number when hyperthreading is enabled" + )] + InvalidMaxVcpuCount(u8), + + /// The memory size is invalid. The memory can only be an unsigned integer. + #[error("the memory size 0x{0:x}MiB is invalid")] + InvalidMemorySize(usize), + + /// The hotplug memory size is invalid. The memory can only be an unsigned integer. + #[error( + "the hotplug memory size '{0}' (MiB) is invalid, must be multiple of {}", + MEMORY_HOTPLUG_ALIGHMENT + )] + InvalidHotplugMemorySize(usize), + + /// The memory type is invalid. + #[error("the memory type '{0}' is invalid")] + InvalidMemType(String), + + /// The memory file path is invalid. + #[error("the memory file path is invalid")] + InvalidMemFilePath(String), + + /// NUMA region memory size is invalid + #[error("Total size of memory in NUMA regions: {0}, should matches memory size in config")] + InvalidNumaRegionMemorySize(usize), + + /// NUMA region vCPU count is invalid + #[error("Total counts of vCPUs in NUMA regions: {0}, should matches max vcpu count in config")] + InvalidNumaRegionCpuCount(u16), + + /// NUMA region vCPU count is invalid + #[error("Max id of vCPUs in NUMA regions: {0}, should matches max vcpu count in config")] + InvalidNumaRegionCpuMaxId(u16), +} diff --git a/src/dragonball/src/api/v1/mod.rs b/src/dragonball/src/api/v1/mod.rs index c1ab3f5d32..2077c982e9 100644 --- a/src/dragonball/src/api/v1/mod.rs +++ b/src/dragonball/src/api/v1/mod.rs @@ -15,3 +15,7 @@ pub use self::boot_source::{BootSourceConfig, BootSourceConfigError, DEFAULT_KER /// Wrapper over the microVM general information. mod instance_info; pub use self::instance_info::{InstanceInfo, InstanceState}; + +/// Wrapper for configuring the memory and CPU of the microVM. +mod machine_config; +pub use self::machine_config::{VmConfigError, MAX_SUPPORTED_VCPUS}; diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 7f8a8b98db..754d877b49 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -9,12 +9,12 @@ use std::fs::File; use std::sync::mpsc::{Receiver, Sender, TryRecvError}; -use log::{debug, error, warn}; +use log::{debug, error, info, warn}; use vmm_sys_util::eventfd::EventFd; use crate::error::{Result, StartMicrovmError, StopMicrovmError}; use crate::event_manager::EventManager; -use crate::vm::{KernelConfigInfo, VmConfigInfo}; +use crate::vm::{CpuTopology, KernelConfigInfo, VmConfigInfo}; use crate::vmm::Vmm; use super::*; @@ -38,6 +38,11 @@ pub enum VmmActionError { /// The action `StopMicroVm` failed either because of bad user input or an internal error. #[error("failed to shutdown the VM: {0}")] StopMicrovm(#[source] StopMicrovmError), + + /// One of the actions `GetVmConfiguration` or `SetVmConfiguration` failed either because of bad + /// input or an internal error. + #[error("failed to set configuration for the VM: {0}")] + MachineConfig(#[source] VmConfigError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -55,6 +60,13 @@ pub enum VmmAction { /// When vmm is used as the crate by the other process, which is need to /// shutdown the vcpu threads and destory all of the object. ShutdownMicroVm, + + /// Get the configuration of the microVM. + GetVmConfiguration, + + /// Set the microVM configuration (memory & vcpu) using `VmConfig` as input. This + /// action can only be called before the microVM has booted. + SetVmConfiguration(VmConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -63,6 +75,8 @@ pub enum VmmAction { pub enum VmmData { /// No data is sent on the channel. Empty, + /// The microVM configuration represented by `VmConfigInfo`. + MachineConfiguration(Box), } /// Request data type used to communicate between the API and the VMM. @@ -114,6 +128,12 @@ impl VmmService { } VmmAction::StartMicroVm => self.start_microvm(vmm, event_mgr), VmmAction::ShutdownMicroVm => self.shutdown_microvm(vmm), + VmmAction::GetVmConfiguration => Ok(VmmData::MachineConfiguration(Box::new( + self.machine_config.clone(), + ))), + VmmAction::SetVmConfiguration(machine_config) => { + self.set_vm_configuration(vmm, machine_config) + } }; debug!("send vmm response: {:?}", response); @@ -193,4 +213,134 @@ impl VmmService { Ok(VmmData::Empty) } + + /// Set virtual machine configuration configurations. + pub fn set_vm_configuration( + &mut self, + vmm: &mut Vmm, + machine_config: VmConfigInfo, + ) -> VmmRequestResult { + use self::VmConfigError::*; + use self::VmmActionError::MachineConfig; + + let vm = vmm + .get_vm_by_id_mut("") + .ok_or(VmmActionError::InvalidVMID)?; + if vm.is_vm_initialized() { + return Err(MachineConfig(UpdateNotAllowedPostBoot)); + } + + // If the check is successful, set it up together. + let mut config = vm.vm_config().clone(); + if config.vcpu_count != machine_config.vcpu_count { + let vcpu_count = machine_config.vcpu_count; + // Check that the vcpu_count value is >=1. + if vcpu_count == 0 { + return Err(MachineConfig(InvalidVcpuCount(vcpu_count))); + } + config.vcpu_count = vcpu_count; + } + + if config.cpu_topology != machine_config.cpu_topology { + let cpu_topology = &machine_config.cpu_topology; + // Check if dies_per_socket, cores_per_die, threads_per_core and socket number is valid + if cpu_topology.threads_per_core < 1 || cpu_topology.threads_per_core > 2 { + return Err(MachineConfig(InvalidThreadsPerCore( + cpu_topology.threads_per_core, + ))); + } + let vcpu_count_from_topo = cpu_topology + .sockets + .checked_mul(cpu_topology.dies_per_socket) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))? + .checked_mul(cpu_topology.cores_per_die) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))? + .checked_mul(cpu_topology.threads_per_core) + .ok_or(MachineConfig(VcpuCountExceedsMaximum))?; + if vcpu_count_from_topo > MAX_SUPPORTED_VCPUS { + return Err(MachineConfig(VcpuCountExceedsMaximum)); + } + if vcpu_count_from_topo < config.vcpu_count { + return Err(MachineConfig(InvalidCpuTopology(vcpu_count_from_topo))); + } + config.cpu_topology = cpu_topology.clone(); + } else { + // the same default + let mut default_cpu_topology = CpuTopology { + threads_per_core: 1, + cores_per_die: config.vcpu_count, + dies_per_socket: 1, + sockets: 1, + }; + if machine_config.max_vcpu_count > config.vcpu_count { + default_cpu_topology.cores_per_die = machine_config.max_vcpu_count; + } + config.cpu_topology = default_cpu_topology; + } + let cpu_topology = &config.cpu_topology; + let max_vcpu_from_topo = cpu_topology.threads_per_core + * cpu_topology.cores_per_die + * cpu_topology.dies_per_socket + * cpu_topology.sockets; + // If the max_vcpu_count inferred by cpu_topology is not equal to + // max_vcpu_count, max_vcpu_count will be changed. currently, max vcpu size + // is used when cpu_topology is not defined and help define the cores_per_die + // for the default cpu topology. + let mut max_vcpu_count = machine_config.max_vcpu_count; + if max_vcpu_count < config.vcpu_count { + return Err(MachineConfig(InvalidMaxVcpuCount(max_vcpu_count))); + } + if max_vcpu_from_topo != max_vcpu_count { + max_vcpu_count = max_vcpu_from_topo; + info!("Since max_vcpu_count is not equal to cpu topo information, we have changed the max vcpu count to {}", max_vcpu_from_topo); + } + config.max_vcpu_count = max_vcpu_count; + + config.cpu_pm = machine_config.cpu_pm; + config.mem_type = machine_config.mem_type; + + let mem_size_mib_value = machine_config.mem_size_mib; + // Support 1TB memory at most, 2MB aligned for huge page. + if mem_size_mib_value == 0 || mem_size_mib_value > 0x10_0000 || mem_size_mib_value % 2 != 0 + { + return Err(MachineConfig(InvalidMemorySize(mem_size_mib_value))); + } + config.mem_size_mib = mem_size_mib_value; + + config.mem_file_path = machine_config.mem_file_path.clone(); + + let reserve_memory_bytes = machine_config.reserve_memory_bytes; + // Reserved memory must be 2MB aligned and less than half of the total memory. + if reserve_memory_bytes % 0x200000 != 0 + || reserve_memory_bytes > (config.mem_size_mib as u64) << 20 + { + return Err(MachineConfig(InvalidReservedMemorySize( + reserve_memory_bytes as usize >> 20, + ))); + } + config.reserve_memory_bytes = reserve_memory_bytes; + if config.mem_type == "hugetlbfs" && config.mem_file_path.is_empty() { + return Err(MachineConfig(InvalidMemFilePath("".to_owned()))); + } + config.vpmu_feature = machine_config.vpmu_feature; + + let vm_id = vm.shared_info().read().unwrap().id.clone(); + let serial_path = match machine_config.serial_path { + Some(value) => value, + None => { + if config.serial_path.is_none() { + String::from("/run/dragonball/") + &vm_id + "_com1" + } else { + // Safe to unwrap() because we have checked it has a value. + config.serial_path.as_ref().unwrap().clone() + } + } + }; + config.serial_path = Some(serial_path); + + vm.set_vm_config(config.clone()); + self.machine_config = config; + + Ok(VmmData::Empty) + } } diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 1089c99bf2..7c83aabb4c 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -689,9 +689,12 @@ impl Vm { .state = InstanceState::Starting; self.init_guest_memory()?; - let vm_as = self.vm_as().cloned().ok_or(StartMicrovmError::AddressManagerError( - AddressManagerError::GuestMemoryNotInitialized, - ))?; + let vm_as = self + .vm_as() + .cloned() + .ok_or(StartMicrovmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + ))?; self.init_vcpu_manager(vm_as.clone(), vcpu_seccomp_filter) .map_err(StartMicrovmError::Vcpu)?;