From d428a3f9b962b3bfa9cdc9f3d8f68c4571705490 Mon Sep 17 00:00:00 2001 From: Zhongtao Hu Date: Thu, 23 Feb 2023 16:54:08 +0800 Subject: [PATCH] runtim-rs: get guest memory details get memory block size and guest mem hotplug probe Fixes:#6356 Signed-off-by: Zhongtao Hu --- src/libs/kata-types/src/capabilities.rs | 22 +++++++-- src/runtime-rs/crates/agent/src/kata/agent.rs | 3 +- src/runtime-rs/crates/agent/src/kata/trans.rs | 12 ++++- src/runtime-rs/crates/agent/src/lib.rs | 1 + .../hypervisor/src/ch/inner_hypervisor.rs | 12 +++++ .../crates/hypervisor/src/ch/mod.rs | 17 ++++++- .../crates/hypervisor/src/dragonball/inner.rs | 17 +++++++ .../crates/hypervisor/src/dragonball/mod.rs | 17 ++++++- src/runtime-rs/crates/hypervisor/src/lib.rs | 5 ++- .../crates/hypervisor/src/qemu/inner.rs | 12 +++++ .../crates/hypervisor/src/qemu/mod.rs | 17 ++++++- .../runtimes/virt_container/src/sandbox.rs | 45 ++++++++++++++++++- 12 files changed, 170 insertions(+), 10 deletions(-) diff --git a/src/libs/kata-types/src/capabilities.rs b/src/libs/kata-types/src/capabilities.rs index 7d24ee2c1d..b45bba9421 100644 --- a/src/libs/kata-types/src/capabilities.rs +++ b/src/libs/kata-types/src/capabilities.rs @@ -19,6 +19,8 @@ pub enum CapabilityBits { FsSharingSupport, /// hypervisor supports hybrid-vsock HybridVsockSupport, + /// hypervisor supports memory hotplug probe interface + GuestMemoryHotplugProbe, } /// Capabilities describe a virtcontainers hypervisor capabilities through a bit mask. @@ -47,17 +49,22 @@ impl Capabilities { self.flags = flags; } - /// is_block_device_supported tells if an hypervisor supports block devices. + /// add CapabilityBits + pub fn add(&mut self, flags: CapabilityBits) { + self.flags |= flags; + } + + /// is_block_device_supported tells if the hypervisor supports block devices. pub fn is_block_device_supported(&self) -> bool { self.flags.and(CapabilityBits::BlockDeviceSupport) != 0 } - /// is_block_device_hotplug_supported tells if an hypervisor supports block devices. + /// is_block_device_hotplug_supported tells if the hypervisor supports block devices. pub fn is_block_device_hotplug_supported(&self) -> bool { self.flags.and(CapabilityBits::BlockDeviceHotplugSupport) != 0 } - /// is_multi_queue_supported tells if an hypervisor supports device multi queue support. + /// is_multi_queue_supported tells if the hypervisor supports device multi queue support. pub fn is_multi_queue_supported(&self) -> bool { self.flags.and(CapabilityBits::MultiQueueSupport) != 0 } @@ -71,6 +78,11 @@ impl Capabilities { pub fn is_fs_sharing_supported(&self) -> bool { self.flags.and(CapabilityBits::FsSharingSupport) != 0 } + + /// is_mem_hotplug_probe_supported tells if the hypervisor supports hotplug probe interface + pub fn is_mem_hotplug_probe_supported(&self) -> bool { + self.flags.and(CapabilityBits::GuestMemoryHotplugProbe) != 0 + } } #[cfg(test)] @@ -117,5 +129,9 @@ mod tests { // test set hybrid-vsock support cap.set(CapabilityBits::HybridVsockSupport); assert!(cap.is_hybrid_vsock_supported()); + // test append capabilities + cap.add(CapabilityBits::GuestMemoryHotplugProbe); + assert!(cap.is_mem_hotplug_probe_supported()); + assert!(cap.is_fs_sharing_supported()); } } diff --git a/src/runtime-rs/crates/agent/src/kata/agent.rs b/src/runtime-rs/crates/agent/src/kata/agent.rs index ee3e97bf85..ad5668ef07 100644 --- a/src/runtime-rs/crates/agent/src/kata/agent.rs +++ b/src/runtime-rs/crates/agent/src/kata/agent.rs @@ -122,5 +122,6 @@ impl_agent!( get_volume_stats | crate::VolumeStatsRequest | crate::VolumeStatsResponse | None, resize_volume | crate::ResizeVolumeRequest | crate::Empty | None, online_cpu_mem | crate::OnlineCPUMemRequest | crate::Empty | None, - get_metrics | crate::Empty | crate::MetricsResponse | None + get_metrics | crate::Empty | crate::MetricsResponse | None, + get_guest_details | crate::GetGuestDetailsRequest | crate::GuestDetailsResponse | None ); diff --git a/src/runtime-rs/crates/agent/src/kata/trans.rs b/src/runtime-rs/crates/agent/src/kata/trans.rs index 2190806822..d7b00aba2b 100644 --- a/src/runtime-rs/crates/agent/src/kata/trans.rs +++ b/src/runtime-rs/crates/agent/src/kata/trans.rs @@ -27,7 +27,7 @@ use crate::{ UpdateInterfaceRequest, UpdateRoutesRequest, VersionCheckResponse, VolumeStatsRequest, VolumeStatsResponse, WaitProcessRequest, WriteStreamRequest, }, - OomEventResponse, WaitProcessResponse, WriteStreamResponse, + GetGuestDetailsRequest, OomEventResponse, WaitProcessResponse, WriteStreamResponse, }; fn trans_vec>(from: Vec) -> Vec { @@ -723,6 +723,16 @@ impl From for agent::SetGuestDateTimeRequest { } } +impl From for agent::GuestDetailsRequest { + fn from(from: GetGuestDetailsRequest) -> Self { + Self { + mem_block_size: from.mem_block_size, + mem_hotplug_probe: from.mem_hotplug_probe, + ..Default::default() + } + } +} + impl From for AgentDetails { fn from(src: agent::AgentDetails) -> Self { Self { diff --git a/src/runtime-rs/crates/agent/src/lib.rs b/src/runtime-rs/crates/agent/src/lib.rs index 8dd71e8d7b..7326963ec9 100644 --- a/src/runtime-rs/crates/agent/src/lib.rs +++ b/src/runtime-rs/crates/agent/src/lib.rs @@ -93,4 +93,5 @@ pub trait Agent: AgentManager + HealthService + Send + Sync { async fn set_ip_tables(&self, req: SetIPTablesRequest) -> Result; async fn get_volume_stats(&self, req: VolumeStatsRequest) -> Result; async fn resize_volume(&self, req: ResizeVolumeRequest) -> Result; + async fn get_guest_details(&self, req: GetGuestDetailsRequest) -> Result; } diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index 6b8b7a0752..85a4ee3fa4 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -741,6 +741,18 @@ impl CloudHypervisorInner { pub(crate) async fn get_hypervisor_metrics(&self) -> Result { todo!() } + + pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) { + todo!() + } + + pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) { + todo!() + } + + pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { + todo!() + } } // Log all output from the CH process until a shutdown signal is received. diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs index 6f8b6c269d..328333b9d9 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -8,7 +8,7 @@ use crate::device::DeviceType; use crate::{Hypervisor, VcpuThreadIds}; use anyhow::{Context, Result}; use async_trait::async_trait; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use persist::sandbox_persist::Persist; use std::sync::Arc; @@ -162,6 +162,21 @@ impl Hypervisor for CloudHypervisor { let inner = self.inner.read().await; inner.get_hypervisor_metrics().await } + + async fn set_capabilities(&self, flag: CapabilityBits) { + let mut inner = self.inner.write().await; + inner.set_capabilities(flag) + } + + async fn set_guest_memory_block_size(&self, size: u32) { + let mut inner = self.inner.write().await; + inner.set_guest_memory_block_size(size); + } + + async fn guest_memory_block_size(&self) -> u32 { + let inner = self.inner.read().await; + inner.guest_memory_block_size_mb() + } } #[async_trait] diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 7879f1c44a..51660f1ba1 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -68,6 +68,9 @@ pub struct DragonballInner { /// dragonball capabilities pub(crate) capabilities: Capabilities, + + /// the size of memory block of guest OS + pub(crate) guest_memory_block_size_mb: u32, } impl DragonballInner { @@ -92,6 +95,7 @@ impl DragonballInner { run_dir: "".to_string(), cached_block_devices: Default::default(), capabilities, + guest_memory_block_size_mb: 0, } } @@ -346,6 +350,18 @@ impl DragonballInner { pub fn hypervisor_config(&self) -> HypervisorConfig { self.config.clone() } + + pub(crate) fn set_capabilities(&mut self, flag: CapabilityBits) { + self.capabilities.add(flag); + } + + pub(crate) fn set_guest_memory_block_size(&mut self, size: u32) { + self.guest_memory_block_size_mb = size; + } + + pub(crate) fn get_guest_memory_block_size(&self) -> u32 { + self.guest_memory_block_size_mb + } } #[async_trait] @@ -387,6 +403,7 @@ impl Persist for DragonballInner { pending_devices: vec![], cached_block_devices: hypervisor_state.cached_block_devices, capabilities: Capabilities::new(), + guest_memory_block_size_mb: 0, }) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index f0ed02f706..8a59e350a7 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -21,7 +21,7 @@ use dragonball::api::v1::{ Backend as DragonballBackend, NetworkInterfaceConfig as DragonballNetworkConfig, VirtioConfig as DragonballVirtioConfig, }; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; use tokio::sync::RwLock; use tracing::instrument; @@ -178,6 +178,21 @@ impl Hypervisor for Dragonball { let inner = self.inner.read().await; inner.get_hypervisor_metrics().await } + + async fn set_capabilities(&self, flag: CapabilityBits) { + let mut inner = self.inner.write().await; + inner.set_capabilities(flag) + } + + async fn set_guest_memory_block_size(&self, size: u32) { + let mut inner = self.inner.write().await; + inner.set_guest_memory_block_size(size); + } + + async fn guest_memory_block_size(&self) -> u32 { + let inner = self.inner.read().await; + inner.get_guest_memory_block_size() + } } #[async_trait] diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 737133de50..3716dff523 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -26,7 +26,7 @@ pub mod ch; use anyhow::Result; use async_trait::async_trait; use hypervisor_persist::HypervisorState; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH; @@ -103,4 +103,7 @@ pub trait Hypervisor: std::fmt::Debug + Send + Sync { async fn save_state(&self) -> Result; async fn capabilities(&self) -> Result; async fn get_hypervisor_metrics(&self) -> Result; + async fn set_capabilities(&self, flag: CapabilityBits); + async fn set_guest_memory_block_size(&self, size: u32); + async fn guest_memory_block_size(&self) -> u32; } diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs index e4a3c4e0b0..18096bfc43 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/inner.rs @@ -140,6 +140,18 @@ impl QemuInner { pub(crate) async fn get_hypervisor_metrics(&self) -> Result { todo!() } + + pub(crate) fn set_capabilities(&mut self, _flag: CapabilityBits) { + todo!() + } + + pub(crate) fn set_guest_memory_block_size(&mut self, _size: u32) { + todo!() + } + + pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { + todo!() + } } use crate::device::DeviceType; diff --git a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs index 65b4b6e57d..904e582865 100644 --- a/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/qemu/mod.rs @@ -10,7 +10,7 @@ use crate::hypervisor_persist::HypervisorState; use crate::Hypervisor; use crate::{HypervisorConfig, VcpuThreadIds}; use inner::QemuInner; -use kata_types::capabilities::Capabilities; +use kata_types::capabilities::{Capabilities, CapabilityBits}; use anyhow::Result; use async_trait::async_trait; @@ -157,4 +157,19 @@ impl Hypervisor for Qemu { let inner = self.inner.read().await; inner.get_hypervisor_metrics().await } + + async fn set_capabilities(&self, flag: CapabilityBits) { + let mut inner = self.inner.write().await; + inner.set_capabilities(flag) + } + + async fn set_guest_memory_block_size(&self, size: u32) { + let mut inner = self.inner.write().await; + inner.set_guest_memory_block_size(size); + } + + async fn guest_memory_block_size(&self) -> u32 { + let inner = self.inner.read().await; + inner.guest_memory_block_size_mb() + } } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index e103f18d70..5e1e999059 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -8,7 +8,9 @@ use std::sync::Arc; use agent::kata::KataAgent; use agent::types::KernelModule; -use agent::{self, Agent, GetIPTablesRequest, SetIPTablesRequest, VolumeStatsRequest}; +use agent::{ + self, Agent, GetGuestDetailsRequest, GetIPTablesRequest, SetIPTablesRequest, VolumeStatsRequest, +}; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use common::message::{Action, Message}; @@ -18,6 +20,7 @@ use hypervisor::VsockConfig; use hypervisor::{dragonball::Dragonball, BlockConfig, Hypervisor, HYPERVISOR_DRAGONBALL}; use hypervisor::{utils::get_hvsock_path, HybridVsockConfig, DEFAULT_GUEST_VSOCK_CID}; use kata_sys_util::hooks::HookStates; +use kata_types::capabilities::CapabilityBits; use kata_types::config::TomlConfig; use persist::{self, sandbox_persist::Persist}; use resource::manager::ManagerArgs; @@ -196,7 +199,41 @@ impl VirtSandbox { // * spec details: https://github.com/opencontainers/runtime-spec/blob/c1662686cff159595277b79322d0272f5182941b/config.md#createruntime-hooks let mut create_runtime_hook_states = HookStates::new(); create_runtime_hook_states.execute_hooks(create_runtime_hooks, Some(st.clone()))?; + Ok(()) + } + // store_guest_details will get the information from the guest OS, like memory block size, agent details and is memory hotplug probe support + async fn store_guest_details(&self) -> Result<()> { + // get the information from agent + let guest_details = self + .agent + .get_guest_details(GetGuestDetailsRequest { + mem_block_size: true, + mem_hotplug_probe: true, + }) + .await + .context("failed to store guest details")?; + + // set memory block size + self.hypervisor + .set_guest_memory_block_size(guest_details.mem_block_size_bytes as u32) + .await; + + // set memory hotplug probe + if guest_details.support_mem_hotplug_probe { + self.hypervisor + .set_capabilities(CapabilityBits::GuestMemoryHotplugProbe) + .await; + } + info!( + sl!(), + "memory block size is {}, memory probe support {}", + self.hypervisor.guest_memory_block_size().await, + self.hypervisor + .capabilities() + .await? + .is_mem_hotplug_probe_supported() + ); Ok(()) } @@ -375,6 +412,12 @@ impl Sandbox for VirtSandbox { .context("create sandbox")?; inner.state = SandboxState::Running; + + // get and store guest details + self.store_guest_details() + .await + .context("failed to store guest details")?; + let agent = self.agent.clone(); let sender = self.msg_sender.clone(); info!(sl!(), "oom watcher start");