diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 027e4000de..7a9ae2d960 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -507,6 +507,40 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "derivative" version = "2.2.0" @@ -977,6 +1011,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -1184,6 +1224,7 @@ dependencies = [ "regex", "safe-path", "serde", + "serde-enum-str", "serde_json", "slog", "slog-scope", @@ -2279,6 +2320,36 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-attributes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685" +dependencies = [ + "darling_core", + "serde-rename-rule", + "syn 1.0.109", +] + +[[package]] +name = "serde-enum-str" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde-attributes", + "syn 1.0.109", +] + +[[package]] +name = "serde-rename-rule" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70" + [[package]] name = "serde_derive" version = "1.0.137" diff --git a/src/libs/kata-types/Cargo.toml b/src/libs/kata-types/Cargo.toml index f0a0f4471e..57e21c6731 100644 --- a/src/libs/kata-types/Cargo.toml +++ b/src/libs/kata-types/Cargo.toml @@ -25,6 +25,7 @@ slog-scope = "4.4.0" serde_json = "1.0.73" thiserror = "1.0" toml = "0.5.8" +serde-enum-str = "0.4" oci = { path = "../oci" } safe-path = { path = "../safe-path" } diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 89da372de8..0b17b10628 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -15,7 +15,7 @@ use serde::Deserialize; use crate::config::default::DEFAULT_AGENT_TYPE_NAME; use crate::config::default::DEFAULT_HYPERVISOR; use crate::config::default::DEFAULT_RUNTIME_NAME; -use crate::config::hypervisor::get_hypervisor_plugin; +use crate::config::hypervisor::{get_hypervisor_plugin, HugePageType}; use crate::config::TomlConfig; use crate::sl; @@ -225,8 +225,11 @@ pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS: &str = pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str = "io.katacontainers.config.hypervisor.enable_mem_prealloc"; /// A sandbox annotation to specify if the memory should be pre-allocated from huge pages. -pub const KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES: &str = +pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES: &str = "io.katacontainers.config.hypervisor.enable_hugepages"; +/// A sandbox annotation to specify huge page mode of memory backend. +pub const KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE: &str = + "io.katacontainers.config.hypervisor.hugepage_type"; /// A sandbox annotation to soecify file based memory backend root directory. pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str = "io.katacontainers.config.hypervisor.file_mem_backend"; @@ -740,14 +743,29 @@ impl Annotation { return Err(bool_err); } }, - KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_value::(key) { - Ok(r) => { - hv.memory_info.enable_hugepages = r.unwrap_or_default(); + KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES => { + match self.get_value::(key) { + Ok(r) => { + hv.memory_info.enable_hugepages = r.unwrap_or_default(); + } + Err(_e) => { + return Err(bool_err); + } } - Err(_e) => { - return Err(bool_err); + } + KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE => { + match self.get_value::(key) { + Ok(r) => { + hv.memory_info.hugepage_type = r.unwrap_or_default(); + } + Err(e) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("parse huge pages type: {}, error: {}", value, e), + )); + } } - }, + } KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { hv.memory_info.validate_memory_backend_path(value)?; hv.memory_info.file_mem_backend = value.to_string(); diff --git a/src/libs/kata-types/src/config/hypervisor/mod.rs b/src/libs/kata-types/src/config/hypervisor/mod.rs index 9056a0dd00..19fb83d61a 100644 --- a/src/libs/kata-types/src/config/hypervisor/mod.rs +++ b/src/libs/kata-types/src/config/hypervisor/mod.rs @@ -29,6 +29,7 @@ use std::sync::{Arc, Mutex}; use lazy_static::lazy_static; use regex::RegexSet; +use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str}; use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX; @@ -540,6 +541,25 @@ impl MachineInfo { } } +/// Huge page type for VM RAM backend +#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq)] +pub enum HugePageType { + /// This will result in the VM memory being allocated using hugetlbfs backend. This is useful + /// when you want to use vhost-user network stacks within the container. This will automatically + /// result in memory pre allocation. + #[serde(rename = "hugetlbfs")] + Hugetlbfs, + /// This will result in the VM memory being allocated using transparant huge page backend. + #[serde(rename = "thp")] + THP, +} + +impl Default for HugePageType { + fn default() -> Self { + Self::Hugetlbfs + } +} + /// Virtual machine memory configuration information. #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct MemoryInfo { @@ -577,12 +597,18 @@ pub struct MemoryInfo { /// Enable huge pages for VM RAM, default false /// - /// Enabling this will result in the VM memory being allocated using huge pages. This is useful - /// when you want to use vhost-user network stacks within the container. This will automatically - /// result in memory pre allocation. + /// Enabling this will result in the VM memory being allocated using huge pages. + /// Its backend type is specified by item "hugepage_type" #[serde(default)] pub enable_hugepages: bool, + /// Select huge page type, default "hugetlbfs" + /// Following huge types are supported: + /// - hugetlbfs + /// - thp + #[serde(default)] + pub hugepage_type: HugePageType, + /// Specifies virtio-mem will be enabled or not. /// /// Please note that this option should be used with the command diff --git a/src/libs/kata-types/tests/test_config.rs b/src/libs/kata-types/tests/test_config.rs index 800a05f70b..d350df724b 100644 --- a/src/libs/kata-types/tests/test_config.rs +++ b/src/libs/kata-types/tests/test_config.rs @@ -10,15 +10,15 @@ mod tests { KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, - KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, - KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES, - KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, - KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, - KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, - KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, - KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES, - KATA_ANNO_CFG_RUNTIME_NAME, + KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES, + KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, + KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, + KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, + KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, + KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_PATH, + KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, + KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, + KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_RUNTIME_NAME, }; use kata_types::config::KataConfig; use kata_types::config::{QemuConfig, TomlConfig}; @@ -138,7 +138,7 @@ mod tests { "./test_file_backend_mem_root".to_string(), ); anno_hash.insert( - KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES.to_string(), + KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES.to_string(), "false".to_string(), ); anno_hash.insert( diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 66a5ac0ee7..64c63efa11 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -602,6 +602,40 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "dashmap" version = "5.5.0" @@ -1415,6 +1449,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -1552,6 +1592,7 @@ dependencies = [ "regex", "safe-path 0.1.0", "serde", + "serde-enum-str", "serde_json", "slog", "slog-scope", @@ -2948,6 +2989,36 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-attributes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685" +dependencies = [ + "darling_core", + "serde-rename-rule", + "syn 1.0.109", +] + +[[package]] +name = "serde-enum-str" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde-attributes", + "syn 1.0.109", +] + +[[package]] +name = "serde-rename-rule" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70" + [[package]] name = "serde_derive" version = "1.0.177" diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 6823dac9bf..68d4d7fbe2 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -7,7 +7,7 @@ use super::vmm_instance::VmmInstance; use crate::{ device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState, - DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, + DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM, }; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; @@ -19,7 +19,10 @@ use dragonball::{ use kata_sys_util::mount; use kata_types::{ capabilities::{Capabilities, CapabilityBits}, - config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH}, + config::{ + hypervisor::{HugePageType, Hypervisor as HypervisorConfig}, + KATA_PATH, + }, }; use nix::mount::MsFlags; use persist::sandbox_persist::Persist; @@ -175,7 +178,10 @@ impl DragonballInner { fn set_vm_base_config(&mut self) -> Result<()> { let serial_path = [&self.run_dir, "console.sock"].join("/"); let (mem_type, mem_file_path) = if self.config.memory_info.enable_hugepages { - (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)) + match self.config.memory_info.hugepage_type { + HugePageType::THP => (String::from(HUGE_SHMEM), String::from("")), + HugePageType::Hugetlbfs => (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)), + } } else { (String::from(SHMEM), String::from("")) }; diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index 69f9fe9b41..9e6184b74b 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -52,6 +52,7 @@ const VM_ROOTFS_FILESYSTEM_EROFS: &str = "erofs"; const DEV_HUGEPAGES: &str = "/dev/hugepages"; pub const HUGETLBFS: &str = "hugetlbfs"; const SHMEM: &str = "shmem"; +const HUGE_SHMEM: &str = "hugeshmem"; pub const HYPERVISOR_DRAGONBALL: &str = "dragonball"; pub const HYPERVISOR_QEMU: &str = "qemu"; diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index 2ddbcdbb9a..cbb916da2c 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -374,6 +374,40 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive-new" version = "0.5.9" @@ -809,6 +843,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -981,6 +1021,7 @@ dependencies = [ "regex", "safe-path", "serde", + "serde-enum-str", "serde_json", "slog", "slog-scope", @@ -1883,6 +1924,36 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-attributes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685" +dependencies = [ + "darling_core", + "serde-rename-rule", + "syn 1.0.109", +] + +[[package]] +name = "serde-enum-str" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde-attributes", + "syn 1.0.109", +] + +[[package]] +name = "serde-rename-rule" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70" + [[package]] name = "serde_derive" version = "1.0.177"