feat(runtime-rs): introduce huge page type to select VM RAM's backend

This commit allows us to specify the huge page backend when enabling huge
page. Currently, we support two backends: thp and hugetlbfs, the default
is hugetlbfs.

To ensure backward compatibility, we introduce another configuration item
"hugepage_type" to select the memory backend, which is available only when
"enable_hugepages" is true. Besides, we add an annotation
"io.katacontainers.config.hypervisor.hugepage_type" to configure huge page
type per pod.

Fixes: #6703

Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Yipeng Yin <yinyipeng@bytedance.com>

src/libs/kata-types/Cargo.toml

@@ -25,6 +25,7 @@ slog-scope = "4.4.0"
 serde_json = "1.0.73"
 thiserror = "1.0"
 toml = "0.5.8"
+serde-enum-str = "0.4"
 
 oci = { path = "../oci" }
 safe-path = { path = "../safe-path" }

src/libs/kata-types/src/annotations/mod.rs

@@ -15,7 +15,7 @@ use serde::Deserialize;
 use crate::config::default::DEFAULT_AGENT_TYPE_NAME;
 use crate::config::default::DEFAULT_HYPERVISOR;
 use crate::config::default::DEFAULT_RUNTIME_NAME;
-use crate::config::hypervisor::get_hypervisor_plugin;
+use crate::config::hypervisor::{get_hypervisor_plugin, HugePageType};
 
 use crate::config::TomlConfig;
 use crate::sl;
@@ -225,8 +225,11 @@ pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS: &str =
 pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str =
     "io.katacontainers.config.hypervisor.enable_mem_prealloc";
 /// A sandbox annotation to specify if the memory should be pre-allocated from huge pages.
-pub const KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES: &str =
+pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES: &str =
     "io.katacontainers.config.hypervisor.enable_hugepages";
+/// A sandbox annotation to specify huge page mode of memory backend.
+pub const KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE: &str =
+    "io.katacontainers.config.hypervisor.hugepage_type";
 /// A sandbox annotation to soecify file based memory backend root directory.
 pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str =
     "io.katacontainers.config.hypervisor.file_mem_backend";
@@ -740,14 +743,29 @@ impl Annotation {
                             return Err(bool_err);
                         }
                     },
-                    KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_value::<bool>(key) {
+                    KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES => {
+                        match self.get_value::<bool>(key) {
                             Ok(r) => {
                                 hv.memory_info.enable_hugepages = r.unwrap_or_default();
                             }
                             Err(_e) => {
                                 return Err(bool_err);
                             }
-                    },
+                        }
+                    }
+                    KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE => {
+                        match self.get_value::<HugePageType>(key) {
+                            Ok(r) => {
+                                hv.memory_info.hugepage_type = r.unwrap_or_default();
+                            }
+                            Err(e) => {
+                                return Err(io::Error::new(
+                                    io::ErrorKind::InvalidData,
+                                    format!("parse huge pages type: {}, error: {}", value, e),
+                                ));
+                            }
+                        }
+                    }
                     KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => {
                         hv.memory_info.validate_memory_backend_path(value)?;
                         hv.memory_info.file_mem_backend = value.to_string();

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -29,6 +29,7 @@ use std::sync::{Arc, Mutex};
 
 use lazy_static::lazy_static;
 use regex::RegexSet;
+use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str};
 
 use super::{default, ConfigOps, ConfigPlugin, TomlConfig};
 use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX;
@@ -540,6 +541,25 @@ impl MachineInfo {
     }
 }
 
+/// Huge page type for VM RAM backend
+#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq)]
+pub enum HugePageType {
+    /// This will result in the VM memory being allocated using hugetlbfs backend. This is useful
+    /// when you want to use vhost-user network stacks within the container. This will automatically
+    /// result in memory pre allocation.
+    #[serde(rename = "hugetlbfs")]
+    Hugetlbfs,
+    /// This will result in the VM memory being allocated using transparant huge page backend.
+    #[serde(rename = "thp")]
+    THP,
+}
+
+impl Default for HugePageType {
+    fn default() -> Self {
+        Self::Hugetlbfs
+    }
+}
+
 /// Virtual machine memory configuration information.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct MemoryInfo {
@@ -577,12 +597,18 @@ pub struct MemoryInfo {
 
     /// Enable huge pages for VM RAM, default false
     ///
-    /// Enabling this will result in the VM memory being allocated using huge pages. This is useful
+    /// Enabling this will result in the VM memory being allocated using huge pages.
-    /// when you want to use vhost-user network stacks within the container. This will automatically
+    /// Its backend type is specified by item "hugepage_type"
-    /// result in memory pre allocation.
     #[serde(default)]
     pub enable_hugepages: bool,
 
+    /// Select huge page type, default "hugetlbfs"
+    /// Following huge types are supported:
+    /// - hugetlbfs
+    /// - thp
+    #[serde(default)]
+    pub hugepage_type: HugePageType,
+
     /// Specifies virtio-mem will be enabled or not.
     ///
     /// Please note that this option should be used with the command

src/libs/kata-types/tests/test_config.rs

@@ -10,15 +10,15 @@ mod tests {
         KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH,
         KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH,
         KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS,
-        KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS,
+        KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES,
-        KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR,
+        KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP,
-        KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES,
+        KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR,
-        KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH,
+        KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH,
-        KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS,
+        KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC,
-        KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH,
+        KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_PATH,
-        KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS,
+        KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON,
-        KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES,
+        KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM,
-        KATA_ANNO_CFG_RUNTIME_NAME,
+        KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_RUNTIME_NAME,
     };
     use kata_types::config::KataConfig;
     use kata_types::config::{QemuConfig, TomlConfig};
@@ -138,7 +138,7 @@ mod tests {
             "./test_file_backend_mem_root".to_string(),
         );
         anno_hash.insert(
-            KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES.to_string(),
+            KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES.to_string(),
             "false".to_string(),
         );
         anno_hash.insert(

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -7,7 +7,7 @@
 use super::vmm_instance::VmmInstance;
 use crate::{
     device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
-    DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM,
+    DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM,
 };
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
@@ -19,7 +19,10 @@ use dragonball::{
 use kata_sys_util::mount;
 use kata_types::{
     capabilities::{Capabilities, CapabilityBits},
-    config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH},
+    config::{
+        hypervisor::{HugePageType, Hypervisor as HypervisorConfig},
+        KATA_PATH,
+    },
 };
 use nix::mount::MsFlags;
 use persist::sandbox_persist::Persist;
@@ -175,7 +178,10 @@ impl DragonballInner {
     fn set_vm_base_config(&mut self) -> Result<()> {
         let serial_path = [&self.run_dir, "console.sock"].join("/");
         let (mem_type, mem_file_path) = if self.config.memory_info.enable_hugepages {
-            (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES))
+            match self.config.memory_info.hugepage_type {
+                HugePageType::THP => (String::from(HUGE_SHMEM), String::from("")),
+                HugePageType::Hugetlbfs => (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)),
+            }
         } else {
             (String::from(SHMEM), String::from(""))
         };

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -52,6 +52,7 @@ const VM_ROOTFS_FILESYSTEM_EROFS: &str = "erofs";
 const DEV_HUGEPAGES: &str = "/dev/hugepages";
 pub const HUGETLBFS: &str = "hugetlbfs";
 const SHMEM: &str = "shmem";
+const HUGE_SHMEM: &str = "hugeshmem";
 
 pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 pub const HYPERVISOR_QEMU: &str = "qemu";