github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-26 07:22:20 +00:00
Code Issues Projects Releases Wiki Activity

feat(runtime-rs): introduce huge page type to select VM RAM's backend

Browse Source 
This commit allows us to specify the huge page backend when enabling huge
page. Currently, we support two backends: thp and hugetlbfs, the default
is hugetlbfs.

To ensure backward compatibility, we introduce another configuration item
"hugepage_type" to select the memory backend, which is available only when
"enable_hugepages" is true. Besides, we add an annotation
"io.katacontainers.config.hypervisor.hugepage_type" to configure huge page
type per pod.

Fixes: #6703

Signed-off-by: Guixiong Wei <weiguixiong@bytedance.com>
Signed-off-by: Yipeng Yin <yinyipeng@bytedance.com>
This commit is contained in:
Guixiong Wei 2023-04-20 17:38:44 +08:00 committed by Yipeng Yin
parent e1f54f96d0
commit 202049f35e
6 changed files with 76 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/libs/kata-types/Cargo.toml
View File

@ -25,6 +25,7 @@ slog-scope = "4.4.0"
serde_json = "1.0.73"
thiserror = "1.0"
toml = "0.5.8"
serde-enum-str = "0.4"
oci = { path = "../oci" }
safe-path = { path = "../safe-path" }

34
src/libs/kata-types/src/annotations/mod.rs
View File

@ -15,7 +15,7 @@ use serde::Deserialize;
use crate::config::default::DEFAULT_AGENT_TYPE_NAME;
use crate::config::default::DEFAULT_HYPERVISOR;
use crate::config::default::DEFAULT_RUNTIME_NAME;
use crate::config::hypervisor::get_hypervisor_plugin;
use crate::config::hypervisor::{get_hypervisor_plugin, HugePageType};
use crate::config::TomlConfig;
use crate::sl;
@ -225,8 +225,11 @@ pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS: &str =
pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str =
"io.katacontainers.config.hypervisor.enable_mem_prealloc";
/// A sandbox annotation to specify if the memory should be pre-allocated from huge pages.
pub const KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES: &str =
pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES: &str =
"io.katacontainers.config.hypervisor.enable_hugepages";
/// A sandbox annotation to specify huge page mode of memory backend.
pub const KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE: &str =
"io.katacontainers.config.hypervisor.hugepage_type";
/// A sandbox annotation to soecify file based memory backend root directory.
pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str =
"io.katacontainers.config.hypervisor.file_mem_backend";
@ -740,14 +743,29 @@ impl Annotation {
return Err(bool_err);
}
},
KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_value::<bool>(key) {
Ok(r) => {
hv.memory_info.enable_hugepages = r.unwrap_or_default();
KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES => {
match self.get_value::<bool>(key) {
Ok(r) => {
hv.memory_info.enable_hugepages = r.unwrap_or_default();
}
Err(_e) => {
return Err(bool_err);
}
}
Err(_e) => {
return Err(bool_err);
}
KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE => {
match self.get_value::<HugePageType>(key) {
Ok(r) => {
hv.memory_info.hugepage_type = r.unwrap_or_default();
}
Err(e) => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("parse huge pages type: {}, error: {}", value, e),
));
}
}
},
}
KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => {
hv.memory_info.validate_memory_backend_path(value)?;
hv.memory_info.file_mem_backend = value.to_string();

32
src/libs/kata-types/src/config/hypervisor/mod.rs
View File

@ -29,6 +29,7 @@ use std::sync::{Arc, Mutex};
use lazy_static::lazy_static;
use regex::RegexSet;
use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str};
use super::{default, ConfigOps, ConfigPlugin, TomlConfig};
use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX;
@ -540,6 +541,25 @@ impl MachineInfo {
}
}
/// Huge page type for VM RAM backend
#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq)]
pub enum HugePageType {
/// This will result in the VM memory being allocated using hugetlbfs backend. This is useful
/// when you want to use vhost-user network stacks within the container. This will automatically
/// result in memory pre allocation.
#[serde(rename = "hugetlbfs")]
Hugetlbfs,
/// This will result in the VM memory being allocated using transparant huge page backend.
#[serde(rename = "thp")]
THP,
}
impl Default for HugePageType {
fn default() -> Self {
Self::Hugetlbfs
}
}
/// Virtual machine memory configuration information.
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct MemoryInfo {
@ -577,12 +597,18 @@ pub struct MemoryInfo {
/// Enable huge pages for VM RAM, default false
///
/// Enabling this will result in the VM memory being allocated using huge pages. This is useful
/// when you want to use vhost-user network stacks within the container. This will automatically
/// result in memory pre allocation.
/// Enabling this will result in the VM memory being allocated using huge pages.
/// Its backend type is specified by item "hugepage_type"
#[serde(default)]
pub enable_hugepages: bool,
/// Select huge page type, default "hugetlbfs"
/// Following huge types are supported:
/// - hugetlbfs
/// - thp
#[serde(default)]
pub hugepage_type: HugePageType,
/// Specifies virtio-mem will be enabled or not.
///
/// Please note that this option should be used with the command

20
src/libs/kata-types/tests/test_config.rs
View File

@ -10,15 +10,15 @@ mod tests {
KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH,
KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH,
KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR,
KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES,
KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH,
KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS,
KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES,
KATA_ANNO_CFG_RUNTIME_NAME,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP,
KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR,
KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH,
KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC,
KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_PATH,
KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM,
KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_RUNTIME_NAME,
};
use kata_types::config::KataConfig;
use kata_types::config::{QemuConfig, TomlConfig};
@ -138,7 +138,7 @@ mod tests {
"./test_file_backend_mem_root".to_string(),
);
anno_hash.insert(
KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES.to_string(),
KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES.to_string(),
"false".to_string(),
);
anno_hash.insert(

12
src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
View File

@ -7,7 +7,7 @@
use super::vmm_instance::VmmInstance;
use crate::{
device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM,
DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM,
};
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
@ -19,7 +19,10 @@ use dragonball::{
use kata_sys_util::mount;
use kata_types::{
capabilities::{Capabilities, CapabilityBits},
config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH},
config::{
hypervisor::{HugePageType, Hypervisor as HypervisorConfig},
KATA_PATH,
},
};
use nix::mount::MsFlags;
use persist::sandbox_persist::Persist;
@ -175,7 +178,10 @@ impl DragonballInner {
fn set_vm_base_config(&mut self) -> Result<()> {
let serial_path = [&self.run_dir, "console.sock"].join("/");
let (mem_type, mem_file_path) = if self.config.memory_info.enable_hugepages {
(String::from(HUGETLBFS), String::from(DEV_HUGEPAGES))
match self.config.memory_info.hugepage_type {
HugePageType::THP => (String::from(HUGE_SHMEM), String::from("")),
HugePageType::Hugetlbfs => (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)),
}
} else {
(String::from(SHMEM), String::from(""))
};

1
src/runtime-rs/crates/hypervisor/src/lib.rs
View File

@ -52,6 +52,7 @@ const VM_ROOTFS_FILESYSTEM_EROFS: &str = "erofs";
const DEV_HUGEPAGES: &str = "/dev/hugepages";
pub const HUGETLBFS: &str = "hugetlbfs";
const SHMEM: &str = "shmem";
const HUGE_SHMEM: &str = "hugeshmem";
pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
pub const HYPERVISOR_QEMU: &str = "qemu";