github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-19 08:28:19 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6706 from guixiongwei/feat/thp

Browse Source 
feat(runtime-rs): introduce huge page mode to select VM RAM's backend
This commit is contained in:
Chao Wu 2023-09-22 15:27:06 +08:00 committed by GitHub
commit 6f98fbafde
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 289 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
src/agent/Cargo.lock generated
View File

@ -507,6 +507,40 @@ dependencies = [
"typenum", "typenum",
] ]
[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "derivative" name = "derivative"
version = "2.2.0" version = "2.2.0"
@ -977,6 +1011,12 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.3" version = "0.2.3"
@ -1184,6 +1224,7 @@ dependencies = [
"regex", "regex",
"safe-path", "safe-path",
"serde", "serde",
"serde-enum-str",
"serde_json", "serde_json",
"slog", "slog",
"slog-scope", "slog-scope",
@ -2279,6 +2320,36 @@ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]]
name = "serde-attributes"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685"
dependencies = [
"darling_core",
"serde-rename-rule",
"syn 1.0.109",
]
[[package]]
name = "serde-enum-str"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa"
dependencies = [
"darling",
"proc-macro2",
"quote",
"serde-attributes",
"syn 1.0.109",
]
[[package]]
name = "serde-rename-rule"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70"
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.137" version = "1.0.137"

1
src/libs/kata-types/Cargo.toml
View File

@ -25,6 +25,7 @@ slog-scope = "4.4.0"
serde_json = "1.0.73" serde_json = "1.0.73"
thiserror = "1.0" thiserror = "1.0"
toml = "0.5.8" toml = "0.5.8"
serde-enum-str = "0.4"
oci = { path = "../oci" } oci = { path = "../oci" }
safe-path = { path = "../safe-path" } safe-path = { path = "../safe-path" }

26
src/libs/kata-types/src/annotations/mod.rs
View File

@ -15,7 +15,7 @@ use serde::Deserialize;
use crate::config::default::DEFAULT_AGENT_TYPE_NAME; use crate::config::default::DEFAULT_AGENT_TYPE_NAME;
use crate::config::default::DEFAULT_HYPERVISOR; use crate::config::default::DEFAULT_HYPERVISOR;
use crate::config::default::DEFAULT_RUNTIME_NAME; use crate::config::default::DEFAULT_RUNTIME_NAME;
use crate::config::hypervisor::get_hypervisor_plugin; use crate::config::hypervisor::{get_hypervisor_plugin, HugePageType};
use crate::config::TomlConfig; use crate::config::TomlConfig;
use crate::sl; use crate::sl;
@ -225,8 +225,11 @@ pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS: &str =
pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str = pub const KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC: &str =
"io.katacontainers.config.hypervisor.enable_mem_prealloc"; "io.katacontainers.config.hypervisor.enable_mem_prealloc";
/// A sandbox annotation to specify if the memory should be pre-allocated from huge pages. /// A sandbox annotation to specify if the memory should be pre-allocated from huge pages.
pub const KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES: &str = pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES: &str =
"io.katacontainers.config.hypervisor.enable_hugepages"; "io.katacontainers.config.hypervisor.enable_hugepages";
/// A sandbox annotation to specify huge page mode of memory backend.
pub const KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE: &str =
"io.katacontainers.config.hypervisor.hugepage_type";
/// A sandbox annotation to soecify file based memory backend root directory. /// A sandbox annotation to soecify file based memory backend root directory.
pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str = pub const KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR: &str =
"io.katacontainers.config.hypervisor.file_mem_backend"; "io.katacontainers.config.hypervisor.file_mem_backend";
@ -740,14 +743,29 @@ impl Annotation {
return Err(bool_err); return Err(bool_err);
} }
}, },
KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES => match self.get_value::<bool>(key) { KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES => {
match self.get_value::<bool>(key) {
Ok(r) => { Ok(r) => {
hv.memory_info.enable_hugepages = r.unwrap_or_default(); hv.memory_info.enable_hugepages = r.unwrap_or_default();
} }
Err(_e) => { Err(_e) => {
return Err(bool_err); return Err(bool_err);
} }
}, }
}
KATA_ANNO_CFG_HYPERVISOR_HUGEPAGE_TYPE => {
match self.get_value::<HugePageType>(key) {
Ok(r) => {
hv.memory_info.hugepage_type = r.unwrap_or_default();
}
Err(e) => {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("parse huge pages type: {}, error: {}", value, e),
));
}
}
}
KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => { KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR => {
hv.memory_info.validate_memory_backend_path(value)?; hv.memory_info.validate_memory_backend_path(value)?;
hv.memory_info.file_mem_backend = value.to_string(); hv.memory_info.file_mem_backend = value.to_string();

32
src/libs/kata-types/src/config/hypervisor/mod.rs
View File

@ -29,6 +29,7 @@ use std::sync::{Arc, Mutex};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use regex::RegexSet; use regex::RegexSet;
use serde_enum_str::{Deserialize_enum_str, Serialize_enum_str};
use super::{default, ConfigOps, ConfigPlugin, TomlConfig}; use super::{default, ConfigOps, ConfigPlugin, TomlConfig};
use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX; use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX;
@ -540,6 +541,25 @@ impl MachineInfo {
} }
} }
/// Huge page type for VM RAM backend
#[derive(Clone, Debug, Deserialize_enum_str, Serialize_enum_str, PartialEq, Eq)]
pub enum HugePageType {
/// This will result in the VM memory being allocated using hugetlbfs backend. This is useful
/// when you want to use vhost-user network stacks within the container. This will automatically
/// result in memory pre allocation.
#[serde(rename = "hugetlbfs")]
Hugetlbfs,
/// This will result in the VM memory being allocated using transparant huge page backend.
#[serde(rename = "thp")]
THP,
}
impl Default for HugePageType {
fn default() -> Self {
Self::Hugetlbfs
}
}
/// Virtual machine memory configuration information. /// Virtual machine memory configuration information.
#[derive(Clone, Debug, Default, Deserialize, Serialize)] #[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct MemoryInfo { pub struct MemoryInfo {
@ -577,12 +597,18 @@ pub struct MemoryInfo {
/// Enable huge pages for VM RAM, default false /// Enable huge pages for VM RAM, default false
/// ///
/// Enabling this will result in the VM memory being allocated using huge pages. This is useful /// Enabling this will result in the VM memory being allocated using huge pages.
/// when you want to use vhost-user network stacks within the container. This will automatically /// Its backend type is specified by item "hugepage_type"
/// result in memory pre allocation.
#[serde(default)] #[serde(default)]
pub enable_hugepages: bool, pub enable_hugepages: bool,
/// Select huge page type, default "hugetlbfs"
/// Following huge types are supported:
/// - hugetlbfs
/// - thp
#[serde(default)]
pub hugepage_type: HugePageType,
/// Specifies virtio-mem will be enabled or not. /// Specifies virtio-mem will be enabled or not.
/// ///
/// Please note that this option should be used with the command /// Please note that this option should be used with the command

20
src/libs/kata-types/tests/test_config.rs
View File

@ -10,15 +10,15 @@ mod tests {
KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH, KATA_ANNO_CFG_EXPERIMENTAL, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_CACHE_NOFLUSH,
KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH, KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER, KATA_ANNO_CFG_HYPERVISOR_CTLPATH,
KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_GUEST_SWAP, KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES,
KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR, KATA_ANNO_CFG_HYPERVISOR_ENABLE_IO_THREADS, KATA_ANNO_CFG_HYPERVISOR_ENABLE_SWAP,
KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES, KATA_ANNO_CFG_HYPERVISOR_FILE_BACKED_MEM_ROOT_DIR,
KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH, KATA_ANNO_CFG_HYPERVISOR_JAILER_PATH,
KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_KERNEL_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_PREALLOC,
KATA_ANNO_CFG_HYPERVISOR_PATH, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_MEMORY_SLOTS, KATA_ANNO_CFG_HYPERVISOR_PATH,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VHOSTUSER_STORE_PATH, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_DAEMON,
KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM, KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS, KATA_ANNO_CFG_HYPERVISOR_VIRTIO_MEM,
KATA_ANNO_CFG_RUNTIME_NAME, KATA_ANNO_CFG_KERNEL_MODULES, KATA_ANNO_CFG_RUNTIME_NAME,
}; };
use kata_types::config::KataConfig; use kata_types::config::KataConfig;
use kata_types::config::{QemuConfig, TomlConfig}; use kata_types::config::{QemuConfig, TomlConfig};
@ -138,7 +138,7 @@ mod tests {
"./test_file_backend_mem_root".to_string(), "./test_file_backend_mem_root".to_string(),
); );
anno_hash.insert( anno_hash.insert(
KATA_ANNO_CFG_HYPERVISOR_HUGE_PAGES.to_string(), KATA_ANNO_CFG_HYPERVISOR_ENABLE_HUGEPAGES.to_string(),
"false".to_string(), "false".to_string(),
); );
anno_hash.insert( anno_hash.insert(

71
src/runtime-rs/Cargo.lock generated
View File

@ -602,6 +602,40 @@ dependencies = [
"typenum", "typenum",
] ]
[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "5.5.0" version = "5.5.0"
@ -1415,6 +1449,12 @@ dependencies = [
"cc", "cc",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.4.0" version = "0.4.0"
@ -1552,6 +1592,7 @@ dependencies = [
"regex", "regex",
"safe-path 0.1.0", "safe-path 0.1.0",
"serde", "serde",
"serde-enum-str",
"serde_json", "serde_json",
"slog", "slog",
"slog-scope", "slog-scope",
@ -2948,6 +2989,36 @@ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]]
name = "serde-attributes"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685"
dependencies = [
"darling_core",
"serde-rename-rule",
"syn 1.0.109",
]
[[package]]
name = "serde-enum-str"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa"
dependencies = [
"darling",
"proc-macro2",
"quote",
"serde-attributes",
"syn 1.0.109",
]
[[package]]
name = "serde-rename-rule"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70"
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.177" version = "1.0.177"

12
src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs
View File

@ -7,7 +7,7 @@
use super::vmm_instance::VmmInstance; use super::vmm_instance::VmmInstance;
use crate::{ use crate::{
device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState, device::DeviceType, hypervisor_persist::HypervisorState, kernel_param::KernelParams, VmmState,
DEV_HUGEPAGES, HUGETLBFS, HYPERVISOR_DRAGONBALL, SHMEM, DEV_HUGEPAGES, HUGETLBFS, HUGE_SHMEM, HYPERVISOR_DRAGONBALL, SHMEM,
}; };
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, Context, Result};
use async_trait::async_trait; use async_trait::async_trait;
@ -19,7 +19,10 @@ use dragonball::{
use kata_sys_util::mount; use kata_sys_util::mount;
use kata_types::{ use kata_types::{
capabilities::{Capabilities, CapabilityBits}, capabilities::{Capabilities, CapabilityBits},
config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH}, config::{
hypervisor::{HugePageType, Hypervisor as HypervisorConfig},
KATA_PATH,
},
}; };
use nix::mount::MsFlags; use nix::mount::MsFlags;
use persist::sandbox_persist::Persist; use persist::sandbox_persist::Persist;
@ -175,7 +178,10 @@ impl DragonballInner {
fn set_vm_base_config(&mut self) -> Result<()> { fn set_vm_base_config(&mut self) -> Result<()> {
let serial_path = [&self.run_dir, "console.sock"].join("/"); let serial_path = [&self.run_dir, "console.sock"].join("/");
let (mem_type, mem_file_path) = if self.config.memory_info.enable_hugepages { let (mem_type, mem_file_path) = if self.config.memory_info.enable_hugepages {
(String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)) match self.config.memory_info.hugepage_type {
HugePageType::THP => (String::from(HUGE_SHMEM), String::from("")),
HugePageType::Hugetlbfs => (String::from(HUGETLBFS), String::from(DEV_HUGEPAGES)),
}
} else { } else {
(String::from(SHMEM), String::from("")) (String::from(SHMEM), String::from(""))
}; };

1
src/runtime-rs/crates/hypervisor/src/lib.rs
View File

@ -52,6 +52,7 @@ const VM_ROOTFS_FILESYSTEM_EROFS: &str = "erofs";
const DEV_HUGEPAGES: &str = "/dev/hugepages"; const DEV_HUGEPAGES: &str = "/dev/hugepages";
pub const HUGETLBFS: &str = "hugetlbfs"; pub const HUGETLBFS: &str = "hugetlbfs";
const SHMEM: &str = "shmem"; const SHMEM: &str = "shmem";
const HUGE_SHMEM: &str = "hugeshmem";
pub const HYPERVISOR_DRAGONBALL: &str = "dragonball"; pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
pub const HYPERVISOR_QEMU: &str = "qemu"; pub const HYPERVISOR_QEMU: &str = "qemu";

71
src/tools/kata-ctl/Cargo.lock generated
View File

@ -374,6 +374,40 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "derive-new" name = "derive-new"
version = "0.5.9" version = "0.5.9"
@ -809,6 +843,12 @@ dependencies = [
"cc", "cc",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.4.0" version = "0.4.0"
@ -981,6 +1021,7 @@ dependencies = [
"regex", "regex",
"safe-path", "safe-path",
"serde", "serde",
"serde-enum-str",
"serde_json", "serde_json",
"slog", "slog",
"slog-scope", "slog-scope",
@ -1883,6 +1924,36 @@ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]]
name = "serde-attributes"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6eb8ec7724e4e524b2492b510e66957fe1a2c76c26a6975ec80823f2439da685"
dependencies = [
"darling_core",
"serde-rename-rule",
"syn 1.0.109",
]
[[package]]
name = "serde-enum-str"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26416dc95fcd46b0e4b12a3758043a229a6914050aaec2e8191949753ed4e9aa"
dependencies = [
"darling",
"proc-macro2",
"quote",
"serde-attributes",
"syn 1.0.109",
]
[[package]]
name = "serde-rename-rule"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "794e44574226fc701e3be5c651feb7939038fc67fb73f6f4dd5c4ba90fd3be70"
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.177" version = "1.0.177"