From 608e003abcf7ce54c12512d79b5196b74c1883db Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 24 Feb 2022 12:41:33 +1100 Subject: [PATCH 1/2] agent: Don't attempt to create directories for hugepage configuration allocate_hugepages() constructs the path for the sysfs directory containing hugepage configuration, then attempts to create this directory if it does not exist. This doesn't make sense: sysfs is a view into kernel configuration, if the kernel has support for the hugepage size, the directory will already be there, if it doesn't, trying to create it won't help. For the same reason, attempting to create the "nr_hugepages" file itself is pointless, so there's no reason to call OpenOptions::create(true). Signed-off-by: David Gibson --- src/agent/src/mount.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index d7dbc08ef4..17ae85bda7 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -335,20 +335,16 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> { // sysfs entry is always of the form hugepages-${pagesize}kB // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt - let path = Path::new(SYS_FS_HUGEPAGES_PREFIX).join(format!("hugepages-{}kB", pagesize / 1024)); - - if !path.exists() { - fs::create_dir_all(&path).context("create hugepages-size directory")?; - } + let path = Path::new(SYS_FS_HUGEPAGES_PREFIX) + .join(format!("hugepages-{}kB", pagesize / 1024)) + .join("nr_hugepages"); // write numpages to nr_hugepages file. - let path = path.join("nr_hugepages"); let numpages = format!("{}", size / pagesize); info!(logger, "write {} pages to {:?}", &numpages, &path); let mut file = OpenOptions::new() .write(true) - .create(true) .open(&path) .context(format!("open nr_hugepages directory {:?}", &path))?; From 42e35505b0fb1f207ca041a4d22de05d41106be8 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 24 Feb 2022 12:53:49 +1100 Subject: [PATCH 2/2] agent: Verify that we allocated as many hugepages as we need allocate_hugepages() writes to the kernel sysfs file to allocate hugepages in the Kata VM. However, even if the write succeeds, it's not certain that the kernel will actually be able to allocate as many hugepages as we requested. This patch reads back the file after writing it to check if we were able to allocate all the required hugepages. fixes #3816 Signed-off-by: David Gibson --- src/agent/src/mount.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 17ae85bda7..41671e897f 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -351,6 +351,23 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> { file.write_all(numpages.as_bytes()) .context(format!("write nr_hugepages failed: {:?}", &path))?; + // Even if the write succeeds, the kernel isn't guaranteed to be + // able to allocate all the pages we requested. Verify that it + // did. + let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?; + let allocated = verify + .trim_end() + .parse::() + .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?; + if allocated != size / pagesize { + return Err(anyhow!( + "Only allocated {} of {} hugepages of size {}", + allocated, + numpages, + pagesize + )); + } + Ok(()) }