Merge pull request #6598 from Apokleos/sandbox_bind_mounts

runtime-rs/sandbox_bindmounts: add support for sandbox bindmounts

src/runtime-rs/config/configuration-dragonball.toml.in

@@ -309,3 +309,12 @@ experimental=@DEFAULTEXPFEATURES@
 # - When running single containers using a tool like ctr, container sizing information will be available.
 static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@
 
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted(ro, rw) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+# Now it supports three kinds of bind mount format:
+# - "/path/to", default readonly mode.
+# - "/path/to:ro", readonly mode.
+# - "/path/to:rw", readwrite mode.
+sandbox_bind_mounts=@DEFBINDMOUNTS@

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -25,7 +25,7 @@ use crate::{
     manager::ManagerArgs,
     network::{self, Network},
     rootfs::{RootFsResource, Rootfs},
-    share_fs::{self, ShareFs},
+    share_fs::{self, sandbox_bind_mounts::SandboxBindMounts, ShareFs},
     volume::{Volume, VolumeResource},
     ResourceConfig,
 };
@@ -97,6 +97,12 @@ impl ResourceManagerInner {
                             .setup_device_before_start_vm(self.hypervisor.as_ref())
                             .await
                             .context("setup share fs device before start vm")?;
+
+                        // setup sandbox bind mounts: setup = true
+                        self.handle_sandbox_bindmounts(true)
+                            .await
+                            .context("failed setup sandbox bindmounts")?;
+
                         Some(share_fs)
                     } else {
                         None
@@ -308,6 +314,22 @@ impl ResourceManagerInner {
         Ok(devices)
     }
 
+    async fn handle_sandbox_bindmounts(&self, setup: bool) -> Result<()> {
+        let bindmounts = self.toml_config.runtime.sandbox_bind_mounts.clone();
+        if bindmounts.is_empty() {
+            info!(sl!(), "sandbox bindmounts empty, just skip it.");
+            return Ok(());
+        }
+
+        let sb_bindmnt = SandboxBindMounts::new(self.sid.clone(), bindmounts)?;
+
+        if setup {
+            sb_bindmnt.setup_sandbox_bind_mounts()
+        } else {
+            sb_bindmnt.cleanup_sandbox_bind_mounts()
+        }
+    }
+
     pub async fn update_cgroups(
         &self,
         cid: &str,
@@ -324,6 +346,12 @@ impl ResourceManagerInner {
             .delete()
             .await
             .context("delete cgroup")?;
+
+        // cleanup sandbox bind mounts: setup = false
+        self.handle_sandbox_bindmounts(false)
+            .await
+            .context("failed to cleanup sandbox bindmounts")?;
+
         // clean up share fs mount
         if let Some(share_fs) = &self.share_fs {
             share_fs

src/runtime-rs/crates/resource/src/share_fs/mod.rs

@@ -18,6 +18,7 @@ pub use utils::{
 mod virtio_fs_share_mount;
 use virtio_fs_share_mount::VirtiofsShareMount;
 pub use virtio_fs_share_mount::EPHEMERAL_PATH;
+pub mod sandbox_bind_mounts;
 
 use std::{collections::HashMap, fmt::Debug, path::PathBuf, sync::Arc};
 

src/runtime-rs/crates/resource/src/share_fs/sandbox_bind_mounts.rs

@@ -0,0 +1,155 @@
+// Copyright (c) 2023 Alibaba Cloud
+// Copyright (c) 2023 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Note:
+// sandbox_bind_mounts supports kinds of mount patterns, for example:
+// (1) "/path/to", with default readonly mode.
+// (2) "/path/to:ro", same as (1).
+// (3) "/path/to:rw", with readwrite mode.
+//
+// sandbox_bind_mounts: ["/path/to", "/path/to:rw", "/mnt/to:ro"]
+//
+
+use std::{
+    collections::HashMap,
+    fs,
+    path::{Path, PathBuf},
+};
+
+use anyhow::{anyhow, Context, Result};
+
+use super::utils::{do_get_host_path, mkdir_with_permissions};
+use kata_sys_util::{fs::get_base_name, mount};
+use kata_types::mount::{SANDBOX_BIND_MOUNTS_DIR, SANDBOX_BIND_MOUNTS_RO, SANDBOX_BIND_MOUNTS_RW};
+
+#[derive(Clone, Default, Debug)]
+pub struct SandboxBindMounts {
+    sid: String,
+    host_mounts_path: PathBuf,
+    sandbox_bindmounts: Vec<String>,
+}
+
+impl SandboxBindMounts {
+    pub fn new(sid: String, sandbox_bindmounts: Vec<String>) -> Result<Self> {
+        // /run/kata-containers/shared/sandboxes/<sid>/rw/passthrough/sandbox-mounts
+        let bindmounts_path =
+            do_get_host_path(SANDBOX_BIND_MOUNTS_DIR, sid.as_str(), "", true, false);
+        let host_mounts_path = PathBuf::from(bindmounts_path);
+
+        Ok(SandboxBindMounts {
+            sid,
+            host_mounts_path,
+            sandbox_bindmounts,
+        })
+    }
+
+    fn parse_sandbox_bind_mounts<'a>(&self, bindmnt_src: &'a str) -> Result<(&'a str, &'a str)> {
+        // get the bindmount's r/w mode
+        let bindmount_mode = if bindmnt_src.ends_with(SANDBOX_BIND_MOUNTS_RW) {
+            SANDBOX_BIND_MOUNTS_RW
+        } else {
+            SANDBOX_BIND_MOUNTS_RO
+        };
+
+        // get the true bindmount from the string
+        let bindmount = bindmnt_src.trim_end_matches(bindmount_mode);
+
+        Ok((bindmount_mode, bindmount))
+    }
+
+    pub fn setup_sandbox_bind_mounts(&self) -> Result<()> {
+        let mut mounted_list: Vec<PathBuf> = Vec::new();
+        let mut mounted_map: HashMap<String, bool> = HashMap::new();
+        for src in &self.sandbox_bindmounts {
+            let (bindmount_mode, bindmount) = self
+                .parse_sandbox_bind_mounts(src)
+                .context("parse sandbox bind mounts failed")?;
+
+            // get the basename of the canonicalized mount path mnt_name: dirX
+            let mnt_name = get_base_name(bindmount)?
+                .into_string()
+                .map_err(|e| anyhow!("failed to get base name {:?}", e))?;
+
+            // if repeated mounted, do umount it and return error
+            if mounted_map.insert(mnt_name.clone(), true).is_some() {
+                for p in &mounted_list {
+                    nix::mount::umount(p)
+                        .context("mounted_map insert one repeated mounted, do umount it")?;
+                }
+
+                return Err(anyhow!(
+                    "sandbox-bindmounts: path {} is already specified.",
+                    bindmount
+                ));
+            }
+
+            // mount_dest: /run/kata-containers/shared/sandboxes/<sid>/rw/passthrough/sandbox-mounts/dirX
+            let mount_dest = self.host_mounts_path.clone().join(mnt_name.as_str());
+            mkdir_with_permissions(self.host_mounts_path.clone().to_path_buf(), 0o750).context(
+                format!(
+                    "create host mounts path {:?}",
+                    self.host_mounts_path.clone()
+                ),
+            )?;
+
+            info!(
+                sl!(),
+                "sandbox-bindmounts mount_src: {:?} => mount_dest: {:?}", bindmount, &mount_dest
+            );
+
+            // mount -o bind,ro host_shared mount_dest
+            // host_shared: ${bindmount}
+            mount::bind_mount_unchecked(Path::new(bindmount), &mount_dest, true).map_err(|e| {
+                for p in &mounted_list {
+                    nix::mount::umount(p).unwrap_or_else(|x| {
+                        format!("do umount failed: {:?}", x);
+                    });
+                }
+                e
+            })?;
+
+            // default sandbox bind mounts mode is ro.
+            if bindmount_mode == SANDBOX_BIND_MOUNTS_RO {
+                info!(sl!(), "sandbox readonly bind mount.");
+                // dest_ro: /run/kata-containers/shared/sandboxes/<sid>/ro/passthrough/sandbox-mounts
+                let mount_dest_ro =
+                    do_get_host_path(SANDBOX_BIND_MOUNTS_DIR, &self.sid, "", true, true);
+                let sandbox_bindmounts_ro = [mount_dest_ro, mnt_name.clone()].join("/");
+
+                mount::bind_remount(sandbox_bindmounts_ro, true)
+                    .context("remount ro directory with ro permission")?;
+            }
+
+            mounted_list.push(mount_dest);
+        }
+
+        Ok(())
+    }
+
+    pub fn cleanup_sandbox_bind_mounts(&self) -> Result<()> {
+        for src in &self.sandbox_bindmounts {
+            let parsed_mnts = self
+                .parse_sandbox_bind_mounts(src)
+                .context("parse sandbox bind mounts")?;
+
+            let mnt_name = get_base_name(parsed_mnts.1)?
+                .into_string()
+                .map_err(|e| anyhow!("failed to convert to string{:?}", e))?;
+
+            // /run/kata-containers/shared/sandboxes/<sid>/passthrough/rw/sandbox-mounts/dir
+            let mnt_dest = self.host_mounts_path.join(mnt_name.as_str());
+            mount::umount_timeout(mnt_dest, 0).context("umount bindmount failed")?;
+        }
+
+        if fs::metadata(self.host_mounts_path.clone())?.is_dir() {
+            fs::remove_dir_all(self.host_mounts_path.clone()).context(format!(
+                "remove sandbox bindmount point {:?}.",
+                self.host_mounts_path.clone()
+            ))?;
+        }
+
+        Ok(())
+    }
+}

src/runtime-rs/crates/resource/src/share_fs/utils.rs

@@ -4,13 +4,27 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::path::{Path, PathBuf};
+use std::{
+    os::unix::fs::PermissionsExt,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Result;
 use kata_sys_util::mount;
 
 use super::*;
 
+pub(crate) fn mkdir_with_permissions(path_target: PathBuf, mode: u32) -> Result<()> {
+    let new_path = &path_target;
+    std::fs::create_dir_all(new_path)
+        .context(format!("unable to create new path: {:?}", new_path))?;
+
+    // mode format: 0o750, ...
+    std::fs::set_permissions(new_path, std::fs::Permissions::from_mode(mode))?;
+
+    Ok(())
+}
+
 pub(crate) fn ensure_dir_exist(path: &Path) -> Result<()> {
     if !path.exists() {
         std::fs::create_dir_all(path).context(format!("failed to create directory {:?}", path))?;

src/runtime-rs/crates/resource/src/share_fs/virtio_fs_share_mount.rs

@@ -12,7 +12,6 @@ use kata_types::k8s::is_watchable_mount;
 use kata_types::mount;
 use nix::sys::stat::stat;
 use std::fs;
-use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
 
 const WATCHABLE_PATH_NAME: &str = "watchable";
@@ -21,7 +20,10 @@ pub const EPHEMERAL_PATH: &str = "/run/kata-containers/sandbox/ephemeral";
 
 use super::{
     get_host_rw_shared_path,
-    utils::{self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path},
+    utils::{
+        self, do_get_host_path, get_host_ro_shared_path, get_host_shared_path,
+        mkdir_with_permissions,
+    },
     ShareFsMount, ShareFsMountResult, ShareFsRootfsConfig, ShareFsVolumeConfig,
     KATA_GUEST_SHARE_DIR, PASSTHROUGH_FS_DIR,
 };
@@ -79,13 +81,11 @@ impl ShareFsMount for VirtiofsShareMount {
                 .join(PASSTHROUGH_FS_DIR)
                 .join(WATCHABLE_PATH_NAME);
 
-            fs::create_dir_all(&watchable_host_path).context(format!(
-                "unable to create watchable path: {:?}",
-                &watchable_host_path,
+            mkdir_with_permissions(watchable_host_path.clone(), 0o750).context(format!(
+                "unable to create watchable path {:?}",
+                watchable_host_path
             ))?;
 
-            fs::set_permissions(watchable_host_path, fs::Permissions::from_mode(0o750))?;
-
             // path: /run/kata-containers/shared/containers/passthrough/watchable/config-map-name
             let file_name = Path::new(&guest_path)
                 .file_name()