runtime-rs: supporting the CLH VMM process running in non-root mode

This change enables to run the Cloud Hypervisor VMM using a non-root user
when rootless flag is set true in configuration.

Fixes: #11414

Signed-off-by: stevenfryto <sunzitai_1832@bupt.edu.cn>

src/runtime-rs/config/configuration-cloud-hypervisor.toml.in

@@ -225,6 +225,11 @@ block_device_driver = "virtio-blk-pci"
 # result in memory pre allocation
 #enable_hugepages = true
 
+# Enable running clh VMM as a non-root user.
+# By default clh VMM run as root. When this is set to true, clh VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+#rootless = true
+
 # Disable the 'seccomp' feature from Cloud Hypervisor, firecracker or dragonball, default false
 # disable_seccomp = true
 

src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs

@@ -5,8 +5,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::inner::CloudHypervisorInner;
+use crate::ch::utils::get_rootless_symlink_sandbox_jailer_root;
 use crate::device::pci_path::PciPath;
 use crate::device::DeviceType;
+use crate::utils::create_dir_all_with_inherit_owner;
 use crate::utils::open_named_tuntap;
 use crate::HybridVsockDevice;
 use crate::NetworkConfig;
@@ -28,10 +30,12 @@ use ch_config::DiskConfig;
 use ch_config::{net_util::MacAddr, DeviceConfig, FsConfig, NetConfig, VsockConfig};
 use kata_sys_util::netns::NetnsGuard;
 use kata_types::config::hypervisor::RateLimiterConfig;
+use kata_types::rootless::is_rootless;
 use safe_path::scoped_join;
 use std::convert::TryFrom;
 use std::os::fd::AsRawFd;
 use std::os::fd::IntoRawFd;
+use std::os::unix::fs::symlink;
 use std::path::PathBuf;
 
 const VIRTIO_FS: &str = "virtio-fs";
@@ -409,7 +413,49 @@ impl CloudHypervisorInner {
                 DeviceType::ShareFs(dev) => {
                     let settings = ShareFsSettings::new(dev.config, self.vm_path.clone());
 
-                    let fs_cfg = FsConfig::try_from(settings)?;
+                    let fs_cfg = if is_rootless() {
+                        // TODO: Replace this symlink workaround if a better approach for rootless socket paths appears.
+                        // In rootless mode the virtiofsd.sock lives under the rootless directory,
+                        // and its full path can exceed the 108-byte Unix domain socket limit.
+                        // To ensure the cloud-hypervisor VMM can connect to virtiofsd, create a
+                        // short symlink inside the rootless directory and point the VMM at it.
+                        let mut fs_cfg = FsConfig::try_from(settings)?;
+                        let rootless_symlink_sanbox_jailer_root =
+                            get_rootless_symlink_sandbox_jailer_root(self.id.as_str());
+
+                        create_dir_all_with_inherit_owner(
+                            rootless_symlink_sanbox_jailer_root.as_str(),
+                            0x750,
+                        )
+                        .map_err(|e| {
+                            anyhow!(
+                                "failed to create rootless sharefs symlink jailer root dir: {}",
+                                e
+                            )
+                        })?;
+                        let virtiofsd_name = fs_cfg.socket.file_name().ok_or_else(|| {
+                            anyhow!(
+                                "failed to get virtiofsd socket file name from path: {:?}",
+                                fs_cfg.socket
+                            )
+                        })?;
+                        let virtiofsd_symlink_path =
+                            PathBuf::from(rootless_symlink_sanbox_jailer_root.as_str())
+                                .join(virtiofsd_name);
+
+                        symlink(&fs_cfg.socket, &virtiofsd_symlink_path).map_err(|e| {
+                            anyhow!(
+                                "failed to create symlink for rootless sharefs socket: {}",
+                                e
+                            )
+                        })?;
+
+                        fs_cfg.socket = virtiofsd_symlink_path;
+
+                        fs_cfg
+                    } else {
+                        FsConfig::try_from(settings)?
+                    };
 
                     shared_fs_devices.push(fs_cfg);
                 }

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -5,9 +5,13 @@
 
 use super::inner::CloudHypervisorInner;
 use crate::ch::utils::get_api_socket_path;
+use crate::ch::utils::get_rootless_symlink_sandbox_path;
 use crate::ch::utils::get_vsock_path;
 use crate::kernel_param::KernelParams;
 use crate::selinux;
+use crate::utils::create_dir_all_with_inherit_owner;
+use crate::utils::set_groups;
+use crate::utils::vm_cleanup;
 use crate::utils::{bytes_to_megs, get_jailer_root, get_sandbox_path, megs_to_bytes};
 use crate::MemoryConfig;
 use crate::VM_ROOTFS_DRIVER_BLK;
@@ -28,14 +32,20 @@ use futures::future::join_all;
 use kata_sys_util::protection::{available_guest_protection, GuestProtection};
 use kata_types::capabilities::{Capabilities, CapabilityBits};
 use kata_types::config::default::DEFAULT_CH_ROOTFS_TYPE;
+use kata_types::config::hypervisor::RootlessUser;
+use kata_types::rootless::is_rootless;
 use lazy_static::lazy_static;
 use nix::sched::{setns, CloneFlags};
+use nix::unistd::setgid;
+use nix::unistd::setuid;
+use nix::unistd::Gid;
+use nix::unistd::Uid;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::HashMap;
 use std::convert::TryFrom;
 use std::fs;
-use std::fs::create_dir_all;
+use std::fs::remove_dir_all;
 use std::os::unix::io::AsRawFd;
 use std::os::unix::net::UnixStream;
 use std::path::Path;
@@ -196,7 +206,8 @@ impl CloudHypervisorInner {
 
         let sandbox_path = get_sandbox_path(&self.id);
 
-        std::fs::create_dir_all(sandbox_path.clone()).context("failed to create sandbox path")?;
+        create_dir_all_with_inherit_owner(sandbox_path.clone(), 0o750)
+            .context("failed to create sandbox path")?;
 
         let vsock_socket_path = get_vsock_path(&self.id)?;
 
@@ -392,6 +403,20 @@ impl CloudHypervisorInner {
             );
         }
 
+        let user: Option<RootlessUser> = if is_rootless() {
+            Some(
+                self.config
+                    .security_info
+                    .rootless_user
+                    .clone()
+                    .ok_or_else(|| {
+                        anyhow!("rootless user must be specified for rootless cloud-hypervisor")
+                    })?,
+            )
+        } else {
+            None
+        };
+
         unsafe {
             let selinux_label = self.config.security_info.selinux_label.clone();
             let _pre = cmd.pre_exec(move || {
@@ -412,6 +437,16 @@ impl CloudHypervisorInner {
                         );
                     }
                 }
+                if let Some(user) = &user {
+                    let groups = user.groups.clone();
+                    let gid = Gid::from_raw(user.gid);
+                    let uid = Uid::from_raw(user.uid);
+
+                    let _ = set_groups(&groups);
+                    let _ = setgid(gid).context("setgid failed");
+                    let _ = setuid(uid).context("setuid failed");
+                }
+
                 Ok(())
             });
         }
@@ -644,11 +679,11 @@ impl CloudHypervisorInner {
         self.run_dir = get_sandbox_path(&self.id);
         self.vm_path = self.run_dir.to_string();
 
-        create_dir_all(&self.run_dir)
+        create_dir_all_with_inherit_owner(&self.run_dir, 0o750)
             .with_context(|| anyhow!("failed to create sandbox directory {}", self.run_dir))?;
 
         if !self.jailer_root.is_empty() {
-            create_dir_all(self.jailer_root.as_str())
+            create_dir_all_with_inherit_owner(self.jailer_root.as_str(), 0o750)
                 .map_err(|e| anyhow!("Failed to create dir {} err : {:?}", self.jailer_root, e))?;
         }
 
@@ -727,7 +762,9 @@ impl CloudHypervisorInner {
     }
 
     pub(crate) async fn cleanup(&self) -> Result<()> {
-        Ok(())
+        info!(sl!(), "CloudHypervisor::cleanup()");
+        remove_dir_all(get_rootless_symlink_sandbox_path(self.id.as_str()))?;
+        vm_cleanup(&self.config, self.vm_path.as_str())
     }
 
     pub(crate) async fn resize_vcpu(
@@ -806,7 +843,7 @@ impl CloudHypervisorInner {
     pub(crate) async fn get_jailer_root(&self) -> Result<String> {
         let root_path = get_jailer_root(&self.id);
 
-        std::fs::create_dir_all(&root_path)?;
+        create_dir_all_with_inherit_owner(&root_path, 0o750)?;
 
         Ok(root_path)
     }

src/runtime-rs/crates/hypervisor/src/ch/utils.rs

@@ -2,9 +2,12 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use anyhow::Result;
+use std::path::Path;
 
-use crate::utils::get_sandbox_path;
+use anyhow::{Ok, Result};
+use kata_types::build_path;
+
+use crate::{utils::get_sandbox_path, JAILER_ROOT};
 
 // The socket used to connect to CH. This is used for CH API communications.
 const CH_API_SOCKET_NAME: &str = "ch-api.sock";
@@ -34,3 +37,17 @@ pub fn get_vsock_path(id: &str) -> Result<String> {
 
     Ok(path)
 }
+
+/// Returns the symlink path of the sandbox for the virtio-fs socket in rootless mode.
+pub fn get_rootless_symlink_sandbox_path(id: &str) -> String {
+    Path::new(build_path(id).as_str())
+        .to_string_lossy()
+        .to_string()
+}
+
+/// Returns the symlink path of the sandbox's jailer root for the virtio-fs socket in rootless mode.
+pub fn get_rootless_symlink_sandbox_jailer_root(id: &str) -> String {
+    let sandbox_path = get_rootless_symlink_sandbox_path(id);
+
+    [&sandbox_path, JAILER_ROOT].join("/")
+}