agent: Modify Kata agent for runk

Generate an oci-kata-agent which is a customized agent to be
called from runk which is a Rust-based standard OCI container
runtime based on Kata agent.

Fixes: #2784

Signed-off-by: Manabu Sugimoto <Manabu.Sugimoto@sony.com>

src/agent/Cargo.lock

@@ -1370,6 +1370,7 @@ dependencies = [
  "async-trait",
  "capctl",
  "caps",
+ "cfg-if 0.1.10",
  "cgroups-rs",
  "futures",
  "inotify",

src/agent/Cargo.toml

@@ -76,3 +76,13 @@ lto = true
 
 [features]
 seccomp = ["rustjail/seccomp"]
+standard-oci-runtime = ["rustjail/standard-oci-runtime"]
+
+[[bin]]
+name = "kata-agent"
+path = "src/main.rs"
+
+[[bin]]
+name = "oci-kata-agent"
+path = "src/main.rs"
+required-features = ["standard-oci-runtime"]

src/agent/Makefile

@@ -33,6 +33,14 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
+##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
+STANDARD_OCI_RUNTIME := no
+
+# Enable standard oci runtime feature of rust build
+ifeq ($(STANDARD_OCI_RUNTIME),yes)
+    override EXTRA_RUSTFEATURES += standard-oci-runtime
+endif
+
 ifneq ($(EXTRA_RUSTFEATURES),)
     override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif

src/agent/rustjail/Cargo.toml

@@ -25,6 +25,7 @@ path-absolutize = "1.2.0"
 anyhow = "1.0.32"
 cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
+cfg-if = "0.1.0"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
 futures = "0.3.17"
@@ -38,3 +39,4 @@ tempfile = "3.1.0"
 
 [features]
 seccomp = ["libseccomp"]
+standard-oci-runtime = []

src/agent/rustjail/src/console.rs

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright 2021 Sony Group Corporation
+//
+
+use anyhow::{anyhow, Result};
+use nix::errno::Errno;
+use nix::pty;
+use nix::sys::{socket, uio};
+use nix::unistd::{self, dup2};
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::path::Path;
+
+pub fn setup_console_socket(csocket_path: &str) -> Result<Option<RawFd>> {
+    if csocket_path.is_empty() {
+        return Ok(None);
+    }
+
+    let socket_fd = socket::socket(
+        socket::AddressFamily::Unix,
+        socket::SockType::Stream,
+        socket::SockFlag::empty(),
+        None,
+    )?;
+
+    match socket::connect(
+        socket_fd,
+        &socket::SockAddr::Unix(socket::UnixAddr::new(Path::new(csocket_path))?),
+    ) {
+        Ok(()) => Ok(Some(socket_fd)),
+        Err(errno) => Err(anyhow!("failed to open console fd: {}", errno)),
+    }
+}
+
+pub fn setup_master_console(socket_fd: RawFd) -> Result<()> {
+    let pseudo = pty::openpty(None, None)?;
+
+    let pty_name: &[u8] = b"/dev/ptmx";
+    let iov = [uio::IoVec::from_slice(pty_name)];
+    let fds = [pseudo.master];
+    let cmsg = socket::ControlMessage::ScmRights(&fds);
+
+    socket::sendmsg(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
+
+    unistd::setsid()?;
+    let ret = unsafe { libc::ioctl(pseudo.slave, libc::TIOCSCTTY) };
+    Errno::result(ret).map_err(|e| anyhow!(e).context("ioctl TIOCSCTTY"))?;
+
+    dup2(pseudo.slave, std::io::stdin().as_raw_fd())?;
+    dup2(pseudo.slave, std::io::stdout().as_raw_fd())?;
+    dup2(pseudo.slave, std::io::stderr().as_raw_fd())?;
+
+    unistd::close(socket_fd)?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::skip_if_not_root;
+    use std::fs::File;
+    use std::os::unix::net::UnixListener;
+    use std::path::PathBuf;
+    use tempfile::{self, tempdir};
+
+    const CONSOLE_SOCKET: &str = "console-socket";
+
+    #[test]
+    fn test_setup_console_socket() {
+        let dir = tempdir()
+            .map_err(|e| anyhow!(e).context("tempdir failed"))
+            .unwrap();
+        let socket_path = dir.path().join(CONSOLE_SOCKET);
+
+        let _listener = UnixListener::bind(&socket_path).unwrap();
+
+        let ret = setup_console_socket(socket_path.to_str().unwrap());
+
+        assert!(ret.is_ok());
+    }
+}

src/agent/rustjail/src/container.rs

@@ -23,6 +23,8 @@ use crate::cgroups::fs::Manager as FsManager;
 #[cfg(test)]
 use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
+#[cfg(feature = "standard-oci-runtime")]
+use crate::console;
 use crate::log_child;
 use crate::process::Process;
 #[cfg(feature = "seccomp")]
@@ -64,7 +66,7 @@ use tokio::sync::Mutex;
 
 use crate::utils;
 
-const EXEC_FIFO_FILENAME: &str = "exec.fifo";
+pub const EXEC_FIFO_FILENAME: &str = "exec.fifo";
 
 const INIT: &str = "INIT";
 const NO_PIVOT: &str = "NO_PIVOT";
@@ -74,6 +76,10 @@ const CLOG_FD: &str = "CLOG_FD";
 const FIFO_FD: &str = "FIFO_FD";
 const HOME_ENV_KEY: &str = "HOME";
 const PIDNS_FD: &str = "PIDNS_FD";
+const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD";
+
+#[cfg(feature = "standard-oci-runtime")]
+const OCI_AGENT_BINARY: &str = "oci-kata-agent";
 
 #[derive(Debug)]
 pub struct ContainerStatus {
@@ -82,7 +88,7 @@ pub struct ContainerStatus {
 }
 
 impl ContainerStatus {
-    fn new() -> Self {
+    pub fn new() -> Self {
         ContainerStatus {
             pre_status: ContainerState::Created,
             cur_status: ContainerState::Created,
@@ -99,6 +105,12 @@ impl ContainerStatus {
     }
 }
 
+impl Default for ContainerStatus {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 pub type Config = CreateOpts;
 type NamespaceType = String;
 
@@ -106,7 +118,7 @@ lazy_static! {
     // This locker ensures the child exit signal will be received by the right receiver.
     pub static ref WAIT_PID_LOCKER: Arc<Mutex<bool>> = Arc::new(Mutex::new(false));
 
-    static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
+    pub static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
         let mut m = HashMap::new();
         m.insert("user", CloneFlags::CLONE_NEWUSER);
         m.insert("ipc", CloneFlags::CLONE_NEWIPC);
@@ -119,7 +131,7 @@ lazy_static! {
     };
 
 // type to name hashmap, better to be in NAMESPACES
-    static ref TYPETONAME: HashMap<&'static str, &'static str> = {
+    pub static ref TYPETONAME: HashMap<&'static str, &'static str> = {
         let mut m = HashMap::new();
         m.insert("ipc", "ipc");
         m.insert("user", "user");
@@ -236,6 +248,8 @@ pub struct LinuxContainer {
     pub status: ContainerStatus,
     pub created: SystemTime,
     pub logger: Logger,
+    #[cfg(feature = "standard-oci-runtime")]
+    pub console_socket: PathBuf,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -359,7 +373,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
             )));
         }
     }
-
     log_child!(cfd_log, "child process start run");
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
@@ -379,6 +392,9 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     let cm: FsManager = serde_json::from_str(cm_str)?;
 
+    #[cfg(feature = "standard-oci-runtime")]
+    let csocket_fd = console::setup_console_socket(&std::env::var(CONSOLE_SOCKET_FD)?)?;
+
     let p = if spec.process.is_some() {
         spec.process.as_ref().unwrap()
     } else {
@@ -670,10 +686,19 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let _ = unistd::close(crfd);
     let _ = unistd::close(cwfd);
 
-    unistd::setsid().context("create a new session")?;
     if oci_process.terminal {
-        unsafe {
-            libc::ioctl(0, libc::TIOCSCTTY);
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "standard-oci-runtime")] {
+                if let Some(csocket_fd) = csocket_fd {
+                    console::setup_master_console(csocket_fd)?;
+                } else {
+                    return Err(anyhow!("failed to get console master socket fd"));
+                }
+            }
+            else {
+                unistd::setsid().context("create a new session")?;
+                unsafe { libc::ioctl(0, libc::TIOCSCTTY) };
+            }
         }
     }
 
@@ -926,8 +951,24 @@ impl BaseContainer for LinuxContainer {
             let _ = unistd::close(pid);
         });
 
-        let exec_path = std::env::current_exe()?;
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "standard-oci-runtime")] {
+                let exec_path = PathBuf::from(OCI_AGENT_BINARY);
+            }
+            else {
+                let exec_path = std::env::current_exe()?;
+            }
+        }
+
         let mut child = std::process::Command::new(exec_path);
+
+        #[allow(unused_mut)]
+        let mut console_name = PathBuf::from("");
+        #[cfg(feature = "standard-oci-runtime")]
+        if !self.console_socket.as_os_str().is_empty() {
+            console_name = self.console_socket.clone();
+        }
+
         let mut child = child
             .arg("init")
             .stdin(child_stdin)
@@ -937,7 +978,8 @@ impl BaseContainer for LinuxContainer {
             .env(NO_PIVOT, format!("{}", self.config.no_pivot_root))
             .env(CRFD_FD, format!("{}", crfd))
             .env(CWFD_FD, format!("{}", cwfd))
-            .env(CLOG_FD, format!("{}", cfd_log));
+            .env(CLOG_FD, format!("{}", cfd_log))
+            .env(CONSOLE_SOCKET_FD, console_name);
 
         if p.init {
             child = child.env(FIFO_FD, format!("{}", fifofd));
@@ -1419,8 +1461,16 @@ impl LinuxContainer {
                 .unwrap()
                 .as_secs(),
             logger: logger.new(o!("module" => "rustjail", "subsystem" => "container", "cid" => id)),
+            #[cfg(feature = "standard-oci-runtime")]
+            console_socket: Path::new("").to_path_buf(),
         })
     }
+
+    #[cfg(feature = "standard-oci-runtime")]
+    pub fn set_console_socket(&mut self, console_socket: &Path) -> Result<()> {
+        self.console_socket = console_socket.to_path_buf();
+        Ok(())
+    }
 }
 
 fn setgroups(grps: &[libc::gid_t]) -> Result<()> {
@@ -1460,7 +1510,7 @@ use std::process::Stdio;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
+pub async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let logger = logger.new(o!("action" => "execute-hook"));
 
     let binary = PathBuf::from(h.path.as_str());

src/agent/rustjail/src/lib.rs

@@ -30,6 +30,8 @@ extern crate regex;
 
 pub mod capabilities;
 pub mod cgroups;
+#[cfg(feature = "standard-oci-runtime")]
+pub mod console;
 pub mod container;
 pub mod mount;
 pub mod pipestream;

src/agent/rustjail/src/specconv.rs

@@ -5,7 +5,7 @@
 
 use oci::Spec;
 
-#[derive(Debug)]
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
 pub struct CreateOpts {
     pub cgroup_name: String,
     pub use_systemd_cgroup: bool,

src/agent/src/rpc.rs

@@ -1802,7 +1802,7 @@ async fn do_add_swap(sandbox: &Arc<Mutex<Sandbox>>, req: &AddSwapRequest) -> Res
 // - config.json at /<CONTAINER_BASE>/<cid>/config.json
 // - container rootfs bind mounted at /<CONTAINER_BASE>/<cid>/rootfs
 // - modify container spec root to point to /<CONTAINER_BASE>/<cid>/rootfs
-fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
+pub fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
     let spec_root = if let Some(sr) = &spec.root {
         sr
     } else {