runtime-rs: support loading kernel modules in guest vm

Users can specify the kernel module to be loaded through the agent
configuration in kata configuration file or in pod anotation file.

And information of those modules will be sent to kata agent when
sandbox is created.

Fixes: #4894

Signed-off-by: Yushuo <y-shuo@linux.alibaba.com>

src/agent/Cargo.lock

@@ -625,6 +625,7 @@ dependencies = [
  "futures",
  "ipnetwork",
  "kata-sys-util",
+ "kata-types",
  "lazy_static",
  "libc",
  "log",

src/agent/Cargo.toml

@@ -21,6 +21,7 @@ thiserror = "1.0.26"
 regex = "1.5.5"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
+kata-types = { path = "../libs/kata-types" }
 sysinfo = "0.23.0"
 
 # Async helpers

src/agent/src/config.rs

@@ -12,6 +12,8 @@ use std::str::FromStr;
 use std::time;
 use tracing::instrument;
 
+use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
+
 const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console";
 const DEV_MODE_FLAG: &str = "agent.devmode";
 const TRACE_MODE_OPTION: &str = "agent.trace";
@@ -28,7 +30,6 @@ const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
 const DEFAULT_CONTAINER_PIPE_SIZE: i32 = 0;
 const VSOCK_ADDR: &str = "vsock://-1";
-const VSOCK_PORT: u16 = 1024;
 
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
@@ -147,7 +148,7 @@ impl Default for AgentConfig {
             debug_console_vport: 0,
             log_vport: 0,
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-            server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
+            server_addr: format!("{}:{}", VSOCK_ADDR, DEFAULT_AGENT_VSOCK_PORT),
             unified_cgroup_hierarchy: false,
             tracing: false,
             endpoints: Default::default(),

src/libs/kata-types/src/config/agent.rs

@@ -9,8 +9,10 @@ use crate::config::{ConfigOps, TomlConfig};
 
 pub use vendor::AgentVendor;
 
+use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT};
+
 /// Kata agent configuration information.
-#[derive(Debug, Default, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize, Clone)]
 pub struct Agent {
     /// If enabled, the agent will log additional debug messages to the system log.
     #[serde(default, rename = "enable_debug")]
@@ -34,11 +36,11 @@ pub struct Agent {
     pub debug_console_enabled: bool,
 
     /// Agent server port
-    #[serde(default)]
+    #[serde(default = "default_server_port")]
     pub server_port: u32,
 
     /// Agent log port
-    #[serde(default)]
+    #[serde(default = "default_log_port")]
     pub log_port: u32,
 
     /// Agent connection dialing timeout value in millisecond
@@ -75,23 +77,31 @@ pub struct Agent {
     pub container_pipe_size: u32,
 }
 
+fn default_server_port() -> u32 {
+    DEFAULT_AGENT_VSOCK_PORT
+}
+
+fn default_log_port() -> u32 {
+    DEFAULT_AGENT_LOG_PORT
+}
+
 fn default_dial_timeout() -> u32 {
-    // 10ms
+    // ms
     10
 }
 
 fn default_reconnect_timeout() -> u32 {
-    // 3s
+    // ms
     3_000
 }
 
 fn default_request_timeout() -> u32 {
-    // 30s
+    // ms
     30_000
 }
 
 fn default_health_check_timeout() -> u32 {
-    // 90s
+    // ms
     90_000
 }
 

src/libs/kata-types/src/config/default.rs

@@ -16,6 +16,8 @@ lazy_static! {
     ];
 }
 pub const DEFAULT_AGENT_NAME: &str = "kata-agent";
+pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024;
+pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025;
 
 pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter";
 

src/runtime-rs/crates/agent/src/kata/agent.rs

@@ -8,6 +8,8 @@ use anyhow::{Context, Result};
 use async_trait::async_trait;
 use ttrpc::context as ttrpc_ctx;
 
+use kata_types::config::Agent as AgentConfig;
+
 use crate::{kata::KataAgent, Agent, AgentManager, HealthService};
 
 /// millisecond to nanosecond
@@ -37,6 +39,10 @@ impl AgentManager for KataAgent {
     async fn stop(&self) {
         self.stop_log_forwarder().await;
     }
+
+    async fn agent_config(&self) -> AgentConfig {
+        self.agent_config().await
+    }
 }
 
 // implement for health service

src/runtime-rs/crates/agent/src/kata/mod.rs

@@ -126,4 +126,9 @@ impl KataAgent {
         let mut inner = self.inner.lock().await;
         inner.log_forwarder.stop();
     }
+
+    pub(crate) async fn agent_config(&self) -> AgentConfig {
+        let inner = self.inner.lock().await;
+        inner.config.clone()
+    }
 }

src/runtime-rs/crates/agent/src/lib.rs

@@ -29,10 +29,16 @@ pub use types::{
 use anyhow::Result;
 use async_trait::async_trait;
 
+use kata_types::config::Agent as AgentConfig;
+
+pub const AGENT_KATA: &str = "kata";
+
 #[async_trait]
 pub trait AgentManager: Send + Sync {
     async fn start(&self, address: &str) -> Result<()>;
     async fn stop(&self);
+
+    async fn agent_config(&self) -> AgentConfig;
 }
 
 #[async_trait]

src/runtime-rs/crates/agent/src/types.rs

@@ -4,6 +4,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use anyhow::{anyhow, Result};
+use std::convert::TryFrom;
+
 use serde::Deserialize;
 
 #[derive(PartialEq, Clone, Default)]
@@ -390,6 +393,56 @@ pub struct KernelModule {
     pub parameters: Vec<String>,
 }
 
+impl KernelModule {
+    pub fn set_kernel_modules(modules: Vec<String>) -> Result<Vec<Self>> {
+        let mut kernel_modules = Vec::new();
+        for module_string in modules {
+            if module_string.is_empty() {
+                continue;
+            }
+            let kernel_module = Self::try_from(module_string)?;
+            kernel_modules.push(kernel_module);
+        }
+        Ok(kernel_modules)
+    }
+}
+
+impl TryFrom<String> for KernelModule {
+    type Error = anyhow::Error;
+    // input string: " ModuleName Param1 Param2 ... "
+    // NOTICE: " ModuleName Param1="spaces in here" " => KernelModule { name: ModuleName, parameters: Param1="spaces in here" }
+    fn try_from(str: String) -> Result<Self> {
+        let split: Vec<&str> = str.split(' ').collect();
+        let mut name = String::new();
+        let mut parameters = Vec::new();
+
+        let mut flag = false;
+        for (index, info) in split.iter().enumerate() {
+            if index == 0 {
+                name = info.to_string();
+            } else if flag {
+                // a former param's string contains \"
+                if let Some(former_param) = parameters.pop() {
+                    let cur_param = format!("{} {}", former_param, info);
+                    parameters.push(cur_param);
+                }
+            } else {
+                parameters.push(info.to_string());
+            }
+
+            if info.contains('\"') {
+                flag = !flag;
+            }
+        }
+
+        if flag {
+            return Err(anyhow!("\" not match"));
+        }
+
+        Ok(KernelModule { name, parameters })
+    }
+}
+
 #[derive(PartialEq, Clone, Default)]
 pub struct CreateSandboxRequest {
     pub hostname: String,
@@ -486,3 +539,44 @@ pub struct VersionCheckResponse {
 pub struct OomEventResponse {
     pub container_id: String,
 }
+
+#[cfg(test)]
+mod test {
+    use std::convert::TryFrom;
+
+    use super::KernelModule;
+
+    #[test]
+    fn test_new_kernel_module() {
+        let kernel_module_str1 = "ModuleName Param1 Param2";
+        let kernel_module1 = KernelModule::try_from(kernel_module_str1.to_string()).unwrap();
+        assert!(kernel_module1.name == "ModuleName");
+        assert!(kernel_module1.parameters[0] == "Param1");
+        assert!(kernel_module1.parameters[1] == "Param2");
+
+        let kernel_module_str2 = "ModuleName Param1=\"spaces in here\"";
+        let kernel_module2 = KernelModule::try_from(kernel_module_str2.to_string()).unwrap();
+        assert!(kernel_module2.name == "ModuleName");
+        assert!(kernel_module2.parameters[0] == "Param1=\"spaces in here\"");
+
+        // exception case
+        let kernel_module_str3 = "ModuleName \"Param1";
+        let kernel_module3 = KernelModule::try_from(kernel_module_str3.to_string());
+        assert!(kernel_module3.is_err());
+    }
+
+    #[test]
+    fn test_kernel_modules() {
+        let kernel_module_str1 = "ModuleName1 Param1 Param2".to_string();
+        let kernel_module_str2 = "".to_string();
+        let kernel_module_str3 = "ModuleName2".to_string();
+        let kernel_modules_str = vec![kernel_module_str1, kernel_module_str2, kernel_module_str3];
+
+        let kernel_modules = KernelModule::set_kernel_modules(kernel_modules_str).unwrap();
+        assert!(kernel_modules.len() == 2);
+        assert!(kernel_modules[0].name == "ModuleName1");
+        assert!(kernel_modules[0].parameters.len() == 2);
+        assert!(kernel_modules[1].name == "ModuleName2");
+        assert!(kernel_modules[1].parameters.is_empty());
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -16,7 +16,7 @@ pub mod sandbox_persist;
 
 use std::sync::Arc;
 
-use agent::kata::KataAgent;
+use agent::{kata::KataAgent, AGENT_KATA};
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use common::{message::Message, RuntimeHandler, RuntimeInstance};
@@ -55,20 +55,7 @@ impl RuntimeHandler for VirtContainer {
         let hypervisor = new_hypervisor(&config).await.context("new hypervisor")?;
 
         // get uds from hypervisor and get config from toml_config
-        let agent = Arc::new(KataAgent::new(kata_types::config::Agent {
-            debug: true,
-            enable_tracing: false,
-            server_port: 1024,
-            log_port: 1025,
-            dial_timeout_ms: 10,
-            reconnect_timeout_ms: 3_000,
-            request_timeout_ms: 30_000,
-            health_check_request_timeout_ms: 90_000,
-            kernel_modules: Default::default(),
-            container_pipe_size: 0,
-            debug_console_enabled: false,
-        }));
-
+        let agent = new_agent(&config).context("new agent")?;
         let resource_manager = Arc::new(ResourceManager::new(
             sid,
             agent.clone(),
@@ -121,3 +108,44 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
         _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)),
     }
 }
+
+fn new_agent(toml_config: &TomlConfig) -> Result<Arc<KataAgent>> {
+    let agent_name = &toml_config.runtime.agent_name;
+    let agent_config = toml_config
+        .agent
+        .get(agent_name)
+        .ok_or_else(|| anyhow!("failed to get agent for {}", &agent_name))
+        .context("get agent")?;
+    match agent_name.as_str() {
+        AGENT_KATA => {
+            let agent = KataAgent::new(agent_config.clone());
+            Ok(Arc::new(agent))
+        }
+        _ => Err(anyhow!("Unsupported agent {}", &agent_name)),
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    fn default_toml_config_agent() -> Result<TomlConfig> {
+        let config_content = r#"
+[agent.kata]
+container_pipe_size=1
+
+[runtime]
+agent_name="kata"
+        "#;
+        TomlConfig::load(config_content).map_err(|e| anyhow!("can not load config toml: {}", e))
+    }
+
+    #[test]
+    fn test_new_agent() {
+        let toml_config = default_toml_config_agent().unwrap();
+
+        let res = new_agent(&toml_config);
+        assert!(res.is_ok());
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -6,7 +6,7 @@
 
 use std::sync::Arc;
 
-use agent::{self, kata::KataAgent, Agent};
+use agent::{self, kata::KataAgent, types::KernelModule, Agent};
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use common::{
@@ -159,6 +159,8 @@ impl Sandbox for VirtSandbox {
             .context("setup device after start vm")?;
 
         // create sandbox in vm
+        let agent_config = self.agent.agent_config().await;
+        let kernel_modules = KernelModule::set_kernel_modules(agent_config.kernel_modules)?;
         let req = agent::CreateSandboxRequest {
             hostname: "".to_string(),
             dns: vec![],
@@ -175,7 +177,7 @@ impl Sandbox for VirtSandbox {
                 .await
                 .security_info
                 .guest_hook_path,
-            kernel_modules: vec![],
+            kernel_modules,
         };
 
         self.agent