From eb6bb6fe0d9eb80adaf23308747cd9f3f49c1a95 Mon Sep 17 00:00:00 2001 From: Zixuan Tan Date: Fri, 28 Jul 2023 00:46:56 +0800 Subject: [PATCH] config: add two options to control vsock passthrough io feature Two toml options, `use_passfd_io` and `passfd_listener_port` are introduced to enable and configure dragonball's vsock fd passthrough io feature. This commit is a preparation for vsock fd passthrough io feature. Fixes: #6714 Signed-off-by: Zixuan Tan --- src/agent/src/config.rs | 12 ++++++++++++ src/libs/kata-types/src/config/agent.rs | 10 ++++++++++ src/libs/kata-types/src/config/default.rs | 1 + src/libs/kata-types/src/config/mod.rs | 2 ++ src/libs/kata-types/src/config/runtime.rs | 12 ++++++++++++ .../config/configuration-dragonball.toml.in | 7 +++++++ .../crates/hypervisor/src/dragonball/inner.rs | 19 ++++++++++++++++++- .../crates/hypervisor/src/dragonball/mod.rs | 5 +++++ .../hypervisor/src/hypervisor_persist.rs | 1 + .../src/container_manager/container.rs | 2 ++ .../crates/runtimes/virt_container/src/lib.rs | 5 +++++ 11 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index 75c0a245c2..abb8be0241 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -18,6 +18,7 @@ const DEV_MODE_FLAG: &str = "agent.devmode"; const TRACE_MODE_OPTION: &str = "agent.trace"; const LOG_LEVEL_OPTION: &str = "agent.log"; const SERVER_ADDR_OPTION: &str = "agent.server_addr"; +const PASSFD_LISTENER_PORT: &str = "agent.passfd_listener_port"; const HOTPLUG_TIMOUT_OPTION: &str = "agent.hotplug_timeout"; const DEBUG_CONSOLE_VPORT_OPTION: &str = "agent.debug_console_vport"; const LOG_VPORT_OPTION: &str = "agent.log_vport"; @@ -61,6 +62,7 @@ pub struct AgentConfig { pub log_vport: i32, pub container_pipe_size: i32, pub server_addr: String, + pub passfd_listener_port: i32, pub unified_cgroup_hierarchy: bool, pub tracing: bool, pub supports_seccomp: bool, @@ -76,6 +78,7 @@ pub struct AgentConfigBuilder { pub log_vport: Option, pub container_pipe_size: Option, pub server_addr: Option, + pub passfd_listener_port: Option, pub unified_cgroup_hierarchy: Option, pub tracing: Option, } @@ -135,6 +138,7 @@ impl Default for AgentConfig { log_vport: 0, container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: format!("{}:{}", VSOCK_ADDR, DEFAULT_AGENT_VSOCK_PORT), + passfd_listener_port: 0, unified_cgroup_hierarchy: false, tracing: false, supports_seccomp: rpc::have_seccomp(), @@ -164,6 +168,7 @@ impl FromStr for AgentConfig { config_override!(agent_config_builder, agent_config, log_vport); config_override!(agent_config_builder, agent_config, container_pipe_size); config_override!(agent_config_builder, agent_config, server_addr); + config_override!(agent_config_builder, agent_config, passfd_listener_port); config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy); config_override!(agent_config_builder, agent_config, tracing); @@ -245,6 +250,13 @@ impl AgentConfig { get_vsock_port, |port| port > 0 ); + parse_cmdline_param!( + param, + PASSFD_LISTENER_PORT, + config.passfd_listener_port, + get_vsock_port, + |port| port > 0 + ); parse_cmdline_param!( param, diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index 1ac7515cad..7f1fae555a 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -11,6 +11,7 @@ pub use vendor::AgentVendor; use super::default::{ DEFAULT_AGENT_DIAL_TIMEOUT_MS, DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT, + DEFAULT_PASSFD_LISTENER_PORT, }; use crate::eother; @@ -60,6 +61,10 @@ pub struct Agent { #[serde(default = "default_log_port")] pub log_port: u32, + /// Agent process io port + #[serde(default = "default_passfd_listener_port")] + pub passfd_listener_port: u32, + /// Agent connection dialing timeout value in millisecond #[serde(default = "default_dial_timeout")] pub dial_timeout_ms: u32, @@ -104,6 +109,7 @@ impl std::default::Default for Agent { debug_console_enabled: false, server_port: DEFAULT_AGENT_VSOCK_PORT, log_port: DEFAULT_AGENT_LOG_PORT, + passfd_listener_port: DEFAULT_PASSFD_LISTENER_PORT, dial_timeout_ms: DEFAULT_AGENT_DIAL_TIMEOUT_MS, reconnect_timeout_ms: 3_000, request_timeout_ms: 30_000, @@ -126,6 +132,10 @@ fn default_log_port() -> u32 { DEFAULT_AGENT_LOG_PORT } +fn default_passfd_listener_port() -> u32 { + DEFAULT_PASSFD_LISTENER_PORT +} + fn default_dial_timeout() -> u32 { // ms 10 diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 0dfd5d09e5..d269ea1abf 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -25,6 +25,7 @@ pub const DEFAULT_AGENT_NAME: &str = "kata-agent"; pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024; pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025; pub const DEFAULT_AGENT_DBG_CONSOLE_PORT: u32 = 1026; +pub const DEFAULT_PASSFD_LISTENER_PORT: u32 = 1027; pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA; pub const DEFAULT_AGENT_DIAL_TIMEOUT_MS: u32 = 10; diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index dcfbadf3fa..beb93e697c 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -54,6 +54,8 @@ pub const DEBUG_CONSOLE_VPORT_OPTION: &str = "agent.debug_console_vport"; pub const LOG_VPORT_OPTION: &str = "agent.log_vport"; /// Option of setting the container's pipe size pub const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size"; +/// Option of setting the fd passthrough io listener port +pub const PASSFD_LISTENER_PORT: &str = "agent.passfd_listener_port"; /// Trait to manipulate global Kata configuration information. pub trait ConfigPlugin: Send + Sync { diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index 99820dd5f4..0954f5039e 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -167,6 +167,18 @@ pub struct Runtime { /// shared_mount declarations #[serde(default)] pub shared_mounts: Vec, + + /// If enabled, the runtime will attempt to use fd passthrough feature for process io. + #[serde(default)] + pub use_passfd_io: bool, + + /// If fd passthrough io is enabled, the runtime will attempt to use the specified port instead of the default port. + #[serde(default = "default_passfd_listener_port")] + pub passfd_listener_port: u32, +} + +fn default_passfd_listener_port() -> u32 { + default::DEFAULT_PASSFD_LISTENER_PORT } impl ConfigOps for Runtime { diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index 7f08a7db92..5a0a6db20a 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -372,3 +372,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@ # to the hypervisor. # (default: /run/kata-containers/dans) dan_conf = "@DEFDANCONF@" + +# If enabled, the runtime will attempt to use fd passthrough feature for process io. +# Note: this feature is only supported by the Dragonball hypervisor. +use_passfd_io = true + +# If fd passthrough io is enabled, the runtime will attempt to use the specified port instead of the default port. +# passfd_listener_port = 1027 diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 28f04296fd..bfe2601788 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -22,7 +22,7 @@ use kata_types::{ capabilities::{Capabilities, CapabilityBits}, config::{ hypervisor::{HugePageType, Hypervisor as HypervisorConfig}, - KATA_PATH, + KATA_PATH, PASSFD_LISTENER_PORT, }, }; use nix::mount::MsFlags; @@ -80,6 +80,10 @@ pub struct DragonballInner { /// the balloon size pub(crate) balloon_size: u32, + + /// guest-side fd passthrough io listener port, used to initialize + /// connections for io + pub(crate) passfd_listener_port: Option, } impl DragonballInner { @@ -108,6 +112,7 @@ impl DragonballInner { guest_memory_block_size_mb: 0, mem_hotplug_size_mb: 0, balloon_size: 0, + passfd_listener_port: None, } } @@ -128,6 +133,12 @@ impl DragonballInner { kernel_params.append(&mut KernelParams::from_string( &self.config.boot_info.kernel_params, )); + if let Some(passfd_listener_port) = self.passfd_listener_port { + kernel_params.append(&mut KernelParams::from_string(&format!( + "{}={}", + PASSFD_LISTENER_PORT, passfd_listener_port + ))); + } info!(sl!(), "prepared kernel_params={:?}", kernel_params); // set boot source @@ -458,6 +469,10 @@ impl DragonballInner { pub(crate) fn guest_memory_block_size_mb(&self) -> u32 { self.guest_memory_block_size_mb } + + pub fn set_passfd_listener_port(&mut self, port: u32) { + self.passfd_listener_port = Some(port); + } } #[async_trait] @@ -477,6 +492,7 @@ impl Persist for DragonballInner { config: self.hypervisor_config(), run_dir: self.run_dir.clone(), cached_block_devices: self.cached_block_devices.clone(), + passfd_listener_port: self.passfd_listener_port, ..Default::default() }) } @@ -502,6 +518,7 @@ impl Persist for DragonballInner { guest_memory_block_size_mb: 0, mem_hotplug_size_mb: 0, balloon_size: 0, + passfd_listener_port: hypervisor_state.passfd_listener_port, }) } } diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs index bec411b88f..980f0e0eb1 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/mod.rs @@ -55,6 +55,11 @@ impl Dragonball { let mut inner = self.inner.write().await; inner.set_hypervisor_config(config) } + + pub async fn set_passfd_listener_port(&mut self, port: u32) { + let mut inner = self.inner.write().await; + inner.set_passfd_listener_port(port) + } } #[async_trait] diff --git a/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs b/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs index ea870f3420..1ae231d834 100644 --- a/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs +++ b/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs @@ -33,4 +33,5 @@ pub struct HypervisorState { /// cached block device pub cached_block_devices: HashSet, pub virtiofs_daemon_pid: i32, + pub passfd_listener_port: Option, } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index cb7548feaa..27563ea5d4 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -43,6 +43,7 @@ pub struct Container { agent: Arc, resource_manager: Arc, logger: slog::Logger, + passfd_listener_addr: Option<(String, u32)>, } impl Container { @@ -84,6 +85,7 @@ impl Container { agent, resource_manager, logger, + None, }) } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs index 60a3a6eb0e..f6dbdc4eae 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -140,6 +140,11 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result> hypervisor .set_hypervisor_config(hypervisor_config.clone()) .await; + if toml_config.runtime.use_passfd_io { + hypervisor + .set_passfd_listener_port(toml_config.runtime.passfd_listener_port) + .await; + } Ok(Arc::new(hypervisor)) } HYPERVISOR_QEMU => {