diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index e8cd3e39a..0c63e3b21 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -1123,6 +1123,7 @@ dependencies = [ "anyhow", "hyper", "hyperlocal", + "kata-types", "tokio", ] diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index e73f8b84e..52b621d12 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -34,6 +34,9 @@ pub use self::runtime::{Runtime, RuntimeVendor, RUNTIME_NAME_VIRTCONTAINER}; pub use self::agent::AGENT_NAME_KATA; +/// kata run dir +pub const KATA_PATH: &str = "/run/kata"; + // TODO: let agent use the constants here for consistency /// Debug console enabled flag for agent pub const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console"; diff --git a/src/libs/shim-interface/Cargo.toml b/src/libs/shim-interface/Cargo.toml index bd6cc6e0a..360b4a42e 100644 --- a/src/libs/shim-interface/Cargo.toml +++ b/src/libs/shim-interface/Cargo.toml @@ -16,3 +16,4 @@ anyhow = "^1.0" tokio = { version = "1.8.0", features = ["rt-multi-thread"] } hyper = { version = "0.14.20", features = ["stream", "server", "http1"] } hyperlocal = "0.8" +kata-types = { path = "../kata-types" } diff --git a/src/libs/shim-interface/src/lib.rs b/src/libs/shim-interface/src/lib.rs index bbc96c09f..add611d4a 100644 --- a/src/libs/shim-interface/src/lib.rs +++ b/src/libs/shim-interface/src/lib.rs @@ -21,7 +21,8 @@ use anyhow::{anyhow, Result}; pub mod shim_mgmt; -pub const KATA_PATH: &str = "/run/kata"; +use kata_types::config::KATA_PATH; + pub const SHIM_MGMT_SOCK_NAME: &str = "shim-monitor.sock"; // return sandbox's storage path diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 68872f7f0..66a5ac0ee 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -3000,10 +3000,10 @@ dependencies = [ "async-trait", "common", "containerd-shim-protos", + "kata-types", "logging", "persist", "runtimes", - "shim-interface", "slog", "slog-scope", "tokio", @@ -3107,6 +3107,7 @@ dependencies = [ "anyhow", "hyper", "hyperlocal", + "kata-types", "tokio", ] diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs index fc020f0d2..106d461e6 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -5,10 +5,9 @@ use super::inner::CloudHypervisorInner; use crate::ch::utils::get_api_socket_path; -use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path}; -use crate::device::DeviceType; +use crate::ch::utils::get_vsock_path; use crate::kernel_param::KernelParams; -use crate::VsockDevice; +use crate::utils::{get_jailer_root, get_sandbox_path}; use crate::VM_ROOTFS_DRIVER_PMEM; use crate::{VcpuThreadIds, VmmState}; use anyhow::{anyhow, Context, Result}; @@ -116,7 +115,7 @@ impl CloudHypervisorInner { .ok_or("missing socket") .map_err(|e| anyhow!(e))?; - let sandbox_path = get_sandbox_path(&self.id)?; + let sandbox_path = get_sandbox_path(&self.id); std::fs::create_dir_all(sandbox_path.clone()).context("failed to create sandbox path")?; @@ -417,20 +416,12 @@ impl CloudHypervisorInner { self.netns = netns; - let vsock_dev = VsockDevice::new(self.id.clone()).await?; - - self.add_device(DeviceType::Vsock(vsock_dev)) - .await - .context("add vsock device")?; - - self.start_hypervisor(self.timeout_secs).await?; - Ok(()) } async fn setup_environment(&mut self) -> Result<()> { // run_dir and vm_path are the same (shared) - self.run_dir = get_sandbox_path(&self.id)?; + self.run_dir = get_sandbox_path(&self.id); self.vm_path = self.run_dir.to_string(); create_dir_all(&self.run_dir) @@ -445,9 +436,8 @@ impl CloudHypervisorInner { } pub(crate) async fn start_vm(&mut self, timeout_secs: i32) -> Result<()> { - self.setup_environment().await?; - self.timeout_secs = timeout_secs; + self.start_hypervisor(self.timeout_secs).await?; self.boot_vm().await?; @@ -524,7 +514,7 @@ impl CloudHypervisorInner { } pub(crate) async fn get_jailer_root(&self) -> Result { - let root_path = get_jailer_root(&self.id)?; + let root_path = get_jailer_root(&self.id); std::fs::create_dir_all(&root_path)?; diff --git a/src/runtime-rs/crates/hypervisor/src/ch/utils.rs b/src/runtime-rs/crates/hypervisor/src/ch/utils.rs index dcea26aeb..05291bb2e 100644 --- a/src/runtime-rs/crates/hypervisor/src/ch/utils.rs +++ b/src/runtime-rs/crates/hypervisor/src/ch/utils.rs @@ -3,7 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; -use shim_interface::KATA_PATH; + +use crate::utils::get_sandbox_path; // The socket used to connect to CH. This is used for CH API communications. const CH_API_SOCKET_NAME: &str = "ch-api.sock"; @@ -12,21 +13,11 @@ const CH_API_SOCKET_NAME: &str = "ch-api.sock"; // Containers agent running inside the CH hosted VM. const CH_VM_SOCKET_NAME: &str = "ch-vm.sock"; -const CH_JAILER_DIR: &str = "root"; - -// Return the path for a _hypothetical_ sandbox: the path does *not* exist -// yet, and for this reason safe-path cannot be used. -pub fn get_sandbox_path(id: &str) -> Result { - let path = [KATA_PATH, id].join("/"); - - Ok(path) -} - // Return the path for a _hypothetical_ API socket path: // the path does *not* exist yet, and for this reason safe-path cannot be // used. pub fn get_api_socket_path(id: &str) -> Result { - let sandbox_path = get_sandbox_path(id)?; + let sandbox_path = get_sandbox_path(id); let path = [&sandbox_path, CH_API_SOCKET_NAME].join("/"); @@ -37,17 +28,9 @@ pub fn get_api_socket_path(id: &str) -> Result { // the path does *not* exist yet, and for this reason safe-path cannot be // used. pub fn get_vsock_path(id: &str) -> Result { - let sandbox_path = get_sandbox_path(id)?; + let sandbox_path = get_sandbox_path(id); let path = [&sandbox_path, CH_VM_SOCKET_NAME].join("/"); Ok(path) } - -pub fn get_jailer_root(id: &str) -> Result { - let sandbox_path = get_sandbox_path(id)?; - - let path = [&sandbox_path, CH_JAILER_DIR].join("/"); - - Ok(path) -} diff --git a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs index c1e650914..1c1ed0b71 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/device_manager.rs @@ -11,9 +11,9 @@ use kata_sys_util::rand::RandomBytes; use tokio::sync::{Mutex, RwLock}; use crate::{ - vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, Hypervisor, NetworkDevice, - VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE, - VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, + vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor, + NetworkDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, + KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM, }; use super::{ @@ -320,6 +320,10 @@ impl DeviceManager { Arc::new(Mutex::new(NetworkDevice::new(device_id.clone(), config))) } + DeviceConfig::HybridVsockCfg(hvconfig) => { + // No need to do find device for hybrid vsock device. + Arc::new(Mutex::new(HybridVsockDevice::new(&device_id, hvconfig))) + } _ => { return Err(anyhow!("invliad device type")); } diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs index 3feb2d2ea..acc6c4b6e 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs @@ -24,7 +24,9 @@ pub use virtio_fs::{ ShareFsOperation, }; pub use virtio_net::{Address, NetworkConfig, NetworkDevice}; -pub use virtio_vsock::{HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice}; +pub use virtio_vsock::{ + HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice, DEFAULT_GUEST_VSOCK_CID, +}; pub mod vhost_user_blk; pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType}; diff --git a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs index d94e8f864..98f48e29a 100644 --- a/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs +++ b/src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs @@ -9,7 +9,18 @@ use rand::Rng; use std::os::unix::prelude::AsRawFd; use tokio::fs::{File, OpenOptions}; -#[derive(Debug)] +use async_trait::async_trait; + +use crate::{ + device::{Device, DeviceType}, + Hypervisor as hypervisor, +}; + +// This is the first usable vsock context ID. All the vsocks +// can use the same ID, since it's only used in the guest. +pub const DEFAULT_GUEST_VSOCK_CID: u32 = 0x3; + +#[derive(Clone, Debug, Default)] pub struct HybridVsockConfig { /// A 32-bit Context Identifier (CID) used to identify the guest. pub guest_cid: u32, @@ -18,7 +29,7 @@ pub struct HybridVsockConfig { pub uds_path: String, } -#[derive(Debug)] +#[derive(Clone, Debug, Default)] pub struct HybridVsockDevice { /// Unique identifier of the device pub id: String, @@ -27,6 +38,47 @@ pub struct HybridVsockDevice { pub config: HybridVsockConfig, } +impl HybridVsockDevice { + pub fn new(device_id: &String, config: &HybridVsockConfig) -> Self { + Self { + id: format!("vsock-{}", device_id), + config: config.clone(), + } + } +} + +#[async_trait] +impl Device for HybridVsockDevice { + async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> { + h.add_device(DeviceType::HybridVsock(self.clone())) + .await + .context("add hybrid vsock device.")?; + + return Ok(()); + } + + async fn detach(&mut self, _h: &dyn hypervisor) -> Result> { + // no need to do detach, just return Ok(None) + Ok(None) + } + + async fn get_device_info(&self) -> DeviceType { + DeviceType::HybridVsock(self.clone()) + } + + async fn increase_attach_count(&mut self) -> Result { + // hybrid vsock devices will not be attached multiple times, Just return Ok(false) + + Ok(false) + } + + async fn decrease_attach_count(&mut self) -> Result { + // hybrid vsock devices will not be detached multiple times, Just return Ok(false) + + Ok(false) + } +} + #[derive(Debug)] pub struct VsockConfig { /// A 32-bit Context Identifier (CID) used to identify the guest. diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index 4e49e0baa..6823dac9b 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -19,11 +19,10 @@ use dragonball::{ use kata_sys_util::mount; use kata_types::{ capabilities::{Capabilities, CapabilityBits}, - config::hypervisor::Hypervisor as HypervisorConfig, + config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH}, }; use nix::mount::MsFlags; use persist::sandbox_persist::Persist; -use shim_interface::KATA_PATH; use std::{collections::HashSet, fs::create_dir_all}; const DRAGONBALL_KERNEL: &str = "vmlinux"; diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs index 47f5d635d..3e2ea3e86 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs @@ -19,7 +19,7 @@ use dragonball::{ use super::DragonballInner; use crate::{ device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig, - ShareFsMountType, ShareFsOperation, VfioBusMode, VfioDevice, VmmState, + ShareFsMountType, ShareFsOperation, VfioBusMode, VfioDevice, VmmState, JAILER_ROOT, }; const MB_TO_B: u32 = 1024 * 1024; @@ -231,11 +231,12 @@ impl DragonballInner { fn add_hvsock(&mut self, config: &HybridVsockConfig) -> Result<()> { let vsock_cfg = VsockDeviceConfigInfo { - id: String::from("root"), + id: String::from(JAILER_ROOT), guest_cid: config.guest_cid, uds_path: Some(config.uds_path.clone()), ..Default::default() }; + debug!(sl!(), "HybridVsock configure: {:?}", &vsock_cfg); self.vmm_instance .insert_vsock(vsock_cfg) diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs index 5fc1da4f1..5de9ceb00 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs @@ -14,35 +14,19 @@ use kata_types::capabilities::Capabilities; use super::inner::DragonballInner; use crate::{ - device::DeviceType, utils, HybridVsockConfig, HybridVsockDevice, VcpuThreadIds, VmmState, + utils::{self, get_hvsock_path, get_jailer_root, get_sandbox_path}, + VcpuThreadIds, VmmState, }; -use shim_interface::KATA_PATH; -const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock"; - -fn get_vsock_path(root: &str) -> String { - [root, DEFAULT_HYBRID_VSOCK_NAME].join("/") -} impl DragonballInner { pub(crate) async fn prepare_vm(&mut self, id: &str, netns: Option) -> Result<()> { self.id = id.to_string(); self.state = VmmState::NotReady; - self.vm_path = [KATA_PATH, id].join("/"); - self.jailer_root = [self.vm_path.as_str(), "root"].join("/"); + self.vm_path = get_sandbox_path(id); + self.jailer_root = get_jailer_root(id); self.netns = netns; - // prepare vsock - let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/"); - let d = DeviceType::HybridVsock(HybridVsockDevice { - id: format!("vsock-{}", &self.id), - config: HybridVsockConfig { - guest_cid: 3, - uds_path, - }, - }); - - self.add_device(d).await.context("add device")?; Ok(()) } @@ -88,7 +72,7 @@ impl DragonballInner { Ok(format!( "{}://{}", HYBRID_VSOCK_SCHEME, - get_vsock_path(&self.jailer_root), + get_hvsock_path(&self.id), )) } diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index e9fa3a9bf..69f9fe9b4 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -17,7 +17,7 @@ pub mod dragonball; mod kernel_param; pub mod qemu; pub use kernel_param::Param; -mod utils; +pub mod utils; use std::collections::HashMap; #[cfg(feature = "cloud-hypervisor")] @@ -56,6 +56,9 @@ const SHMEM: &str = "shmem"; pub const HYPERVISOR_DRAGONBALL: &str = "dragonball"; pub const HYPERVISOR_QEMU: &str = "qemu"; +pub const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock"; +pub const JAILER_ROOT: &str = "root"; + #[derive(PartialEq, Debug, Clone)] pub(crate) enum VmmState { NotReady, diff --git a/src/runtime-rs/crates/hypervisor/src/utils.rs b/src/runtime-rs/crates/hypervisor/src/utils.rs index 8ecf98950..d2078c2d2 100644 --- a/src/runtime-rs/crates/hypervisor/src/utils.rs +++ b/src/runtime-rs/crates/hypervisor/src/utils.rs @@ -6,6 +6,10 @@ use std::collections::HashSet; +use kata_types::config::KATA_PATH; + +use crate::{DEFAULT_HYBRID_VSOCK_NAME, JAILER_ROOT}; + pub fn get_child_threads(pid: u32) -> HashSet { let mut result = HashSet::new(); let path_name = format!("/proc/{}/task", pid); @@ -25,3 +29,21 @@ pub fn get_child_threads(pid: u32) -> HashSet { } result } + +// Return the path for a _hypothetical_ sandbox: the path does *not* exist +// yet, and for this reason safe-path cannot be used. +pub fn get_sandbox_path(sid: &str) -> String { + [KATA_PATH, sid].join("/") +} + +pub fn get_hvsock_path(sid: &str) -> String { + let jailer_root_path = get_jailer_root(sid); + + [jailer_root_path, DEFAULT_HYBRID_VSOCK_NAME.to_owned()].join("/") +} + +pub fn get_jailer_root(sid: &str) -> String { + let sandbox_path = get_sandbox_path(sid); + + [&sandbox_path, JAILER_ROOT].join("/") +} diff --git a/src/runtime-rs/crates/persist/src/lib.rs b/src/runtime-rs/crates/persist/src/lib.rs index 3b728c39c..0c6510a0d 100644 --- a/src/runtime-rs/crates/persist/src/lib.rs +++ b/src/runtime-rs/crates/persist/src/lib.rs @@ -6,8 +6,8 @@ pub mod sandbox_persist; use anyhow::{anyhow, Context, Ok, Result}; +use kata_types::config::KATA_PATH; use serde::de; -use shim_interface::KATA_PATH; use std::{fs::File, io::BufReader}; pub const PERSIST_FILE: &str = "state.json"; diff --git a/src/runtime-rs/crates/resource/src/lib.rs b/src/runtime-rs/crates/resource/src/lib.rs index c12aa0e48..f7df9b687 100644 --- a/src/runtime-rs/crates/resource/src/lib.rs +++ b/src/runtime-rs/crates/resource/src/lib.rs @@ -17,7 +17,7 @@ pub mod manager; mod manager_inner; pub mod network; pub mod resource_persist; -use hypervisor::BlockConfig; +use hypervisor::{BlockConfig, HybridVsockConfig}; use network::NetworkConfig; pub mod rootfs; pub mod share_fs; @@ -32,6 +32,7 @@ pub enum ResourceConfig { Network(NetworkConfig), ShareFs(SharedFsInfo), VmRootfs(BlockConfig), + HybridVsock(HybridVsockConfig), } #[derive(Debug, Clone, Copy, PartialEq)] diff --git a/src/runtime-rs/crates/resource/src/manager_inner.rs b/src/runtime-rs/crates/resource/src/manager_inner.rs index cadc75f67..1f51ca9c0 100644 --- a/src/runtime-rs/crates/resource/src/manager_inner.rs +++ b/src/runtime-rs/crates/resource/src/manager_inner.rs @@ -126,6 +126,11 @@ impl ResourceManagerInner { .await .context("do handle device failed.")?; } + ResourceConfig::HybridVsock(hv) => { + do_handle_device(&self.device_manager, &DeviceConfig::HybridVsockCfg(hv)) + .await + .context("do handle hybrid-vsock device failed.")?; + } }; } diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs index 6ede6fb2c..a86aa07d7 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs @@ -15,6 +15,7 @@ use common::message::{Action, Message}; use common::{Sandbox, SandboxNetworkEnv}; use containerd_shim_protos::events::task::TaskOOM; use hypervisor::{dragonball::Dragonball, BlockConfig, Hypervisor, HYPERVISOR_DRAGONBALL}; +use hypervisor::{utils::get_hvsock_path, HybridVsockConfig, DEFAULT_GUEST_VSOCK_CID}; use kata_sys_util::hooks::HookStates; use kata_types::config::TomlConfig; use persist::{self, sandbox_persist::Persist}; @@ -101,6 +102,14 @@ impl VirtSandbox { ) -> Result> { let mut resource_configs = vec![]; + // Prepare VM hybrid vsock device config and add the hybrid vsock device first. + info!(sl!(), "prepare hybrid vsock resource for sandbox."); + let vm_hvsock = ResourceConfig::HybridVsock(HybridVsockConfig { + guest_cid: DEFAULT_GUEST_VSOCK_CID, + uds_path: get_hvsock_path(id), + }); + resource_configs.push(vm_hvsock); + // prepare network config if !network_env.network_created { if let Some(network_resource) = self.prepare_network_resource(&network_env).await { diff --git a/src/runtime-rs/crates/service/Cargo.toml b/src/runtime-rs/crates/service/Cargo.toml index b92fbbf34..693cce330 100644 --- a/src/runtime-rs/crates/service/Cargo.toml +++ b/src/runtime-rs/crates/service/Cargo.toml @@ -17,6 +17,6 @@ ttrpc = { version = "0.7.1" } common = { path = "../runtimes/common" } containerd-shim-protos = { version = "0.3.0", features = ["async"]} logging = { path = "../../../libs/logging"} -shim-interface = { path = "../../../libs/shim-interface" } +kata-types = { path = "../../../libs/kata-types" } runtimes = { path = "../runtimes" } persist = { path = "../persist" } diff --git a/src/runtime-rs/crates/service/src/manager.rs b/src/runtime-rs/crates/service/src/manager.rs index 46ffc18db..2b16fe3f4 100644 --- a/src/runtime-rs/crates/service/src/manager.rs +++ b/src/runtime-rs/crates/service/src/manager.rs @@ -17,8 +17,8 @@ use containerd_shim_protos::{ protobuf::{well_known_types::any::Any, Message as ProtobufMessage}, shim_async, }; +use kata_types::config::KATA_PATH; use runtimes::RuntimeHandlerManager; -use shim_interface::KATA_PATH; use tokio::{ io::AsyncWriteExt, process::Command,