runtime-rs: bring hybridVsock devices in manager.

Currently, virtio_vsock are still outside of the device
manager. This causes some management issues,such as the
inability to unify PCI address management.

Just do some work for hybrid vsock.

Fixes: #7655

Signed-off-by: alex.lyn <alex.lyn@antgroup.com>

src/libs/Cargo.lock

@@ -1123,6 +1123,7 @@ dependencies = [
  "anyhow",
  "hyper",
  "hyperlocal",
+ "kata-types",
  "tokio",
 ]
 

src/libs/kata-types/src/config/mod.rs

@@ -34,6 +34,9 @@ pub use self::runtime::{Runtime, RuntimeVendor, RUNTIME_NAME_VIRTCONTAINER};
 
 pub use self::agent::AGENT_NAME_KATA;
 
+/// kata run dir
+pub const KATA_PATH: &str = "/run/kata";
+
 // TODO: let agent use the constants here for consistency
 /// Debug console enabled flag for agent
 pub const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console";

src/libs/shim-interface/Cargo.toml

@@ -16,3 +16,4 @@ anyhow = "^1.0"
 tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
 hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
 hyperlocal = "0.8"
+kata-types = { path = "../kata-types" }

src/libs/shim-interface/src/lib.rs

@@ -21,7 +21,8 @@ use anyhow::{anyhow, Result};
 
 pub mod shim_mgmt;
 
-pub const KATA_PATH: &str = "/run/kata";
+use kata_types::config::KATA_PATH;
+
 pub const SHIM_MGMT_SOCK_NAME: &str = "shim-monitor.sock";
 
 // return sandbox's storage path

src/runtime-rs/Cargo.lock

@@ -3000,10 +3000,10 @@ dependencies = [
  "async-trait",
  "common",
  "containerd-shim-protos",
+ "kata-types",
  "logging",
  "persist",
  "runtimes",
- "shim-interface",
  "slog",
  "slog-scope",
  "tokio",
@@ -3107,6 +3107,7 @@ dependencies = [
  "anyhow",
  "hyper",
  "hyperlocal",
+ "kata-types",
  "tokio",
 ]
 

src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs

@@ -5,10 +5,9 @@
 
 use super::inner::CloudHypervisorInner;
 use crate::ch::utils::get_api_socket_path;
-use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path};
+use crate::ch::utils::get_vsock_path;
-use crate::device::DeviceType;
 use crate::kernel_param::KernelParams;
-use crate::VsockDevice;
+use crate::utils::{get_jailer_root, get_sandbox_path};
 use crate::VM_ROOTFS_DRIVER_PMEM;
 use crate::{VcpuThreadIds, VmmState};
 use anyhow::{anyhow, Context, Result};
@@ -116,7 +115,7 @@ impl CloudHypervisorInner {
             .ok_or("missing socket")
             .map_err(|e| anyhow!(e))?;
 
-        let sandbox_path = get_sandbox_path(&self.id)?;
+        let sandbox_path = get_sandbox_path(&self.id);
 
         std::fs::create_dir_all(sandbox_path.clone()).context("failed to create sandbox path")?;
 
@@ -417,20 +416,12 @@ impl CloudHypervisorInner {
 
         self.netns = netns;
 
-        let vsock_dev = VsockDevice::new(self.id.clone()).await?;
-
-        self.add_device(DeviceType::Vsock(vsock_dev))
-            .await
-            .context("add vsock device")?;
-
-        self.start_hypervisor(self.timeout_secs).await?;
-
         Ok(())
     }
 
     async fn setup_environment(&mut self) -> Result<()> {
         // run_dir and vm_path are the same (shared)
-        self.run_dir = get_sandbox_path(&self.id)?;
+        self.run_dir = get_sandbox_path(&self.id);
         self.vm_path = self.run_dir.to_string();
 
         create_dir_all(&self.run_dir)
@@ -445,9 +436,8 @@ impl CloudHypervisorInner {
     }
 
     pub(crate) async fn start_vm(&mut self, timeout_secs: i32) -> Result<()> {
-        self.setup_environment().await?;
-
         self.timeout_secs = timeout_secs;
+        self.start_hypervisor(self.timeout_secs).await?;
 
         self.boot_vm().await?;
 
@@ -524,7 +514,7 @@ impl CloudHypervisorInner {
     }
 
     pub(crate) async fn get_jailer_root(&self) -> Result<String> {
-        let root_path = get_jailer_root(&self.id)?;
+        let root_path = get_jailer_root(&self.id);
 
         std::fs::create_dir_all(&root_path)?;
 

src/runtime-rs/crates/hypervisor/src/ch/utils.rs

@@ -3,7 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use anyhow::Result;
-use shim_interface::KATA_PATH;
+
+use crate::utils::get_sandbox_path;
 
 // The socket used to connect to CH. This is used for CH API communications.
 const CH_API_SOCKET_NAME: &str = "ch-api.sock";
@@ -12,21 +13,11 @@ const CH_API_SOCKET_NAME: &str = "ch-api.sock";
 // Containers agent running inside the CH hosted VM.
 const CH_VM_SOCKET_NAME: &str = "ch-vm.sock";
 
-const CH_JAILER_DIR: &str = "root";
-
-// Return the path for a _hypothetical_ sandbox: the path does *not* exist
-// yet, and for this reason safe-path cannot be used.
-pub fn get_sandbox_path(id: &str) -> Result<String> {
-    let path = [KATA_PATH, id].join("/");
-
-    Ok(path)
-}
-
 // Return the path for a _hypothetical_ API socket path:
 // the path does *not* exist yet, and for this reason safe-path cannot be
 // used.
 pub fn get_api_socket_path(id: &str) -> Result<String> {
-    let sandbox_path = get_sandbox_path(id)?;
+    let sandbox_path = get_sandbox_path(id);
 
     let path = [&sandbox_path, CH_API_SOCKET_NAME].join("/");
 
@@ -37,17 +28,9 @@ pub fn get_api_socket_path(id: &str) -> Result<String> {
 // the path does *not* exist yet, and for this reason safe-path cannot be
 // used.
 pub fn get_vsock_path(id: &str) -> Result<String> {
-    let sandbox_path = get_sandbox_path(id)?;
+    let sandbox_path = get_sandbox_path(id);
 
     let path = [&sandbox_path, CH_VM_SOCKET_NAME].join("/");
 
     Ok(path)
 }
-
-pub fn get_jailer_root(id: &str) -> Result<String> {
-    let sandbox_path = get_sandbox_path(id)?;
-
-    let path = [&sandbox_path, CH_JAILER_DIR].join("/");
-
-    Ok(path)
-}

src/runtime-rs/crates/hypervisor/src/device/device_manager.rs

@@ -11,9 +11,9 @@ use kata_sys_util::rand::RandomBytes;
 use tokio::sync::{Mutex, RwLock};
 
 use crate::{
-    vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, Hypervisor, NetworkDevice,
+    vhost_user_blk::VhostUserBlkDevice, BlockConfig, BlockDevice, HybridVsockDevice, Hypervisor,
-    VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE, KATA_NVDIMM_DEV_TYPE,
+    NetworkDevice, VfioDevice, VhostUserConfig, KATA_BLK_DEV_TYPE, KATA_MMIO_BLK_DEV_TYPE,
-    VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
+    KATA_NVDIMM_DEV_TYPE, VIRTIO_BLOCK_MMIO, VIRTIO_BLOCK_PCI, VIRTIO_PMEM,
 };
 
 use super::{
@@ -320,6 +320,10 @@ impl DeviceManager {
 
                 Arc::new(Mutex::new(NetworkDevice::new(device_id.clone(), config)))
             }
+            DeviceConfig::HybridVsockCfg(hvconfig) => {
+                // No need to do find device for hybrid vsock device.
+                Arc::new(Mutex::new(HybridVsockDevice::new(&device_id, hvconfig)))
+            }
             _ => {
                 return Err(anyhow!("invliad device type"));
             }

src/runtime-rs/crates/hypervisor/src/device/driver/mod.rs

@@ -24,7 +24,9 @@ pub use virtio_fs::{
     ShareFsOperation,
 };
 pub use virtio_net::{Address, NetworkConfig, NetworkDevice};
-pub use virtio_vsock::{HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice};
+pub use virtio_vsock::{
+    HybridVsockConfig, HybridVsockDevice, VsockConfig, VsockDevice, DEFAULT_GUEST_VSOCK_CID,
+};
 
 pub mod vhost_user_blk;
 pub use vhost_user::{VhostUserConfig, VhostUserDevice, VhostUserType};

src/runtime-rs/crates/hypervisor/src/device/driver/virtio_vsock.rs

@@ -9,7 +9,18 @@ use rand::Rng;
 use std::os::unix::prelude::AsRawFd;
 use tokio::fs::{File, OpenOptions};
 
-#[derive(Debug)]
+use async_trait::async_trait;
+
+use crate::{
+    device::{Device, DeviceType},
+    Hypervisor as hypervisor,
+};
+
+// This is the first usable vsock context ID. All the vsocks
+// can use the same ID, since it's only used in the guest.
+pub const DEFAULT_GUEST_VSOCK_CID: u32 = 0x3;
+
+#[derive(Clone, Debug, Default)]
 pub struct HybridVsockConfig {
     /// A 32-bit Context Identifier (CID) used to identify the guest.
     pub guest_cid: u32,
@@ -18,7 +29,7 @@ pub struct HybridVsockConfig {
     pub uds_path: String,
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct HybridVsockDevice {
     /// Unique identifier of the device
     pub id: String,
@@ -27,6 +38,47 @@ pub struct HybridVsockDevice {
     pub config: HybridVsockConfig,
 }
 
+impl HybridVsockDevice {
+    pub fn new(device_id: &String, config: &HybridVsockConfig) -> Self {
+        Self {
+            id: format!("vsock-{}", device_id),
+            config: config.clone(),
+        }
+    }
+}
+
+#[async_trait]
+impl Device for HybridVsockDevice {
+    async fn attach(&mut self, h: &dyn hypervisor) -> Result<()> {
+        h.add_device(DeviceType::HybridVsock(self.clone()))
+            .await
+            .context("add hybrid vsock device.")?;
+
+        return Ok(());
+    }
+
+    async fn detach(&mut self, _h: &dyn hypervisor) -> Result<Option<u64>> {
+        // no need to do detach, just return Ok(None)
+        Ok(None)
+    }
+
+    async fn get_device_info(&self) -> DeviceType {
+        DeviceType::HybridVsock(self.clone())
+    }
+
+    async fn increase_attach_count(&mut self) -> Result<bool> {
+        // hybrid vsock devices will not be attached multiple times, Just return Ok(false)
+
+        Ok(false)
+    }
+
+    async fn decrease_attach_count(&mut self) -> Result<bool> {
+        // hybrid vsock devices will not be detached multiple times, Just return Ok(false)
+
+        Ok(false)
+    }
+}
+
 #[derive(Debug)]
 pub struct VsockConfig {
     /// A 32-bit Context Identifier (CID) used to identify the guest.

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -19,11 +19,10 @@ use dragonball::{
 use kata_sys_util::mount;
 use kata_types::{
     capabilities::{Capabilities, CapabilityBits},
-    config::hypervisor::Hypervisor as HypervisorConfig,
+    config::{hypervisor::Hypervisor as HypervisorConfig, KATA_PATH},
 };
 use nix::mount::MsFlags;
 use persist::sandbox_persist::Persist;
-use shim_interface::KATA_PATH;
 use std::{collections::HashSet, fs::create_dir_all};
 
 const DRAGONBALL_KERNEL: &str = "vmlinux";

src/runtime-rs/crates/hypervisor/src/dragonball/inner_device.rs

@@ -19,7 +19,7 @@ use dragonball::{
 use super::DragonballInner;
 use crate::{
     device::DeviceType, HybridVsockConfig, NetworkConfig, ShareFsDeviceConfig, ShareFsMountConfig,
-    ShareFsMountType, ShareFsOperation, VfioBusMode, VfioDevice, VmmState,
+    ShareFsMountType, ShareFsOperation, VfioBusMode, VfioDevice, VmmState, JAILER_ROOT,
 };
 
 const MB_TO_B: u32 = 1024 * 1024;
@@ -231,11 +231,12 @@ impl DragonballInner {
 
     fn add_hvsock(&mut self, config: &HybridVsockConfig) -> Result<()> {
         let vsock_cfg = VsockDeviceConfigInfo {
-            id: String::from("root"),
+            id: String::from(JAILER_ROOT),
             guest_cid: config.guest_cid,
             uds_path: Some(config.uds_path.clone()),
             ..Default::default()
         };
+        debug!(sl!(), "HybridVsock configure: {:?}", &vsock_cfg);
 
         self.vmm_instance
             .insert_vsock(vsock_cfg)

src/runtime-rs/crates/hypervisor/src/dragonball/inner_hypervisor.rs

@@ -14,35 +14,19 @@ use kata_types::capabilities::Capabilities;
 
 use super::inner::DragonballInner;
 use crate::{
-    device::DeviceType, utils, HybridVsockConfig, HybridVsockDevice, VcpuThreadIds, VmmState,
+    utils::{self, get_hvsock_path, get_jailer_root, get_sandbox_path},
+    VcpuThreadIds, VmmState,
 };
-use shim_interface::KATA_PATH;
-const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
-
-fn get_vsock_path(root: &str) -> String {
-    [root, DEFAULT_HYBRID_VSOCK_NAME].join("/")
-}
 
 impl DragonballInner {
     pub(crate) async fn prepare_vm(&mut self, id: &str, netns: Option<String>) -> Result<()> {
         self.id = id.to_string();
         self.state = VmmState::NotReady;
 
-        self.vm_path = [KATA_PATH, id].join("/");
+        self.vm_path = get_sandbox_path(id);
-        self.jailer_root = [self.vm_path.as_str(), "root"].join("/");
+        self.jailer_root = get_jailer_root(id);
         self.netns = netns;
 
-        // prepare vsock
-        let uds_path = [&self.jailer_root, DEFAULT_HYBRID_VSOCK_NAME].join("/");
-        let d = DeviceType::HybridVsock(HybridVsockDevice {
-            id: format!("vsock-{}", &self.id),
-            config: HybridVsockConfig {
-                guest_cid: 3,
-                uds_path,
-            },
-        });
-
-        self.add_device(d).await.context("add device")?;
         Ok(())
     }
 
@@ -88,7 +72,7 @@ impl DragonballInner {
         Ok(format!(
             "{}://{}",
             HYBRID_VSOCK_SCHEME,
-            get_vsock_path(&self.jailer_root),
+            get_hvsock_path(&self.id),
         ))
     }
 

src/runtime-rs/crates/hypervisor/src/lib.rs

@@ -17,7 +17,7 @@ pub mod dragonball;
 mod kernel_param;
 pub mod qemu;
 pub use kernel_param::Param;
-mod utils;
+pub mod utils;
 use std::collections::HashMap;
 
 #[cfg(feature = "cloud-hypervisor")]
@@ -56,6 +56,9 @@ const SHMEM: &str = "shmem";
 pub const HYPERVISOR_DRAGONBALL: &str = "dragonball";
 pub const HYPERVISOR_QEMU: &str = "qemu";
 
+pub const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
+pub const JAILER_ROOT: &str = "root";
+
 #[derive(PartialEq, Debug, Clone)]
 pub(crate) enum VmmState {
     NotReady,

src/runtime-rs/crates/hypervisor/src/utils.rs

@@ -6,6 +6,10 @@
 
 use std::collections::HashSet;
 
+use kata_types::config::KATA_PATH;
+
+use crate::{DEFAULT_HYBRID_VSOCK_NAME, JAILER_ROOT};
+
 pub fn get_child_threads(pid: u32) -> HashSet<u32> {
     let mut result = HashSet::new();
     let path_name = format!("/proc/{}/task", pid);
@@ -25,3 +29,21 @@ pub fn get_child_threads(pid: u32) -> HashSet<u32> {
     }
     result
 }
+
+// Return the path for a _hypothetical_ sandbox: the path does *not* exist
+// yet, and for this reason safe-path cannot be used.
+pub fn get_sandbox_path(sid: &str) -> String {
+    [KATA_PATH, sid].join("/")
+}
+
+pub fn get_hvsock_path(sid: &str) -> String {
+    let jailer_root_path = get_jailer_root(sid);
+
+    [jailer_root_path, DEFAULT_HYBRID_VSOCK_NAME.to_owned()].join("/")
+}
+
+pub fn get_jailer_root(sid: &str) -> String {
+    let sandbox_path = get_sandbox_path(sid);
+
+    [&sandbox_path, JAILER_ROOT].join("/")
+}

src/runtime-rs/crates/persist/src/lib.rs

@@ -6,8 +6,8 @@
 
 pub mod sandbox_persist;
 use anyhow::{anyhow, Context, Ok, Result};
+use kata_types::config::KATA_PATH;
 use serde::de;
-use shim_interface::KATA_PATH;
 use std::{fs::File, io::BufReader};
 
 pub const PERSIST_FILE: &str = "state.json";

src/runtime-rs/crates/resource/src/lib.rs

@@ -17,7 +17,7 @@ pub mod manager;
 mod manager_inner;
 pub mod network;
 pub mod resource_persist;
-use hypervisor::BlockConfig;
+use hypervisor::{BlockConfig, HybridVsockConfig};
 use network::NetworkConfig;
 pub mod rootfs;
 pub mod share_fs;
@@ -32,6 +32,7 @@ pub enum ResourceConfig {
     Network(NetworkConfig),
     ShareFs(SharedFsInfo),
     VmRootfs(BlockConfig),
+    HybridVsock(HybridVsockConfig),
 }
 
 #[derive(Debug, Clone, Copy, PartialEq)]

src/runtime-rs/crates/resource/src/manager_inner.rs

@@ -126,6 +126,11 @@ impl ResourceManagerInner {
                         .await
                         .context("do handle device failed.")?;
                 }
+                ResourceConfig::HybridVsock(hv) => {
+                    do_handle_device(&self.device_manager, &DeviceConfig::HybridVsockCfg(hv))
+                        .await
+                        .context("do handle hybrid-vsock device failed.")?;
+                }
             };
         }
 

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -15,6 +15,7 @@ use common::message::{Action, Message};
 use common::{Sandbox, SandboxNetworkEnv};
 use containerd_shim_protos::events::task::TaskOOM;
 use hypervisor::{dragonball::Dragonball, BlockConfig, Hypervisor, HYPERVISOR_DRAGONBALL};
+use hypervisor::{utils::get_hvsock_path, HybridVsockConfig, DEFAULT_GUEST_VSOCK_CID};
 use kata_sys_util::hooks::HookStates;
 use kata_types::config::TomlConfig;
 use persist::{self, sandbox_persist::Persist};
@@ -101,6 +102,14 @@ impl VirtSandbox {
     ) -> Result<Vec<ResourceConfig>> {
         let mut resource_configs = vec![];
 
+        // Prepare VM hybrid vsock device config and add the hybrid vsock device first.
+        info!(sl!(), "prepare hybrid vsock resource for sandbox.");
+        let vm_hvsock = ResourceConfig::HybridVsock(HybridVsockConfig {
+            guest_cid: DEFAULT_GUEST_VSOCK_CID,
+            uds_path: get_hvsock_path(id),
+        });
+        resource_configs.push(vm_hvsock);
+
         // prepare network config
         if !network_env.network_created {
             if let Some(network_resource) = self.prepare_network_resource(&network_env).await {

src/runtime-rs/crates/service/Cargo.toml

@@ -17,6 +17,6 @@ ttrpc = { version = "0.7.1" }
 common = { path = "../runtimes/common" }
 containerd-shim-protos = { version = "0.3.0", features = ["async"]}
 logging = { path = "../../../libs/logging"}
-shim-interface = { path = "../../../libs/shim-interface" }
+kata-types = { path = "../../../libs/kata-types" }
 runtimes = { path = "../runtimes" }
 persist = { path = "../persist" }

src/runtime-rs/crates/service/src/manager.rs

@@ -17,8 +17,8 @@ use containerd_shim_protos::{
     protobuf::{well_known_types::any::Any, Message as ProtobufMessage},
     shim_async,
 };
+use kata_types::config::KATA_PATH;
 use runtimes::RuntimeHandlerManager;
-use shim_interface::KATA_PATH;
 use tokio::{
     io::AsyncWriteExt,
     process::Command,