diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 6842ffb59b..598d1940a9 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -81,9 +81,17 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.57" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" + +[[package]] +name = "api_client" +version = "0.1.0" +source = "git+https://github.com/cloud-hypervisor/cloud-hypervisor?tag=v27.0#2ba6a9bfcfd79629aecf77504fa554ab821d138e" +dependencies = [ + "vmm-sys-util 0.10.0", +] [[package]] name = "arc-swap" @@ -412,6 +420,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ch-config" +version = "0.1.0" +dependencies = [ + "anyhow", + "api_client", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "chrono" version = "0.4.22" @@ -934,9 +953,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -949,9 +968,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -959,15 +978,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -976,9 +995,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-lite" @@ -997,9 +1016,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -1008,15 +1027,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-timer" @@ -1026,9 +1045,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", @@ -1114,7 +1133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df0ee4b237afb71e99f7e2fbd840ffec2d6c4bb569f69b2af18aa1f63077d38" dependencies = [ "dashmap", - "futures 0.3.21", + "futures 0.3.26", "futures-timer", "no-std-compat", "nonzero_ext", @@ -1236,8 +1255,10 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "ch-config", "dbs-utils", "dragonball", + "futures 0.3.26", "go-flag", "kata-sys-util", "kata-types", @@ -1246,6 +1267,7 @@ dependencies = [ "nix 0.24.2", "persist", "rand 0.8.5", + "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "seccompiler", "serde", "serde_json", @@ -1551,14 +1573,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1612,7 +1634,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" dependencies = [ "bytes 1.1.0", - "futures 0.3.21", + "futures 0.3.26", "log", "netlink-packet-core", "netlink-sys", @@ -1627,7 +1649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027" dependencies = [ "bytes 1.1.0", - "futures 0.3.21", + "futures 0.3.26", "libc", "log", "tokio", @@ -1783,7 +1805,7 @@ dependencies = [ "bitflags", "blake3", "fuse-backend-rs", - "futures 0.3.21", + "futures 0.3.26", "lazy_static", "libc", "log", @@ -1811,7 +1833,7 @@ dependencies = [ "bitflags", "dbs-uhttp", "fuse-backend-rs", - "futures 0.3.21", + "futures 0.3.26", "governor", "lazy_static", "libc", @@ -1931,7 +1953,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1955,7 +1977,7 @@ dependencies = [ "kata-sys-util", "kata-types", "libc", - "safe-path", + "safe-path 0.1.0", "serde", "serde_json", "shim-interface", @@ -2025,9 +2047,9 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -2321,7 +2343,7 @@ dependencies = [ "bitflags", "byte-unit 4.0.17", "cgroups-rs", - "futures 0.3.21", + "futures 0.3.26", "hypervisor", "kata-sys-util", "kata-types", @@ -2361,7 +2383,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46f1cfa18f8cebe685373a2697915d7e0db3b4554918bba118385e0f71f258a7" dependencies = [ - "futures 0.3.21", + "futures 0.3.26", "log", "netlink-packet-route", "netlink-proto", @@ -2432,6 +2454,15 @@ dependencies = [ "libc", ] +[[package]] +name = "safe-path" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "980abdd3220aa19b67ca3ea07b173ca36383f18ae48cde696d90c8af39447ffb" +dependencies = [ + "libc", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -2455,18 +2486,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.143" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.143" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -2475,9 +2506,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.83" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa", "ryu", @@ -2729,9 +2760,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.96" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -2857,22 +2888,22 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.1" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95eec79ea28c00a365f539f1961e9278fbcaf81c0ff6aaf0e93c181352446948" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ + "autocfg", "bytes 1.1.0", "libc", "memchr", "mio", "num_cpus", - "once_cell", "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -2907,7 +2938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e0723fc001950a3b018947b05eeb45014fd2b7c6e8f292502193ab74486bdb6" dependencies = [ "bytes 0.4.12", - "futures 0.3.21", + "futures 0.3.26", "libc", "tokio", "vsock", @@ -2971,7 +3002,7 @@ checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8" dependencies = [ "async-trait", "byteorder", - "futures 0.3.21", + "futures 0.3.26", "libc", "log", "nix 0.23.1", @@ -3105,7 +3136,7 @@ dependencies = [ "awaitgroup", "common", "containerd-shim-protos", - "futures 0.3.21", + "futures 0.3.26", "hypervisor", "kata-sys-util", "kata-types", @@ -3366,43 +3397,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index 0edbfc8bca..7bd49dd8be 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -32,4 +32,14 @@ shim-interface = { path = "../../../libs/shim-interface" } dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs","dbs-upcall"] } +ch-config = { path = "ch-config", optional = true } + +futures = "0.3.25" +safe-path = "0.1.0" + [features] +default = [] + +# Feature is not yet complete, so not enabled by default. +# See https://github.com/kata-containers/kata-containers/issues/6264. +cloud-hypervisor = ["ch-config"] diff --git a/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml new file mode 100644 index 0000000000..2fd58f9f3d --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/ch-config/Cargo.toml @@ -0,0 +1,22 @@ +# Copyright (c) 2022-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "ch-config" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.68" +serde = { version = "1.0.145", features = ["rc", "derive"] } +serde_json = "1.0.91" +tokio = { version = "1.25.0", features = ["sync", "rt"] } + +# Cloud Hypervisor public HTTP API functions +# Note that the version specified is not necessarily the version of CH +# being used. This version is used to pin the CH config structure +# which is relatively static. +api_client = { git = "https://github.com/cloud-hypervisor/cloud-hypervisor", crate = "api_client", tag = "v27.0" } diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs new file mode 100644 index 0000000000..fe812c7ca8 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/ch_api.rs @@ -0,0 +1,274 @@ +// Copyright (c) 2022-2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use crate::net_util::MAC_ADDR_LEN; +use crate::{ + ConsoleConfig, ConsoleOutputMode, CpuTopology, CpusConfig, DeviceConfig, FsConfig, MacAddr, + MemoryConfig, NetConfig, PayloadConfig, PmemConfig, RngConfig, VmConfig, VsockConfig, +}; +use anyhow::{anyhow, Context, Result}; +use api_client::simple_api_full_command_and_response; + +use std::fmt::Display; +use std::net::Ipv4Addr; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use tokio::task; + +pub async fn cloud_hypervisor_vmm_ping(mut socket: UnixStream) -> Result> { + task::spawn_blocking(move || -> Result> { + let response = simple_api_full_command_and_response(&mut socket, "GET", "vmm.ping", None) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await? +} + +pub async fn cloud_hypervisor_vmm_shutdown(mut socket: UnixStream) -> Result> { + task::spawn_blocking(move || -> Result> { + let response = + simple_api_full_command_and_response(&mut socket, "PUT", "vmm.shutdown", None) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await? +} + +pub async fn cloud_hypervisor_vm_create( + sandbox_path: String, + vsock_socket_path: String, + mut socket: UnixStream, + shared_fs_devices: Option>, + pmem_devices: Option>, +) -> Result> { + let cfg = cloud_hypervisor_vm_create_cfg( + sandbox_path, + vsock_socket_path, + shared_fs_devices, + pmem_devices, + ) + .await?; + + let serialised = serde_json::to_string_pretty(&cfg)?; + + task::spawn_blocking(move || -> Result> { + let data = Some(serialised.as_str()); + + let response = simple_api_full_command_and_response(&mut socket, "PUT", "vm.create", data) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await? +} + +pub async fn cloud_hypervisor_vm_start(mut socket: UnixStream) -> Result> { + task::spawn_blocking(move || -> Result> { + let response = simple_api_full_command_and_response(&mut socket, "PUT", "vm.boot", None) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await? +} + +#[allow(dead_code)] +pub async fn cloud_hypervisor_vm_stop(mut socket: UnixStream) -> Result> { + task::spawn_blocking(move || -> Result> { + let response = + simple_api_full_command_and_response(&mut socket, "PUT", "vm.shutdown", None) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await? +} + +#[allow(dead_code)] +pub async fn cloud_hypervisor_vm_device_add(mut socket: UnixStream) -> Result> { + let device_config = DeviceConfig::default(); + + task::spawn_blocking(move || -> Result> { + let response = simple_api_full_command_and_response( + &mut socket, + "PUT", + "vm.add-device", + Some(&serde_json::to_string(&device_config)?), + ) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await? +} + +pub async fn cloud_hypervisor_vm_fs_add( + mut socket: UnixStream, + fs_config: FsConfig, +) -> Result> { + let result = task::spawn_blocking(move || -> Result> { + let response = simple_api_full_command_and_response( + &mut socket, + "PUT", + "vm.add-fs", + Some(&serde_json::to_string(&fs_config)?), + ) + .map_err(|e| anyhow!(e))?; + + Ok(response) + }) + .await?; + + result +} + +pub async fn cloud_hypervisor_vm_create_cfg( + // FIXME: + _sandbox_path: String, + vsock_socket_path: String, + shared_fs_devices: Option>, + pmem_devices: Option>, +) -> Result { + let topology = CpuTopology { + threads_per_core: 1, + cores_per_die: 12, + dies_per_package: 1, + packages: 1, + }; + + let cpus = CpusConfig { + boot_vcpus: 1, + max_vcpus: 12, + max_phys_bits: 46, + topology: Some(topology), + ..Default::default() + }; + + let rng = RngConfig { + src: PathBuf::from("/dev/urandom"), + ..Default::default() + }; + + let kernel_args = vec![ + "root=/dev/pmem0p1", + "rootflags=dax,data=ordered,errors=remount-ro", + "ro", + "rootfstype=ext4", + "panic=1", + "no_timer_check", + "noreplace-smp", + "console=ttyS0,115200n8", + "systemd.log_target=console", + "systemd.unit=kata-containers", + "systemd.mask=systemd-networkd.service", + "systemd.mask=systemd-networkd.socket", + "agent.log=debug", + ]; + + let cmdline = kernel_args.join(" "); + + let kernel = PathBuf::from("/opt/kata/share/kata-containers/vmlinux.container"); + + // Note that PmemConfig replaces the PayloadConfig.initrd. + let payload = PayloadConfig { + kernel: Some(kernel), + cmdline: Some(cmdline), + ..Default::default() + }; + + let serial = ConsoleConfig { + mode: ConsoleOutputMode::Tty, + ..Default::default() + }; + + let ip = Ipv4Addr::new(192, 168, 10, 10); + let mask = Ipv4Addr::new(255, 255, 255, 0); + + let mac_str = "12:34:56:78:90:01"; + + let mac = parse_mac(mac_str)?; + + let network = NetConfig { + ip, + mask, + mac, + ..Default::default() + }; + + let memory = MemoryConfig { + size: (1024 * 1024 * 2048), + + // Required + shared: true, + + prefault: false, + hugepages: false, + mergeable: false, + + // FIXME: + hotplug_size: Some(16475226112), + + ..Default::default() + }; + + let fs = shared_fs_devices; + let pmem = pmem_devices; + + let vsock = VsockConfig { + cid: 3, + socket: PathBuf::from(vsock_socket_path), + ..Default::default() + }; + + let cfg = VmConfig { + cpus, + memory, + fs, + serial, + pmem, + payload: Some(payload), + vsock: Some(vsock), + rng, + net: Some(vec![network]), + ..Default::default() + }; + + Ok(cfg) +} + +fn parse_mac(s: &S) -> Result +where + S: AsRef + ?Sized + Display, +{ + let v: Vec<&str> = s.as_ref().split(':').collect(); + let mut bytes = [0u8; MAC_ADDR_LEN]; + + if v.len() != MAC_ADDR_LEN { + return Err(anyhow!( + "invalid MAC {} (length {}, expected {})", + s, + v.len(), + MAC_ADDR_LEN + )); + } + + for i in 0..MAC_ADDR_LEN { + if v[i].len() != 2 { + return Err(anyhow!( + "invalid MAC {} (segment {} length {}, expected {})", + s, + i, + v.len(), + 2 + )); + } + + bytes[i] = + u8::from_str_radix(v[i], 16).context(format!("failed to parse MAC address: {}", s))?; + } + + Ok(MacAddr { bytes }) +} diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs new file mode 100644 index 0000000000..3e3fb3412a --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/lib.rs @@ -0,0 +1,481 @@ +// Copyright (c) 2022-2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; +use std::net::Ipv4Addr; +use std::path::PathBuf; + +pub mod ch_api; +pub mod net_util; +mod virtio_devices; + +use crate::virtio_devices::RateLimiterConfig; +pub use net_util::MacAddr; + +pub const MAX_NUM_PCI_SEGMENTS: u16 = 16; + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct BalloonConfig { + pub size: u64, + /// Option to deflate the balloon in case the guest is out of memory. + #[serde(default)] + pub deflate_on_oom: bool, + /// Option to enable free page reporting from the guest. + #[serde(default)] + pub free_page_reporting: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct CmdlineConfig { + pub args: String, +} + +impl CmdlineConfig { + fn is_empty(&self) -> bool { + self.args.is_empty() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct ConsoleConfig { + //#[serde(default = "default_consoleconfig_file")] + pub file: Option, + pub mode: ConsoleOutputMode, + #[serde(default)] + pub iommu: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub enum ConsoleOutputMode { + #[default] + Off, + Pty, + Tty, + File, + Null, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct CpuAffinity { + pub vcpu: u8, + pub host_cpus: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct CpusConfig { + pub boot_vcpus: u8, + pub max_vcpus: u8, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub topology: Option, + #[serde(default)] + pub kvm_hyperv: bool, + #[serde(skip_serializing_if = "u8_is_zero")] + pub max_phys_bits: u8, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub affinity: Option>, + #[serde(default)] + pub features: CpuFeatures, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct CpuFeatures { + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub amx: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct CpuTopology { + pub threads_per_core: u8, + pub cores_per_die: u8, + pub dies_per_package: u8, + pub packages: u8, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct DeviceConfig { + pub path: PathBuf, + #[serde(default)] + pub iommu: bool, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub pci_segment: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct DiskConfig { + pub path: Option, + #[serde(default)] + pub readonly: bool, + #[serde(default)] + pub direct: bool, + #[serde(default)] + pub iommu: bool, + //#[serde(default = "default_diskconfig_num_queues")] + pub num_queues: usize, + //#[serde(default = "default_diskconfig_queue_size")] + pub queue_size: u16, + #[serde(default)] + pub vhost_user: bool, + pub vhost_socket: Option, + #[serde(default)] + pub rate_limiter_config: Option, + #[serde(default)] + pub id: Option, + // For testing use only. Not exposed in API. + #[serde(default)] + pub disable_io_uring: bool, + #[serde(default)] + pub pci_segment: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct FsConfig { + pub tag: String, + pub socket: PathBuf, + //#[serde(default = "default_fsconfig_num_queues")] + pub num_queues: usize, + //#[serde(default = "default_fsconfig_queue_size")] + pub queue_size: u16, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub pci_segment: u16, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub enum HotplugMethod { + #[default] + Acpi, + VirtioMem, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct InitramfsConfig { + pub path: PathBuf, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct KernelConfig { + pub path: PathBuf, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct MemoryConfig { + pub size: u64, + #[serde(default)] + pub mergeable: bool, + #[serde(default)] + pub hotplug_method: HotplugMethod, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub hotplug_size: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub hotplugged_size: Option, + #[serde(default)] + pub shared: bool, + #[serde(default)] + pub hugepages: bool, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub hugepage_size: Option, + #[serde(default)] + pub prefault: bool, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub zones: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct MemoryZoneConfig { + pub id: String, + pub size: u64, + #[serde(default)] + pub file: Option, + #[serde(default)] + pub shared: bool, + #[serde(default)] + pub hugepages: bool, + #[serde(default)] + pub hugepage_size: Option, + #[serde(default)] + pub host_numa_node: Option, + #[serde(default)] + pub hotplug_size: Option, + #[serde(default)] + pub hotplugged_size: Option, + #[serde(default)] + pub prefault: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub struct NetConfig { + //#[serde(default = "default_netconfig_tap")] + #[serde(skip_serializing_if = "Option::is_none")] + pub tap: Option, + //#[serde(default = "default_netconfig_ip")] + pub ip: Ipv4Addr, + //#[serde(default = "default_netconfig_mask")] + pub mask: Ipv4Addr, + //#[serde(default = "default_netconfig_mac")] + pub mac: MacAddr, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub host_mac: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub mtu: Option, + #[serde(default)] + pub iommu: bool, + //#[serde(default = "default_netconfig_num_queues")] + #[serde(skip_serializing_if = "usize_is_zero")] + pub num_queues: usize, + //#[serde(default = "default_netconfig_queue_size")] + #[serde(skip_serializing_if = "u16_is_zero")] + pub queue_size: u16, + #[serde(default)] + pub vhost_user: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub vhost_socket: Option, + #[serde(default)] + pub vhost_mode: VhostMode, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub fds: Option>, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub rate_limiter_config: Option, + #[serde(default)] + #[serde(skip_serializing_if = "u16_is_zero")] + pub pci_segment: u16, +} + +impl Default for NetConfig { + fn default() -> Self { + NetConfig { + tap: None, + ip: Ipv4Addr::new(0, 0, 0, 0), + mask: Ipv4Addr::new(0, 0, 0, 0), + mac: MacAddr::default(), + host_mac: None, + mtu: None, + iommu: false, + num_queues: 0, + queue_size: 0, + vhost_user: false, + vhost_socket: None, + vhost_mode: VhostMode::default(), + id: None, + fds: None, + rate_limiter_config: None, + pci_segment: 0, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct NumaConfig { + #[serde(default)] + pub guest_numa_id: u32, + #[serde(default)] + pub cpus: Option>, + #[serde(default)] + pub distances: Option>, + #[serde(default)] + pub memory_zones: Option>, + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub sgx_epc_sections: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct NumaDistance { + #[serde(default)] + pub destination: u32, + #[serde(default)] + pub distance: u8, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct PayloadConfig { + #[serde(default)] + pub firmware: Option, + #[serde(default)] + pub kernel: Option, + #[serde(default)] + pub cmdline: Option, + #[serde(default)] + pub initramfs: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct PlatformConfig { + //#[serde(default = "default_platformconfig_num_pci_segments")] + pub num_pci_segments: u16, + #[serde(default)] + pub iommu_segments: Option>, + #[serde(default)] + pub serial_number: Option, + #[serde(default)] + pub uuid: Option, + #[serde(default)] + pub oem_strings: Option>, + #[cfg(feature = "tdx")] + #[serde(default)] + pub tdx: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct PmemConfig { + pub file: PathBuf, + #[serde(default)] + pub size: Option, + #[serde(default)] + pub iommu: bool, + #[serde(default)] + pub discard_writes: bool, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub pci_segment: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct RngConfig { + pub src: PathBuf, + #[serde(default)] + pub iommu: bool, +} + +#[cfg(target_arch = "x86_64")] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct SgxEpcConfig { + pub id: String, + #[serde(default)] + pub size: u64, + #[serde(default)] + pub prefault: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct UserDeviceConfig { + pub socket: PathBuf, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub pci_segment: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct VdpaConfig { + pub path: PathBuf, + //#[serde(default = "default_vdpaconfig_num_queues")] + pub num_queues: usize, + #[serde(default)] + pub iommu: bool, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub pci_segment: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub enum VhostMode { + #[default] + Client, + Server, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct VmConfig { + #[serde(default)] + pub cpus: CpusConfig, + #[serde(default)] + pub memory: MemoryConfig, + #[serde(skip_serializing_if = "Option::is_none")] + pub kernel: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub initramfs: Option, + #[serde(default)] + #[serde(skip_serializing_if = "CmdlineConfig::is_empty")] + pub cmdline: CmdlineConfig, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub payload: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disks: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub net: Option>, + #[serde(default)] + pub rng: RngConfig, + #[serde(skip_serializing_if = "Option::is_none")] + pub balloon: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub fs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub pmem: Option>, + //#[serde(default = "ConsoleConfig::default_serial")] + pub serial: ConsoleConfig, + //#[serde(default = "ConsoleConfig::default_console")] + pub console: ConsoleConfig, + #[serde(skip_serializing_if = "Option::is_none")] + pub devices: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_devices: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub vdpa: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub vsock: Option, + #[serde(default)] + pub iommu: bool, + #[cfg(target_arch = "x86_64")] + #[serde(skip_serializing_if = "Option::is_none")] + pub sgx_epc: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub numa: Option>, + #[serde(default)] + pub watchdog: bool, + #[cfg(feature = "guest_debug")] + pub gdb: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub platform: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct VsockConfig { + pub cid: u64, + pub socket: PathBuf, + #[serde(default)] + pub iommu: bool, + #[serde(default)] + pub id: Option, + #[serde(default)] + pub pci_segment: u16, +} + +//-------------------------------------------------------------------- +// For serde serialization + +#[allow(clippy::trivially_copy_pass_by_ref)] +fn u8_is_zero(v: &u8) -> bool { + *v == 0 +} + +#[allow(clippy::trivially_copy_pass_by_ref)] +fn usize_is_zero(v: &usize) -> bool { + *v == 0 +} + +#[allow(clippy::trivially_copy_pass_by_ref)] +fn u16_is_zero(v: &u16) -> bool { + *v == 0 +} diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/net_util.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/net_util.rs new file mode 100644 index 0000000000..00a0794628 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/net_util.rs @@ -0,0 +1,32 @@ +// Copyright (c) 2022-2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize, Serializer}; +use std::fmt; + +pub const MAC_ADDR_LEN: usize = 6; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Default)] +pub struct MacAddr { + pub bytes: [u8; MAC_ADDR_LEN], +} + +// Note: Implements ToString automatically. +impl fmt::Display for MacAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let b = &self.bytes; + write!( + f, + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + b[0], b[1], b[2], b[3], b[4], b[5] + ) + } +} + +// Requried to remove the `bytes` member from the serialized JSON! +impl Serialize for MacAddr { + fn serialize(&self, serializer: S) -> Result { + self.to_string().serialize(serializer) + } +} diff --git a/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs b/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs new file mode 100644 index 0000000000..02bf04bf96 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/ch-config/src/virtio_devices.rs @@ -0,0 +1,19 @@ +// Copyright (c) 2022-2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct TokenBucketConfig { + pub size: u64, + pub one_time_burst: Option, + pub refill_time: u64, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RateLimiterConfig { + pub bandwidth: Option, + pub ops: Option, +} diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs new file mode 100644 index 0000000000..7f65ac11b5 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner.rs @@ -0,0 +1,148 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2022 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use super::HypervisorState; +use crate::device::Device; +use crate::VmmState; +use anyhow::Result; +use async_trait::async_trait; +use kata_types::capabilities::{Capabilities, CapabilityBits}; +use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; +use kata_types::config::hypervisor::HYPERVISOR_NAME_CH; +use persist::sandbox_persist::Persist; +use std::os::unix::net::UnixStream; +use tokio::process::Child; +use tokio::sync::watch::{channel, Receiver, Sender}; +use tokio::task::JoinHandle; + +#[derive(Debug)] +pub struct CloudHypervisorInner { + pub(crate) state: VmmState, + pub(crate) id: String, + + pub(crate) api_socket: Option, + pub(crate) extra_args: Option>, + + pub(crate) config: Option, + + pub(crate) process: Option, + pub(crate) pid: Option, + + pub(crate) timeout_secs: i32, + + pub(crate) netns: Option, + + // Sandbox-specific directory + pub(crate) vm_path: String, + + // Hypervisor runtime directory + pub(crate) run_dir: String, + + // Subdirectory of vm_path. + pub(crate) jailer_root: String, + + /// List of devices that will be added to the VM once it boots + pub(crate) pending_devices: Option>, + + pub(crate) _capabilities: Capabilities, + + pub(crate) shutdown_tx: Option>, + pub(crate) shutdown_rx: Option>, + pub(crate) tasks: Option>>>, +} + +unsafe impl Send for CloudHypervisorInner {} +unsafe impl Sync for CloudHypervisorInner {} + +const CH_DEFAULT_TIMEOUT_SECS: u32 = 10; + +impl CloudHypervisorInner { + pub fn new() -> Self { + let mut capabilities = Capabilities::new(); + capabilities.set( + CapabilityBits::BlockDeviceSupport + | CapabilityBits::BlockDeviceHotplugSupport + | CapabilityBits::FsSharingSupport, + ); + + let (tx, rx) = channel(true); + + Self { + api_socket: None, + extra_args: None, + + process: None, + pid: None, + + config: None, + state: VmmState::NotReady, + timeout_secs: CH_DEFAULT_TIMEOUT_SECS as i32, + id: String::default(), + jailer_root: String::default(), + vm_path: String::default(), + run_dir: String::default(), + netns: None, + pending_devices: None, + _capabilities: capabilities, + shutdown_tx: Some(tx), + shutdown_rx: Some(rx), + tasks: None, + } + } + + pub fn set_hypervisor_config(&mut self, config: HypervisorConfig) { + self.config = Some(config); + } + + pub fn hypervisor_config(&self) -> HypervisorConfig { + self.config.clone().unwrap_or_default() + } +} + +impl Default for CloudHypervisorInner { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Persist for CloudHypervisorInner { + type State = HypervisorState; + type ConstructorArgs = (); + + // Return a state object that will be saved by the caller. + async fn save(&self) -> Result { + Ok(HypervisorState { + hypervisor_type: HYPERVISOR_NAME_CH.to_string(), + id: self.id.clone(), + vm_path: self.vm_path.clone(), + jailed: false, + jailer_root: String::default(), + netns: None, + config: self.hypervisor_config(), + run_dir: self.run_dir.clone(), + cached_block_devices: Default::default(), + ..Default::default() + }) + } + + // Set the hypervisor state to the specified state + async fn restore( + _hypervisor_args: Self::ConstructorArgs, + hypervisor_state: Self::State, + ) -> Result { + let ch = Self { + config: Some(hypervisor_state.config), + state: VmmState::NotReady, + id: hypervisor_state.id, + vm_path: hypervisor_state.vm_path, + run_dir: hypervisor_state.run_dir, + + ..Default::default() + }; + + Ok(ch) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs new file mode 100644 index 0000000000..03cf95daf8 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_device.rs @@ -0,0 +1,235 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2019-2022 Ant Group +// Copyright (c) 2022 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use super::inner::CloudHypervisorInner; +use crate::device::{Device, ShareFsDeviceConfig}; +use crate::HybridVsockConfig; +use crate::VmmState; +use anyhow::{anyhow, Context, Result}; +use ch_config::ch_api::cloud_hypervisor_vm_fs_add; +use ch_config::{FsConfig, PmemConfig}; +use safe_path::scoped_join; +use std::convert::TryFrom; +use std::path::PathBuf; + +const VIRTIO_FS: &str = "virtio-fs"; + +impl CloudHypervisorInner { + pub(crate) async fn add_device(&mut self, device: Device) -> Result<()> { + if self.state != VmmState::VmRunning { + let mut devices: Vec = if let Some(devices) = self.pending_devices.take() { + devices + } else { + vec![] + }; + + devices.insert(0, device); + + self.pending_devices = Some(devices); + + return Ok(()); + } + + self.handle_add_device(device).await?; + + Ok(()) + } + + async fn handle_add_device(&mut self, device: Device) -> Result<()> { + match device { + Device::ShareFsDevice(cfg) => self.handle_share_fs_device(cfg).await, + Device::HybridVsock(cfg) => self.handle_hvsock_device(&cfg).await, + _ => return Err(anyhow!("unhandled device: {:?}", device)), + } + } + + /// Add the device that were requested to be added before the VMM was + /// started. + #[allow(dead_code)] + pub(crate) async fn handle_pending_devices_after_boot(&mut self) -> Result<()> { + if self.state != VmmState::VmRunning { + return Err(anyhow!( + "cannot handle pending devices with VMM state {:?}", + self.state + )); + } + + if let Some(mut devices) = self.pending_devices.take() { + while let Some(dev) = devices.pop() { + self.add_device(dev).await.context("add_device")?; + } + } + + Ok(()) + } + + pub(crate) async fn remove_device(&mut self, _device: Device) -> Result<()> { + Ok(()) + } + + async fn handle_share_fs_device(&mut self, cfg: ShareFsDeviceConfig) -> Result<()> { + if cfg.fs_type != VIRTIO_FS { + return Err(anyhow!("cannot handle share fs type: {:?}", cfg.fs_type)); + } + + let socket = self + .api_socket + .as_ref() + .ok_or("missing socket") + .map_err(|e| anyhow!(e))?; + + let num_queues: usize = if cfg.queue_num > 0 { + cfg.queue_num as usize + } else { + 1 + }; + + let queue_size: u16 = if cfg.queue_num > 0 { + u16::try_from(cfg.queue_size)? + } else { + 1024 + }; + + let socket_path = if cfg.sock_path.starts_with('/') { + PathBuf::from(cfg.sock_path) + } else { + scoped_join(&self.vm_path, cfg.sock_path)? + }; + + let fs_config = FsConfig { + tag: cfg.mount_tag, + socket: socket_path, + num_queues, + queue_size, + ..Default::default() + }; + + let response = cloud_hypervisor_vm_fs_add( + socket.try_clone().context("failed to clone socket")?, + fs_config, + ) + .await?; + + if let Some(detail) = response { + debug!(sl!(), "fs add response: {:?}", detail); + } + + Ok(()) + } + + async fn handle_hvsock_device(&mut self, _cfg: &HybridVsockConfig) -> Result<()> { + Ok(()) + } + + pub(crate) async fn get_shared_fs_devices(&mut self) -> Result>> { + let pending_root_devices = self.pending_devices.take(); + + let mut root_devices = Vec::::new(); + + if let Some(devices) = pending_root_devices { + for dev in devices { + match dev { + Device::ShareFsDevice(dev) => { + let settings = ShareFsSettings::new(dev, self.vm_path.clone()); + + let fs_cfg = FsConfig::try_from(settings)?; + + root_devices.push(fs_cfg); + } + _ => continue, + }; + } + + Ok(Some(root_devices)) + } else { + Ok(None) + } + } + + pub(crate) async fn get_boot_file(&mut self) -> Result { + if let Some(ref config) = self.config { + let boot_info = &config.boot_info; + + let file = if !boot_info.initrd.is_empty() { + boot_info.initrd.clone() + } else if !boot_info.image.is_empty() { + boot_info.image.clone() + } else { + return Err(anyhow!("missing boot file (no image or initrd)")); + }; + + Ok(PathBuf::from(file)) + } else { + Err(anyhow!("no hypervisor config")) + } + } + + pub(crate) async fn get_pmem_devices(&mut self) -> Result>> { + let file = self.get_boot_file().await?; + + let pmem_cfg = PmemConfig { + file, + size: None, + iommu: false, + discard_writes: true, + id: None, + pci_segment: 0, + }; + + let pmem_devices = vec![pmem_cfg]; + + Ok(Some(pmem_devices)) + } +} + +#[derive(Debug)] +pub struct ShareFsSettings { + cfg: ShareFsDeviceConfig, + vm_path: String, +} + +impl ShareFsSettings { + pub fn new(cfg: ShareFsDeviceConfig, vm_path: String) -> Self { + ShareFsSettings { cfg, vm_path } + } +} + +impl TryFrom for FsConfig { + type Error = anyhow::Error; + + fn try_from(settings: ShareFsSettings) -> Result { + let cfg = settings.cfg; + let vm_path = settings.vm_path; + + let num_queues: usize = if cfg.queue_num > 0 { + cfg.queue_num as usize + } else { + 1 + }; + + let queue_size: u16 = if cfg.queue_num > 0 { + u16::try_from(cfg.queue_size)? + } else { + 1024 + }; + + let socket_path = if cfg.sock_path.starts_with('/') { + PathBuf::from(cfg.sock_path) + } else { + PathBuf::from(vm_path).join(cfg.sock_path) + }; + + let fs_cfg = FsConfig { + tag: cfg.mount_tag, + socket: socket_path, + num_queues, + queue_size, + ..Default::default() + }; + + Ok(fs_cfg) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs new file mode 100644 index 0000000000..b3271ee791 --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/ch/inner_hypervisor.rs @@ -0,0 +1,486 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2022 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use super::inner::CloudHypervisorInner; +use crate::ch::utils::get_api_socket_path; +use crate::ch::utils::{get_jailer_root, get_sandbox_path, get_vsock_path}; +use crate::Device; +use crate::VsockConfig; +use crate::{VcpuThreadIds, VmmState}; +use anyhow::{anyhow, Context, Result}; +use ch_config::ch_api::{ + cloud_hypervisor_vm_create, cloud_hypervisor_vm_start, cloud_hypervisor_vmm_ping, + cloud_hypervisor_vmm_shutdown, +}; +use core::future::poll_fn; +use futures::executor::block_on; +use futures::future::join_all; +use kata_types::capabilities::{Capabilities, CapabilityBits}; +use std::fs::create_dir_all; +use std::os::unix::net::UnixStream; +use std::path::Path; +use std::process::Stdio; +use tokio::io::AsyncBufReadExt; +use tokio::io::BufReader; +use tokio::process::{Child, Command}; +use tokio::sync::watch::Receiver; +use tokio::task; +use tokio::task::JoinHandle; +use tokio::time::Duration; + +const CH_NAME: &str = "cloud-hypervisor"; + +/// Number of milliseconds to wait before retrying a CH operation. +const CH_POLL_TIME_MS: u64 = 50; + +impl CloudHypervisorInner { + async fn start_hypervisor(&mut self, timeout_secs: i32) -> Result<()> { + self.cloud_hypervisor_launch(timeout_secs) + .await + .context("launch failed")?; + + self.cloud_hypervisor_setup_comms() + .await + .context("comms setup failed")?; + + self.cloud_hypervisor_check_running() + .await + .context("hypervisor running check failed")?; + + self.state = VmmState::VmmServerReady; + + Ok(()) + } + + async fn boot_vm(&mut self) -> Result<()> { + let shared_fs_devices = self.get_shared_fs_devices().await?; + + let pmem_devices = self.get_pmem_devices().await?; + + let socket = self + .api_socket + .as_ref() + .ok_or("missing socket") + .map_err(|e| anyhow!(e))?; + + let sandbox_path = get_sandbox_path(&self.id)?; + + std::fs::create_dir_all(sandbox_path.clone()).context("failed to create sandbox path")?; + + let vsock_socket_path = get_vsock_path(&self.id)?; + + let response = cloud_hypervisor_vm_create( + sandbox_path, + vsock_socket_path, + socket.try_clone().context("failed to clone socket")?, + shared_fs_devices, + pmem_devices, + ) + .await?; + + if let Some(detail) = response { + debug!(sl!(), "vm boot response: {:?}", detail); + } + + let response = + cloud_hypervisor_vm_start(socket.try_clone().context("failed to clone socket")?) + .await?; + + if let Some(detail) = response { + debug!(sl!(), "vm start response: {:?}", detail); + } + + self.state = VmmState::VmRunning; + + Ok(()) + } + + async fn cloud_hypervisor_setup_comms(&mut self) -> Result<()> { + let api_socket_path = get_api_socket_path(&self.id)?; + + // The hypervisor has just been spawned, but may not yet have created + // the API socket, so repeatedly try to connect for up to + // timeout_secs. + let join_handle: JoinHandle> = + task::spawn_blocking(move || -> Result { + let api_socket: UnixStream; + + loop { + let result = UnixStream::connect(api_socket_path.clone()); + + if let Ok(result) = result { + api_socket = result; + break; + } + + std::thread::sleep(Duration::from_millis(CH_POLL_TIME_MS)); + } + + Ok(api_socket) + }); + + let timeout_msg = format!( + "API socket connect timed out after {} seconds", + self.timeout_secs + ); + + let result = + tokio::time::timeout(Duration::from_secs(self.timeout_secs as u64), join_handle) + .await + .context(timeout_msg)?; + + let result = result?; + + let api_socket = result?; + + self.api_socket = Some(api_socket); + + Ok(()) + } + + async fn cloud_hypervisor_check_running(&mut self) -> Result<()> { + let timeout_secs = self.timeout_secs; + + let timeout_msg = format!( + "API socket connect timed out after {} seconds", + timeout_secs + ); + + let join_handle = self.cloud_hypervisor_ping_until_ready(CH_POLL_TIME_MS); + + let result = tokio::time::timeout(Duration::new(timeout_secs as u64, 0), join_handle) + .await + .context(timeout_msg)?; + + result + } + + async fn cloud_hypervisor_ensure_not_launched(&self) -> Result<()> { + if let Some(child) = &self.process { + return Err(anyhow!( + "{} already running with PID {}", + CH_NAME, + child.id().unwrap_or(0) + )); + } + + Ok(()) + } + + async fn cloud_hypervisor_launch(&mut self, _timeout_secs: i32) -> Result<()> { + self.cloud_hypervisor_ensure_not_launched().await?; + + let debug = false; + + let disable_seccomp = true; + + let api_socket_path = get_api_socket_path(&self.id)?; + + let _ = std::fs::remove_file(api_socket_path.clone()); + + let binary_path = self + .config + .as_ref() + .ok_or("no hypervisor config for CH") + .map_err(|e| anyhow!(e))? + .path + .to_string(); + + let path = Path::new(&binary_path).canonicalize()?; + + let mut cmd = Command::new(path); + + cmd.current_dir("/"); + + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + cmd.env("RUST_BACKTRACE", "full"); + + cmd.args(["--api-socket", &api_socket_path]); + + if let Some(extra_args) = &self.extra_args { + cmd.args(extra_args); + } + + if debug { + cmd.arg("-v"); + } + + if disable_seccomp { + cmd.args(["--seccomp", "false"]); + } + + let child = cmd.spawn().context(format!("{} spawn failed", CH_NAME))?; + + // Save process PID + self.pid = child.id(); + + let shutdown = self + .shutdown_rx + .as_ref() + .ok_or("no receiver channel") + .map_err(|e| anyhow!(e))? + .clone(); + + let ch_outputlogger_task = tokio::spawn(cloud_hypervisor_log_output(child, shutdown)); + + let tasks = vec![ch_outputlogger_task]; + + self.tasks = Some(tasks); + + Ok(()) + } + + async fn cloud_hypervisor_shutdown(&mut self) -> Result<()> { + let socket = self + .api_socket + .as_ref() + .ok_or("missing socket") + .map_err(|e| anyhow!(e))?; + + let response = + cloud_hypervisor_vmm_shutdown(socket.try_clone().context("shutdown failed")?).await?; + + if let Some(detail) = response { + debug!(sl!(), "shutdown response: {:?}", detail); + } + + // Trigger a controlled shutdown + self.shutdown_tx + .as_mut() + .ok_or("no shutdown channel") + .map_err(|e| anyhow!(e))? + .send(true) + .map_err(|e| anyhow!(e).context("failed to request shutdown"))?; + + let tasks = self + .tasks + .take() + .ok_or("no tasks") + .map_err(|e| anyhow!(e))?; + + let results = join_all(tasks).await; + + let mut wait_errors: Vec = vec![]; + + for result in results { + if let Err(e) = result { + eprintln!("wait task error: {:#?}", e); + + wait_errors.push(e); + } + } + + if wait_errors.is_empty() { + Ok(()) + } else { + Err(anyhow!("wait all tasks failed: {:#?}", wait_errors)) + } + } + + #[allow(dead_code)] + async fn cloud_hypervisor_wait(&mut self) -> Result<()> { + let mut child = self + .process + .take() + .ok_or(format!("{} not running", CH_NAME)) + .map_err(|e| anyhow!(e))?; + + let _pid = child + .id() + .ok_or(format!("{} missing PID", CH_NAME)) + .map_err(|e| anyhow!(e))?; + + // Note that this kills _and_ waits for the process! + child.kill().await?; + + Ok(()) + } + + async fn cloud_hypervisor_ping_until_ready(&mut self, _poll_time_ms: u64) -> Result<()> { + let socket = self + .api_socket + .as_ref() + .ok_or("missing socket") + .map_err(|e| anyhow!(e))?; + + loop { + let response = + cloud_hypervisor_vmm_ping(socket.try_clone().context("failed to clone socket")?) + .await + .context("ping failed"); + + if let Ok(response) = response { + if let Some(detail) = response { + debug!(sl!(), "ping response: {:?}", detail); + } + break; + } + + tokio::time::sleep(Duration::from_millis(CH_POLL_TIME_MS)).await; + } + + Ok(()) + } + + pub(crate) async fn prepare_vm(&mut self, id: &str, netns: Option) -> Result<()> { + self.id = id.to_string(); + self.state = VmmState::NotReady; + + self.setup_environment().await?; + + self.netns = netns; + + let vsock_cfg = VsockConfig::new(self.id.clone()).await?; + + let dev = Device::Vsock(vsock_cfg); + self.add_device(dev).await.context("add vsock device")?; + + self.start_hypervisor(self.timeout_secs).await?; + + Ok(()) + } + + async fn setup_environment(&mut self) -> Result<()> { + // run_dir and vm_path are the same (shared) + self.run_dir = get_sandbox_path(&self.id)?; + self.vm_path = self.run_dir.to_string(); + + create_dir_all(&self.run_dir) + .with_context(|| anyhow!("failed to create sandbox directory {}", self.run_dir))?; + + if !self.jailer_root.is_empty() { + create_dir_all(self.jailer_root.as_str()) + .map_err(|e| anyhow!("Failed to create dir {} err : {:?}", self.jailer_root, e))?; + } + + Ok(()) + } + + pub(crate) async fn start_vm(&mut self, timeout_secs: i32) -> Result<()> { + self.setup_environment().await?; + + self.timeout_secs = timeout_secs; + + self.boot_vm().await?; + + Ok(()) + } + + pub(crate) fn stop_vm(&mut self) -> Result<()> { + block_on(self.cloud_hypervisor_shutdown())?; + + Ok(()) + } + + pub(crate) fn pause_vm(&self) -> Result<()> { + Ok(()) + } + + pub(crate) fn resume_vm(&self) -> Result<()> { + Ok(()) + } + + pub(crate) async fn save_vm(&self) -> Result<()> { + Ok(()) + } + + pub(crate) async fn get_agent_socket(&self) -> Result { + const HYBRID_VSOCK_SCHEME: &str = "hvsock"; + + let vsock_path = get_vsock_path(&self.id)?; + + let uri = format!("{}://{}", HYBRID_VSOCK_SCHEME, vsock_path); + + Ok(uri) + } + + pub(crate) async fn disconnect(&mut self) { + self.state = VmmState::NotReady; + } + + pub(crate) async fn get_thread_ids(&self) -> Result { + Ok(VcpuThreadIds::default()) + } + + pub(crate) async fn cleanup(&self) -> Result<()> { + Ok(()) + } + + pub(crate) async fn get_pids(&self) -> Result> { + Ok(Vec::::new()) + } + + pub(crate) async fn check(&self) -> Result<()> { + Ok(()) + } + + pub(crate) async fn get_jailer_root(&self) -> Result { + let root_path = get_jailer_root(&self.id)?; + + std::fs::create_dir_all(&root_path)?; + + Ok(root_path) + } + + pub(crate) async fn capabilities(&self) -> Result { + let mut caps = Capabilities::default(); + caps.set(CapabilityBits::FsSharingSupport); + Ok(caps) + } +} + +// Log all output from the CH process until a shutdown signal is received. +// When that happens, stop logging and wait for the child process to finish +// before returning. +async fn cloud_hypervisor_log_output(mut child: Child, mut shutdown: Receiver) -> Result<()> { + let stdout = child + .stdout + .as_mut() + .ok_or("failed to get child stdout") + .map_err(|e| anyhow!(e))?; + + let stdout_reader = BufReader::new(stdout); + let mut stdout_lines = stdout_reader.lines(); + + let stderr = child + .stderr + .as_mut() + .ok_or("failed to get child stderr") + .map_err(|e| anyhow!(e))?; + + let stderr_reader = BufReader::new(stderr); + let mut stderr_lines = stderr_reader.lines(); + + loop { + tokio::select! { + _ = shutdown.changed() => { + info!(sl!(), "got shutdown request"); + break; + }, + stderr_line = poll_fn(|cx| Pin::new(&mut stderr_lines).poll_next_line(cx)) => { + if let Ok(line) = stderr_line { + let line = line.ok_or("missing stderr line").map_err(|e| anyhow!(e))?; + + info!(sl!(), "{:?}", line; "stream" => "stderr"); + } + }, + stdout_line = poll_fn(|cx| Pin::new(&mut stdout_lines).poll_next_line(cx)) => { + if let Ok(line) = stdout_line { + let line = line.ok_or("missing stdout line").map_err(|e| anyhow!(e))?; + + info!(sl!(), "{:?}", line; "stream" => "stdout"); + } + }, + }; + } + + // Note that this kills _and_ waits for the process! + child.kill().await?; + + Ok(()) +} diff --git a/src/runtime-rs/crates/hypervisor/src/ch/mod.rs b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs new file mode 100644 index 0000000000..d589c18dfe --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/ch/mod.rs @@ -0,0 +1,163 @@ +// Copyright (c) 2019-2022 Alibaba Cloud +// Copyright (c) 2022 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use super::HypervisorState; +use crate::{device::Device, Hypervisor, VcpuThreadIds}; +use anyhow::{Context, Result}; +use async_trait::async_trait; +use kata_types::capabilities::Capabilities; +use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; +use persist::sandbox_persist::Persist; +use std::sync::Arc; +use tokio::sync::RwLock; + +// Convenience macro to obtain the scope logger +#[macro_export] +macro_rules! sl { + () => { + slog_scope::logger().new(o!("subsystem" => "cloud-hypervisor")) + }; + } + +mod inner; +mod inner_device; +mod inner_hypervisor; +mod utils; + +use inner::CloudHypervisorInner; + +#[derive(Debug, Default, Clone)] +pub struct CloudHypervisor { + inner: Arc>, +} + +unsafe impl Send for CloudHypervisor {} +unsafe impl Sync for CloudHypervisor {} + +impl CloudHypervisor { + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(CloudHypervisorInner::new())), + } + } + + pub async fn set_hypervisor_config(&mut self, config: HypervisorConfig) { + let mut inner = self.inner.write().await; + inner.set_hypervisor_config(config) + } +} + +#[async_trait] +impl Hypervisor for CloudHypervisor { + async fn prepare_vm(&self, id: &str, netns: Option) -> Result<()> { + let mut inner = self.inner.write().await; + inner.prepare_vm(id, netns).await + } + + async fn start_vm(&self, timeout: i32) -> Result<()> { + let mut inner = self.inner.write().await; + inner.start_vm(timeout).await + } + + async fn stop_vm(&self) -> Result<()> { + let mut inner = self.inner.write().await; + inner.stop_vm() + } + + async fn pause_vm(&self) -> Result<()> { + let inner = self.inner.write().await; + inner.pause_vm() + } + + async fn resume_vm(&self) -> Result<()> { + let inner = self.inner.write().await; + inner.resume_vm() + } + + async fn save_vm(&self) -> Result<()> { + let inner = self.inner.write().await; + inner.save_vm().await + } + + async fn add_device(&self, device: Device) -> Result<()> { + let mut inner = self.inner.write().await; + inner.add_device(device).await + } + + async fn remove_device(&self, device: Device) -> Result<()> { + let mut inner = self.inner.write().await; + inner.remove_device(device).await + } + + async fn get_agent_socket(&self) -> Result { + let inner = self.inner.write().await; + inner.get_agent_socket().await + } + + async fn disconnect(&self) { + let mut inner = self.inner.write().await; + inner.disconnect().await + } + + async fn hypervisor_config(&self) -> HypervisorConfig { + let inner = self.inner.write().await; + inner.hypervisor_config() + } + + async fn get_thread_ids(&self) -> Result { + let inner = self.inner.read().await; + inner.get_thread_ids().await + } + + async fn cleanup(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.cleanup().await + } + + async fn get_pids(&self) -> Result> { + let inner = self.inner.read().await; + inner.get_pids().await + } + + async fn check(&self) -> Result<()> { + let inner = self.inner.read().await; + inner.check().await + } + + async fn get_jailer_root(&self) -> Result { + let inner = self.inner.read().await; + inner.get_jailer_root().await + } + + async fn save_state(&self) -> Result { + self.save().await + } + + async fn capabilities(&self) -> Result { + let inner = self.inner.read().await; + inner.capabilities().await + } +} + +#[async_trait] +impl Persist for CloudHypervisor { + type State = HypervisorState; + type ConstructorArgs = (); + + async fn save(&self) -> Result { + let inner = self.inner.read().await; + inner.save().await.context("save CH hypervisor state") + } + + async fn restore( + hypervisor_args: Self::ConstructorArgs, + hypervisor_state: Self::State, + ) -> Result { + let inner = CloudHypervisorInner::restore(hypervisor_args, hypervisor_state).await?; + Ok(Self { + inner: Arc::new(RwLock::new(inner)), + }) + } +} diff --git a/src/runtime-rs/crates/hypervisor/src/ch/utils.rs b/src/runtime-rs/crates/hypervisor/src/ch/utils.rs new file mode 100644 index 0000000000..dcea26aebe --- /dev/null +++ b/src/runtime-rs/crates/hypervisor/src/ch/utils.rs @@ -0,0 +1,53 @@ +// Copyright (c) 2022-2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use shim_interface::KATA_PATH; + +// The socket used to connect to CH. This is used for CH API communications. +const CH_API_SOCKET_NAME: &str = "ch-api.sock"; + +// The socket that allows runtime-rs to connect direct through to the Kata +// Containers agent running inside the CH hosted VM. +const CH_VM_SOCKET_NAME: &str = "ch-vm.sock"; + +const CH_JAILER_DIR: &str = "root"; + +// Return the path for a _hypothetical_ sandbox: the path does *not* exist +// yet, and for this reason safe-path cannot be used. +pub fn get_sandbox_path(id: &str) -> Result { + let path = [KATA_PATH, id].join("/"); + + Ok(path) +} + +// Return the path for a _hypothetical_ API socket path: +// the path does *not* exist yet, and for this reason safe-path cannot be +// used. +pub fn get_api_socket_path(id: &str) -> Result { + let sandbox_path = get_sandbox_path(id)?; + + let path = [&sandbox_path, CH_API_SOCKET_NAME].join("/"); + + Ok(path) +} + +// Return the path for a _hypothetical_ sandbox specific VSOCK socket path: +// the path does *not* exist yet, and for this reason safe-path cannot be +// used. +pub fn get_vsock_path(id: &str) -> Result { + let sandbox_path = get_sandbox_path(id)?; + + let path = [&sandbox_path, CH_VM_SOCKET_NAME].join("/"); + + Ok(path) +} + +pub fn get_jailer_root(id: &str) -> Result { + let sandbox_path = get_sandbox_path(id)?; + + let path = [&sandbox_path, CH_JAILER_DIR].join("/"); + + Ok(path) +} diff --git a/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs b/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs index a04fd24b09..ea870f3420 100644 --- a/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs +++ b/src/runtime-rs/crates/hypervisor/src/hypervisor_persist.rs @@ -8,7 +8,7 @@ use crate::HypervisorConfig; use serde::{Deserialize, Serialize}; use std::collections::HashSet; -#[derive(Serialize, Deserialize, Default)] +#[derive(Serialize, Deserialize, Default, Clone, Debug)] pub struct HypervisorState { // Type of hypervisor, E.g. dragonball/qemu/firecracker/acrn. pub hypervisor_type: String, diff --git a/src/runtime-rs/crates/hypervisor/src/lib.rs b/src/runtime-rs/crates/hypervisor/src/lib.rs index f2bcc21c7b..3c417f195e 100644 --- a/src/runtime-rs/crates/hypervisor/src/lib.rs +++ b/src/runtime-rs/crates/hypervisor/src/lib.rs @@ -19,11 +19,17 @@ pub use kernel_param::Param; mod utils; use std::collections::HashMap; +#[cfg(feature = "cloud-hypervisor")] +pub mod ch; + use anyhow::Result; use async_trait::async_trait; use hypervisor_persist::HypervisorState; use kata_types::capabilities::Capabilities; use kata_types::config::hypervisor::Hypervisor as HypervisorConfig; + +pub use kata_types::config::hypervisor::HYPERVISOR_NAME_CH; + // Config which driver to use as vm root dev const VM_ROOTFS_DRIVER_BLK: &str = "virtio-blk"; const VM_ROOTFS_DRIVER_PMEM: &str = "virtio-pmem"; @@ -48,7 +54,7 @@ const SHMEM: &str = "shmem"; pub const HYPERVISOR_DRAGONBALL: &str = "dragonball"; pub const HYPERVISOR_QEMU: &str = "qemu"; -#[derive(PartialEq)] +#[derive(PartialEq, Debug, Clone)] pub(crate) enum VmmState { NotReady, VmmServerReady, @@ -56,7 +62,7 @@ pub(crate) enum VmmState { } // vcpu mapping from vcpu number to thread number -#[derive(Debug)] +#[derive(Debug, Default)] pub struct VcpuThreadIds { pub vcpus: HashMap, } diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml index 0e3fbdc60a..6dea5e7628 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml @@ -35,3 +35,9 @@ oci = { path = "../../../../libs/oci" } persist = { path = "../../persist"} resource = { path = "../../resource" } +[features] +default = [] + +# Feature is not yet complete, so not enabled by default. +# See https://github.com/kata-containers/kata-containers/issues/6264. +cloud-hypervisor = [] diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs index 99489d7b30..e619188a90 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/lib.rs @@ -25,6 +25,12 @@ use hypervisor::{qemu::Qemu, HYPERVISOR_QEMU}; use kata_types::config::{ hypervisor::register_hypervisor_plugin, DragonballConfig, QemuConfig, TomlConfig, }; + +#[cfg(feature = "cloud-hypervisor")] +use hypervisor::ch::CloudHypervisor; +#[cfg(feature = "cloud-hypervisor")] +use kata_types::config::{hypervisor::HYPERVISOR_NAME_CH, CloudHypervisorConfig}; + use resource::ResourceManager; use sandbox::VIRTCONTAINER; use tokio::sync::mpsc::Sender; @@ -39,8 +45,16 @@ impl RuntimeHandler for VirtContainer { // register let dragonball_config = Arc::new(DragonballConfig::new()); register_hypervisor_plugin("dragonball", dragonball_config); + let qemu_config = Arc::new(QemuConfig::new()); register_hypervisor_plugin("qemu", qemu_config); + + #[cfg(feature = "cloud-hypervisor")] + { + let ch_config = Arc::new(CloudHypervisorConfig::new()); + register_hypervisor_plugin(HYPERVISOR_NAME_CH, ch_config); + } + Ok(()) } @@ -118,6 +132,17 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result> .await; Ok(Arc::new(hypervisor)) } + + #[cfg(feature = "cloud-hypervisor")] + HYPERVISOR_NAME_CH => { + let mut hypervisor = CloudHypervisor::new(); + + hypervisor + .set_hypervisor_config(hypervisor_config.clone()) + .await; + + Ok(Arc::new(hypervisor)) + } _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)), } }