From 0398073c55ea09bf466faaad68efd61f26bba154 Mon Sep 17 00:00:00 2001 From: Sumedh Alok Sharma Date: Thu, 10 Jul 2025 17:43:14 +0000 Subject: [PATCH] agent-ctl: Add option --vm to boot pod VM for testing. This change introduces a new command line option `--vm` to boot up a pod VM for testing. The tool connects with kata agent running inside the VM to send the test commands. The tool uses `hypervisor` crates from runtime-rs for VM lifecycle management. Current implementation supports Qemu & Cloud Hypervisor as VMMs. In summary: - tool parses the VMM specific runtime-rs kata config file in /opt/kata/share/defaults/kata-containers/runtime-rs/* - prepares and starts a VM using runtime-rs::hypervisor vm APIs - retrieves agent's server address to setup connection - tests the requested commands & shutdown the VM Fixes #11566 Signed-off-by: Sumedh Alok Sharma --- src/tools/agent-ctl/Cargo.lock | 629 +++++++++++++++--- src/tools/agent-ctl/Cargo.toml | 7 +- src/tools/agent-ctl/src/client.rs | 189 ++++-- src/tools/agent-ctl/src/main.rs | 31 +- src/tools/agent-ctl/src/rpc.rs | 2 +- src/tools/agent-ctl/src/types.rs | 1 + src/tools/agent-ctl/src/vm/mod.rs | 56 ++ src/tools/agent-ctl/src/vm/vm_ops.rs | 166 +++++ src/tools/agent-ctl/src/vm/vm_utils.rs | 53 ++ .../api-tests/test_vm_GuestDetails.bats | 34 + 10 files changed, 1022 insertions(+), 146 deletions(-) create mode 100644 src/tools/agent-ctl/src/vm/mod.rs create mode 100644 src/tools/agent-ctl/src/vm/vm_ops.rs create mode 100644 src/tools/agent-ctl/src/vm/vm_utils.rs create mode 100755 tests/functional/kata-agent-apis/api-tests/test_vm_GuestDetails.bats diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index 81dfdc08c1..b07a43d5f7 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -2,6 +2,27 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn 2.0.87", +] + +[[package]] +name = "actix-rt" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" +dependencies = [ + "actix-macros", + "futures-core", + "tokio", +] + [[package]] name = "addr2line" version = "0.22.0" @@ -39,7 +60,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "cipher", "cpufeatures", "zeroize", @@ -150,6 +171,14 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "api_client" +version = "0.1.0" +source = "git+https://github.com/cloud-hypervisor/cloud-hypervisor?tag=v27.0#2ba6a9bfcfd79629aecf77504fa554ab821d138e" +dependencies = [ + "vmm-sys-util 0.10.0", +] + [[package]] name = "arc-swap" version = "1.6.0" @@ -271,7 +300,7 @@ checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ "async-lock 2.8.0", "autocfg", - "cfg-if", + "cfg-if 1.0.1", "concurrent-queue", "futures-lite 1.13.0", "log", @@ -290,7 +319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1237c0ae75a0f3765f58910ff9cdd0a12eeb39ab2f4c7de23262f337f0aacbb3" dependencies = [ "async-lock 3.4.0", - "cfg-if", + "cfg-if 1.0.1", "concurrent-queue", "futures-io", "futures-lite 2.0.0", @@ -332,7 +361,7 @@ dependencies = [ "async-lock 2.8.0", "async-signal", "blocking", - "cfg-if", + "cfg-if 1.0.1", "event-listener 3.1.0", "futures-lite 1.13.0", "rustix 0.38.34", @@ -359,7 +388,7 @@ dependencies = [ "async-io 2.4.1", "async-lock 3.4.0", "atomic-waker", - "cfg-if", + "cfg-if 1.0.1", "futures-core", "futures-io", "rustix 1.0.7", @@ -474,8 +503,8 @@ dependencies = [ "axum-core", "bytes 1.7.2", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", "itoa", "matchit", @@ -500,8 +529,8 @@ dependencies = [ "async-trait", "bytes 1.7.2", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -519,7 +548,7 @@ checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.1", "libc", "miniz_oxide 0.7.3", "object", @@ -829,7 +858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbdc32a78afc325d71a48d13084f1c3ddf67cc5dc06c6e5439a8630b14612cad" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.1", "libc", ] @@ -890,6 +919,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.1" @@ -914,6 +949,21 @@ dependencies = [ "thiserror 1.0.40", ] +[[package]] +name = "ch-config" +version = "0.1.0" +dependencies = [ + "anyhow", + "api_client", + "kata-sys-util", + "kata-types", + "nix 0.26.4", + "serde", + "serde_json", + "thiserror 1.0.40", + "tokio", +] + [[package]] name = "chrono" version = "0.4.38" @@ -1089,7 +1139,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", ] [[package]] @@ -1098,7 +1148,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eb9105919ca8e40d437fc9cbb8f1975d916f1bd28afe795a48aae32a2cc8920" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -1121,7 +1171,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "crossbeam-epoch", "crossbeam-utils", ] @@ -1133,7 +1183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.1", "crossbeam-utils", ] @@ -1143,7 +1193,7 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "crossbeam-utils", ] @@ -1153,7 +1203,7 @@ version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", ] [[package]] @@ -1215,7 +1265,7 @@ version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "cpufeatures", "curve25519-dalek-derive", "digest", @@ -1314,6 +1364,20 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dbs-utils" +version = "0.2.1" +dependencies = [ + "anyhow", + "event-manager", + "libc", + "log", + "serde", + "thiserror 1.0.40", + "timerfd", + "vmm-sys-util 0.11.2", +] + [[package]] name = "der" version = "0.7.9" @@ -1428,7 +1492,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "dirs-sys-next", ] @@ -1454,6 +1518,12 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "downcast" version = "0.11.0" @@ -1681,6 +1751,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-manager" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "377fa591135fbe23396a18e2655a6d5481bf7c5823cdfa3cc81b01a229cbe640" +dependencies = [ + "libc", + "vmm-sys-util 0.14.0", +] + [[package]] name = "fail" version = "0.5.1" @@ -1729,7 +1809,7 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "libc", "libredox", "windows-sys 0.59.0", @@ -1933,7 +2013,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "js-sys", "libc", "wasi", @@ -1974,6 +2054,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "go-flag" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4a40c9ca507513f573aabaf6a8558173a1ac9aa1363d8de30c7f89b34f8d2b" +dependencies = [ + "cfg-if 0.1.10", +] + [[package]] name = "group" version = "0.13.0" @@ -1996,7 +2085,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.1.0", "indexmap 2.6.0", "slab", "tokio", @@ -2088,6 +2177,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes 1.7.2", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -2108,6 +2208,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes 1.7.2", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -2115,7 +2226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes 1.7.2", - "http", + "http 1.1.0", ] [[package]] @@ -2126,8 +2237,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes 1.7.2", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2149,6 +2260,29 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes 1.7.2", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.6.0" @@ -2159,8 +2293,8 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2177,8 +2311,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.6.0", "hyper-util", "rustls", "rustls-native-certs", @@ -2195,7 +2329,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper", + "hyper 1.6.0", "hyper-util", "pin-project-lite", "tokio", @@ -2212,9 +2346,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", - "hyper", + "http 1.1.0", + "http-body 1.0.1", + "hyper 1.6.0", "libc", "pin-project-lite", "socket2 0.5.10", @@ -2223,6 +2357,65 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex", + "hyper 0.14.32", + "pin-project", + "tokio", +] + +[[package]] +name = "hypervisor" +version = "0.1.0" +dependencies = [ + "actix-rt", + "anyhow", + "async-trait", + "ch-config", + "crossbeam-channel", + "dbs-utils", + "futures", + "go-flag", + "hyper 0.14.32", + "hyperlocal", + "kata-sys-util", + "kata-types", + "lazy_static", + "libc", + "logging", + "nix 0.26.4", + "oci-spec", + "path-clean", + "persist", + "protobuf 3.7.2", + "protocols", + "qapi", + "qapi-qmp", + "qapi-spec", + "rand", + "rust-ini", + "safe-path 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "seccompiler", + "serde", + "serde_json", + "shim-interface", + "slog", + "slog-scope", + "tests_utils", + "thiserror 1.0.40", + "tokio", + "tracing", + "ttrpc", + "ttrpc-codegen 0.4.2", + "vmm-sys-util 0.11.2", +] + [[package]] name = "iana-time-zone" version = "0.1.56" @@ -2378,7 +2571,7 @@ dependencies = [ "async-compression", "async-trait", "base64 0.22.1", - "cfg-if", + "cfg-if 1.0.1", "filetime", "flate2", "futures", @@ -2472,7 +2665,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", ] [[package]] @@ -2493,7 +2686,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" dependencies = [ "bitflags 2.6.0", - "cfg-if", + "cfg-if 1.0.1", "libc", ] @@ -2595,18 +2788,20 @@ dependencies = [ "clap", "hex", "humantime", + "hypervisor", "image-rs", + "kata-types", "lazy_static", "libc", "log", "logging", - "nix 0.23.2", + "nix 0.24.3", "oci-spec", - "protobuf", + "protobuf 3.7.2", "protocols", "rand", "rustjail", - "safe-path", + "safe-path 0.1.0", "serde", "serde_json", "slog", @@ -2635,7 +2830,7 @@ dependencies = [ "pci-ids", "rand", "runtime-spec", - "safe-path", + "safe-path 0.1.0", "serde", "serde_json", "slog", @@ -2659,7 +2854,7 @@ dependencies = [ "num_cpus", "oci-spec", "regex", - "safe-path", + "safe-path 0.1.0", "serde", "serde-enum-str", "serde_json", @@ -2737,7 +2932,6 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ - "cfg-if", "windows-targets 0.48.0", ] @@ -2845,7 +3039,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "digest", ] @@ -2946,7 +3140,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "downcast", "fragile", "mockall_derive", @@ -2960,7 +3154,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "proc-macro2", "quote", "syn 2.0.87", @@ -2986,7 +3180,19 @@ checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ "bitflags 1.3.2", "cc", - "cfg-if", + "cfg-if 1.0.1", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nix" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.1", "libc", "memoffset 0.6.5", ] @@ -2999,7 +3205,7 @@ checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.1", "libc", ] @@ -3010,7 +3216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.1", "libc", "memoffset 0.7.1", "pin-utils", @@ -3023,7 +3229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags 2.6.0", - "cfg-if", + "cfg-if 1.0.1", "cfg_aliases", "libc", ] @@ -3119,7 +3325,7 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom", - "http", + "http 1.1.0", "rand", "reqwest", "serde", @@ -3169,7 +3375,7 @@ dependencies = [ "bytes 1.7.2", "chrono", "futures-util", - "http", + "http 1.1.0", "http-auth", "jwt", "lazy_static", @@ -3210,7 +3416,7 @@ source = "git+https://github.com/confidential-containers/guest-components?rev=4c dependencies = [ "anyhow", "base64 0.22.1", - "cfg-if", + "cfg-if 1.0.1", "prost 0.13.5", "serde", "serde_json", @@ -3258,7 +3464,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac", - "http", + "http 1.1.0", "itertools 0.10.5", "log", "oauth2", @@ -3293,6 +3499,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -3363,7 +3579,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "libc", "redox_syscall 0.5.7", "smallvec", @@ -3391,6 +3607,12 @@ dependencies = [ "slash-formatter", ] +[[package]] +name = "path-clean" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" + [[package]] name = "path-dedot" version = "1.2.4" @@ -3454,6 +3676,21 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "persist" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "kata-sys-util", + "kata-types", + "libc", + "safe-path 0.1.0", + "serde", + "serde_json", + "shim-interface", +] + [[package]] name = "petgraph" version = "0.5.1" @@ -3607,7 +3844,7 @@ checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.1", "concurrent-queue", "libc", "log", @@ -3621,7 +3858,7 @@ version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b53a684391ad002dd6a596ceb6c74fd004fdce75f4be2e3f615068abbea5fd50" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "concurrent-queue", "hermit-abi 0.5.2", "pin-project-lite", @@ -3647,7 +3884,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "cpufeatures", "opaque-debug", "universal-hash", @@ -3884,6 +4121,12 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "protobuf" version = "3.7.2" @@ -3895,6 +4138,15 @@ dependencies = [ "thiserror 1.0.40", ] +[[package]] +name = "protobuf-codegen" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" +dependencies = [ + "protobuf 2.28.0", +] + [[package]] name = "protobuf-codegen" version = "3.7.2" @@ -3903,7 +4155,7 @@ checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace" dependencies = [ "anyhow", "once_cell", - "protobuf", + "protobuf 3.7.2", "protobuf-parse", "regex", "tempfile", @@ -3919,7 +4171,7 @@ dependencies = [ "anyhow", "indexmap 2.6.0", "log", - "protobuf", + "protobuf 3.7.2", "protobuf-support", "tempfile", "thiserror 1.0.40", @@ -3939,12 +4191,13 @@ dependencies = [ name = "protocols" version = "0.1.0" dependencies = [ + "async-trait", "oci-spec", - "protobuf", + "protobuf 3.7.2", "serde", "serde_json", "ttrpc", - "ttrpc-codegen", + "ttrpc-codegen 0.6.0", ] [[package]] @@ -3967,6 +4220,65 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "qapi" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6412bdd014ebee03ddbbe79ac03a0b622cce4d80ba45254f6357c847f06fa38" +dependencies = [ + "bytes 1.7.2", + "futures", + "log", + "memchr", + "qapi-qmp", + "qapi-spec", + "serde", + "serde_json", + "tokio", + "tokio-util", +] + +[[package]] +name = "qapi-codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb959fed63a69baa2e3ae57224d885e686bc3f56c9bb3b03406969980ea57a44" +dependencies = [ + "qapi-parser", +] + +[[package]] +name = "qapi-parser" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b37f643cfdf67a409a9323334138a11636a5db5d56cedcc780d7a82a7fb7659" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "qapi-qmp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b944db7e544d2fa97595e9a000a6ba5c62c426fa185e7e00aabe4b5640b538" +dependencies = [ + "qapi-codegen", + "qapi-spec", + "serde", +] + +[[package]] +name = "qapi-spec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e6bdbbe5d13015b21a49a778a29ae3cee9c450c3154e1648aed670d57fe5ba" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", +] + [[package]] name = "quinn" version = "0.11.5" @@ -4167,10 +4479,10 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-rustls", "hyper-util", "ipnet", @@ -4220,7 +4532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.1", "getrandom", "libc", "untrusted 0.9.0", @@ -4304,6 +4616,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if 1.0.1", + "ordered-multimap", +] + [[package]] name = "rust_decimal" version = "1.37.1" @@ -4397,7 +4719,7 @@ dependencies = [ "bit-vec", "capctl", "caps", - "cfg-if", + "cfg-if 1.0.1", "cgroups-rs", "futures", "inotify", @@ -4407,12 +4729,12 @@ dependencies = [ "nix 0.26.4", "oci-spec", "path-absolutize", - "protobuf", + "protobuf 3.7.2", "protocols", "regex", "rlimit", "runtime-spec", - "safe-path", + "safe-path 0.1.0", "scan_fmt", "scopeguard", "serde", @@ -4420,7 +4742,7 @@ dependencies = [ "slog", "slog-scope", "tokio", - "tokio-vsock", + "tokio-vsock 0.3.4", "xattr 0.2.3", "zbus", ] @@ -4500,6 +4822,15 @@ dependencies = [ "libc", ] +[[package]] +name = "safe-path" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "980abdd3220aa19b67ca3ea07b173ca36383f18ae48cde696d90c8af39447ffb" +dependencies = [ + "libc", +] + [[package]] name = "salsa20" version = "0.10.2" @@ -4586,6 +4917,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "seccompiler" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d1292a1131b22ccea49f30bd106f1238b5ddeec1a98d39268dcc31d540e68" +dependencies = [ + "libc", +] + [[package]] name = "security-framework" version = "3.2.0" @@ -4848,7 +5188,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "cpufeatures", "digest", ] @@ -4870,7 +5210,7 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "cpufeatures", "digest", ] @@ -4885,6 +5225,19 @@ dependencies = [ "keccak", ] +[[package]] +name = "shim-interface" +version = "0.1.0" +dependencies = [ + "anyhow", + "hyper 0.14.32", + "hyperlocal", + "kata-sys-util", + "kata-types", + "nix 0.26.4", + "tokio", +] + [[package]] name = "shlex" version = "1.3.0" @@ -4919,7 +5272,7 @@ dependencies = [ "async-trait", "aws-lc-rs", "base64 0.22.1", - "cfg-if", + "cfg-if 1.0.1", "chrono", "const-oid", "crypto_secretbox", @@ -5218,7 +5571,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "fastrand 1.9.0", "redox_syscall 0.3.5", "rustix 0.37.28", @@ -5242,6 +5595,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "tests_utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "kata-types", + "rand", +] + [[package]] name = "thiserror" version = "1.0.40" @@ -5288,7 +5650,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "once_cell", ] @@ -5323,6 +5685,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "timerfd" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84e482e368cf7efa2c8b570f476e5b9fd9fd5e9b9219fc567832b05f13511091" +dependencies = [ + "rustix 0.38.34", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -5454,7 +5825,20 @@ dependencies = [ "futures", "libc", "tokio", - "vsock", + "vsock 0.2.6", +] + +[[package]] +name = "tokio-vsock" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a15c15b1bc91f90902347eff163b5b682643aff0c8e972912cca79bd9208dd" +dependencies = [ + "bytes 1.7.2", + "futures", + "libc", + "tokio", + "vsock 0.3.0", ] [[package]] @@ -5530,10 +5914,10 @@ dependencies = [ "base64 0.22.1", "bytes 1.7.2", "h2", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -5652,28 +6036,59 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c580c498a547b4c083ec758be543e11a0772e03013aef4cdb1fbe77c8b62cae" dependencies = [ + "async-trait", "byteorder", "crossbeam", + "futures", "home", "libc", "log", "nix 0.26.4", - "protobuf", - "protobuf-codegen", + "protobuf 3.7.2", + "protobuf-codegen 3.7.2", "thiserror 1.0.40", + "tokio", + "tokio-vsock 0.4.0", "windows-sys 0.48.0", ] +[[package]] +name = "ttrpc-codegen" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7f7631d7a9ebed715a47cd4cb6072cbc7ae1d4ec01598971bbec0024340c2" +dependencies = [ + "protobuf 2.28.0", + "protobuf-codegen 3.7.2", + "protobuf-support", + "ttrpc-compiler 0.6.2", +] + [[package]] name = "ttrpc-codegen" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e5c657ef5cea6f6c6073c1be0787ba4482f42a569d4821e467daec795271f86" dependencies = [ - "protobuf", - "protobuf-codegen", + "protobuf 3.7.2", + "protobuf-codegen 3.7.2", "protobuf-support", - "ttrpc-compiler", + "ttrpc-compiler 0.8.0", +] + +[[package]] +name = "ttrpc-compiler" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0672eb06e5663ad190c7b93b2973f5d730259859b62e4e3381301a12a7441107" +dependencies = [ + "derive-new", + "prost 0.8.0", + "prost-build 0.8.0", + "prost-types 0.8.0", + "protobuf 2.28.0", + "protobuf-codegen 2.28.0", + "tempfile", ] [[package]] @@ -5686,8 +6101,8 @@ dependencies = [ "prost 0.8.0", "prost-build 0.8.0", "prost-types 0.8.0", - "protobuf", - "protobuf-codegen", + "protobuf 3.7.2", + "protobuf-codegen 3.7.2", "tempfile", ] @@ -5829,6 +6244,36 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vmm-sys-util" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08604d7be03eb26e33b3cee3ed4aef2bf550b305d1cca60e84da5d28d3790b62" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "vmm-sys-util" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48b7b084231214f7427041e4220d77dfe726897a6d41fddee450696e66ff2a29" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "vmm-sys-util" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d21f366bf22bfba3e868349978766a965cbe628c323d58e026be80b8357ab789" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "vsock" version = "0.2.6" @@ -5839,6 +6284,16 @@ dependencies = [ "nix 0.23.2", ] +[[package]] +name = "vsock" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8e1df0bf1e1b28095c24564d1b90acae64ca69b097ed73896e342fa6649c57" +dependencies = [ + "libc", + "nix 0.24.3", +] + [[package]] name = "waker-fn" version = "1.1.0" @@ -5876,7 +6331,7 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "once_cell", "wasm-bindgen-macro", ] @@ -5902,7 +6357,7 @@ version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ - "cfg-if", + "cfg-if 1.0.1", "js-sys", "wasm-bindgen", "web-sys", diff --git a/src/tools/agent-ctl/Cargo.toml b/src/tools/agent-ctl/Cargo.toml index 5badfbab7e..425e56cbec 100644 --- a/src/tools/agent-ctl/Cargo.toml +++ b/src/tools/agent-ctl/Cargo.toml @@ -30,7 +30,7 @@ rand = "0.8.4" protobuf = "3.2.0" log = "0.4.22" -nix = "0.23.0" +nix = "0.24.2" libc = "0.2.112" # XXX: Must be the same as the version used by the agent ttrpc = "0.8.4" @@ -49,6 +49,11 @@ image-rs = { git = "https://github.com/confidential-containers/guest-components" "signature-cosign-rustls", ] } +kata-types = { path = "../../libs/kata-types" } + +# hypervisor crate from runtime-rs +hypervisor = { path = "../../runtime-rs/crates/hypervisor", features = ["cloud-hypervisor"]} + safe-path = { path = "../../libs/safe-path" } tokio = { version = "1.44.2", features = ["signal"] } diff --git a/src/tools/agent-ctl/src/client.rs b/src/tools/agent-ctl/src/client.rs index 401f7acb69..c292e91af3 100644 --- a/src/tools/agent-ctl/src/client.rs +++ b/src/tools/agent-ctl/src/client.rs @@ -7,14 +7,15 @@ use crate::types::*; use crate::utils; +use crate::vm; use anyhow::{anyhow, Result}; use byteorder::ByteOrder; -use nix::sys::socket::{connect, socket, AddressFamily, SockAddr, SockFlag, SockType, UnixAddr}; +use nix::sys::socket::{connect, socket, AddressFamily, SockFlag, SockType, UnixAddr, VsockAddr}; use protocols::agent::*; use protocols::agent_ttrpc::*; use protocols::health::*; use protocols::health_ttrpc::*; -use slog::{debug, info}; +use slog::{debug, info, warn}; use std::convert::TryFrom; use std::fs; use std::io::Write; // XXX: for flush() @@ -107,6 +108,12 @@ const METADATA_CFG_NS: &str = "agent-ctl-cfg"; // automatically. const AUTO_VALUES_CFG_NAME: &str = "auto-values"; +// Retry count and dial timeout to try connecting to the agent +// Static value taken from runtime-rs configuration calculation +// # Retry times = reconnect_timeout_ms / dial_timeout_ms (default: 300) +const RETRY_AGENT_CONNECT: u64 = 300; +const DIAL_TIMEOUT: u64 = 10; + static AGENT_CMDS: &[AgentCmd] = &[ AgentCmd { name: "AddARPNeighbors", @@ -403,25 +410,43 @@ fn get_builtin_cmd_func(name: &str) -> Result { } fn client_create_vsock_fd(cid: libc::c_uint, port: u32) -> Result { - let fd = socket( - AddressFamily::Vsock, - SockType::Stream, - SockFlag::SOCK_CLOEXEC, - None, - ) - .map_err(|e| anyhow!(e))?; + let sock_addr = VsockAddr::new(cid, port); - let sock_addr = SockAddr::new_vsock(cid, port); + for i in 0..RETRY_AGENT_CONNECT { + let fd = socket( + AddressFamily::Vsock, + SockType::Stream, + SockFlag::SOCK_CLOEXEC, + None, + ) + .map_err(|e| anyhow!(e))?; - connect(fd, &sock_addr).map_err(|e| anyhow!(e))?; + // Connect the socket to vsock server. + match connect(fd, &sock_addr) { + Ok(_) => return Ok(fd), + Err(e) => { + debug!( + sl!(), + "Failed to connect to vsock in attempt:{} error:{:?}", i, e + ); + sleep(Duration::from_millis(DIAL_TIMEOUT)); + continue; + } + } + } - Ok(fd) + Err(anyhow!("Failed to establish vsock connection with agent")) } // Setup the existing stream by making a Hybrid VSOCK host-initiated // connection request to the Hybrid VSOCK-capable hypervisor (CLH or FC), // asking it to route the connection to the Kata Agent running inside the VM. -fn setup_hybrid_vsock(mut stream: &UnixStream, hybrid_vsock_port: u64) -> Result<()> { +fn setup_hybrid_vsock(path: &str, hybrid_vsock_port: u64) -> Result { + debug!( + sl!(), + "setup_hybrid_vsock path:{} port: {}", path, hybrid_vsock_port + ); + // Challenge message sent to the Hybrid VSOCK capable hypervisor asking // for a connection to a real VSOCK server running in the VM on the // port specified as part of this message. @@ -431,39 +456,39 @@ fn setup_hybrid_vsock(mut stream: &UnixStream, hybrid_vsock_port: u64) -> Result // hypervisor informing the client that the CONNECT_CMD was successful. const OK_CMD: &str = "OK"; - // Contact the agent by dialing it's port number and - // waiting for the hybrid vsock hypervisor to route the call for us ;) - // - // See: https://github.com/firecracker-microvm/firecracker/blob/main/docs/vsock.md#host-initiated-connections - let msg = format!("{} {}\n", CONNECT_CMD, hybrid_vsock_port); + for i in 0..RETRY_AGENT_CONNECT { + let mut stream = UnixStream::connect(path)?; + // Contact the agent by dialing it's port number and + // waiting for the hybrid vsock hypervisor to route the call for us ;) + // + // See: https://github.com/firecracker-microvm/firecracker/blob/main/docs/vsock.md#host-initiated-connections + let msg = format!("{} {}\n", CONNECT_CMD, hybrid_vsock_port); + stream.write_all(msg.as_bytes())?; - stream.write_all(msg.as_bytes())?; + // Now, see if we get the expected response + let mut reader = BufReader::new(&mut stream); - // Now, see if we get the expected response - let stream_reader = stream.try_clone()?; - let mut reader = BufReader::new(&stream_reader); + let mut msg = String::new(); + reader.read_line(&mut msg)?; - let mut msg = String::new(); - reader.read_line(&mut msg)?; + if msg.starts_with(OK_CMD) { + let response = msg + .strip_prefix(OK_CMD) + .ok_or(format!("invalid response: {:?}", msg)) + .map_err(|e| anyhow!(e))? + .trim(); - if msg.starts_with(OK_CMD) { - let response = msg - .strip_prefix(OK_CMD) - .ok_or(format!("invalid response: {:?}", msg)) - .map_err(|e| anyhow!(e))? - .trim(); - - debug!(sl!(), "Hybrid VSOCK host-side port: {:?}", response); - } else { - return Err(anyhow!( - "failed to setup Hybrid VSOCK connection: response was: {:?}", - msg - )); + // The Unix stream is now connected directly to the VSOCK socket + // the Kata agent is listening to in the VM. + debug!(sl!(), "Hybrid VSOCK host-side port: {:?}", response); + return Ok(stream); + } else { + debug!(sl!(), "attempt:{} message: {:?}", i, msg); + sleep(Duration::from_millis(DIAL_TIMEOUT)); + continue; + } } - - // The Unix stream is now connected directly to the VSOCK socket - // the Kata agent is listening to in the VM. - Ok(()) + Err(anyhow!("Failed to establish hvsock connection with agent")) } fn create_ttrpc_client( @@ -524,27 +549,21 @@ fn create_ttrpc_client( } }; - let sock_addr = SockAddr::Unix(unix_addr); - - connect(socket_fd, &sock_addr).map_err(|e| { + connect(socket_fd, &unix_addr).map_err(|e| { anyhow!(e).context("Failed to connect to Unix Domain abstract socket") })?; socket_fd + } else if hybrid_vsock { + let stream = setup_hybrid_vsock(&path, hybrid_vsock_port)?; + stream.into_raw_fd() } else { let stream = match UnixStream::connect(path) { Ok(s) => s, - Err(e) => { - return Err( - anyhow!(e).context("failed to create named UNIX Domain stream socket") - ) + Err(err) => { + return Err(anyhow!("failed to setup unix stream: {:?}", err)); } }; - - if hybrid_vsock { - setup_hybrid_vsock(&stream, hybrid_vsock_port)? - } - stream.into_raw_fd() } } @@ -605,7 +624,7 @@ fn announce(cfg: &Config) { info!(sl!(), "announce"; "config" => format!("{:?}", cfg)); } -pub fn client(cfg: &Config, commands: Vec<&str>) -> Result<()> { +pub fn client(cfg: &mut Config, commands: Vec<&str>) -> Result<()> { if commands.len() == 1 && commands[0].eq("list") { println!("Built-in commands:\n"); @@ -628,6 +647,68 @@ pub fn client(cfg: &Config, commands: Vec<&str>) -> Result<()> { announce(cfg); + let vm_ref = handle_vm(cfg)?; + + info!(sl!(), "run commands"); + let result = run_commands(cfg, commands); + + // stop the vm if booted + if vm_ref.is_some() { + info!(sl!(), "stopping test vm"); + // TODO: The error handling here is for cloud-hypervisor. + // We use tokio::runtime to call the async operations of + // runtime-rs::crates::hypervisor::_vm + // These methods can spawn some additional functions, ex + // `clh::inner_hypervisor::cloud_hypervisor_log_output` + // But since we return from the tokio::runtime block + // the runtime is dropped. During stop_vm call, cloud hypervisor + // waits for the logger task which is in cancelled state as a result. + match vm::remove_vm(vm_ref.unwrap()) { + Ok(_) => info!(sl!(), "Successfully shut down test vm"), + Err(e) => warn!(sl!(), "Error shutting down vm:{:?}", e), + } + } + + result.map_err(|e| anyhow!(e)) +} + +fn handle_vm(cfg: &mut Config) -> Result> { + info!(sl!(), "handle vm request"); + + // Return if no vm requested + if cfg.hypervisor_name.is_empty() { + return Ok(None); + } + + // Boot the test vm + let vm_instance = vm::setup_vm(&cfg.hypervisor_name)?; + info!( + sl!(), + "booted test vm with hypervisor: {:?}", vm_instance.hypervisor_name + ); + + // set the vsock server address for connecting with ttrpc server + if !vm_instance.socket_addr.is_empty() { + match vm_instance.hybrid_vsock { + true => { + // hybrid vsock URI expects unix prefix + let addr_fields: Vec<&str> = vm_instance.socket_addr.split("://").collect(); + cfg.server_address = format!("{}://{}", "unix", addr_fields[1]); + cfg.hybrid_vsock = true; + } + false => { + let addr = vm_instance.socket_addr.clone(); + cfg.server_address = format!("{}:{}", addr, 1024); + cfg.hybrid_vsock = false; + } + } + } + + info!(sl!(), "socket server addr: {}", cfg.server_address); + Ok(Some(vm_instance)) +} + +fn run_commands(cfg: &Config, commands: Vec<&str>) -> Result<()> { // Create separate connections for each of the services provided // by the agent. let client = kata_service_agent( diff --git a/src/tools/agent-ctl/src/main.rs b/src/tools/agent-ctl/src/main.rs index 08a3bf82bc..1f36d511a0 100644 --- a/src/tools/agent-ctl/src/main.rs +++ b/src/tools/agent-ctl/src/main.rs @@ -25,6 +25,7 @@ mod image; mod rpc; mod types; mod utils; +mod vm; const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info; @@ -73,6 +74,10 @@ fn make_examples_text(program_name: &str) -> String { # Abstract socket $ {program} connect --server-address "{abstract_server_address}" --cmd Check +- Boot up a test VM and connect to the agent (socket address determined by the tool): + + $ {program} connect --vm qemu --cmd Check + - Query the agent environment: $ {program} connect --server-address "{vsock_server_address}" --cmd GetGuestDetails @@ -140,12 +145,25 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { let interactive = args.contains_id("interactive"); let ignore_errors = args.contains_id("ignore-errors"); + // boot-up a test vm for testing commands + let hypervisor_name = args + .get_one::("vm") + .map(|s| s.as_str()) + .unwrap_or_default() + .to_string(); + let server_address = args .get_one::("server-address") .map(|s| s.as_str()) - .ok_or_else(|| anyhow!("need server adddress"))? + .unwrap_or_default() .to_string(); + // if vm is requested, we retrieve the server + // address after the boot-up is completed + if hypervisor_name.is_empty() && server_address.is_empty() { + return Err(anyhow!("need server address")); + } + let mut commands: Vec<&str> = Vec::new(); if !interactive { @@ -187,7 +205,7 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { let hybrid_vsock = args.contains_id("hybrid-vsock"); let no_auto_values = args.contains_id("no-auto-values"); - let cfg = Config { + let mut cfg = Config { server_address, bundle_dir, timeout_nano, @@ -196,9 +214,10 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { hybrid_vsock, ignore_errors, no_auto_values, + hypervisor_name, }; - let result = rpc::run(&logger, &cfg, commands); + let result = rpc::run(&logger, &mut cfg, commands); result.map_err(|e| anyhow!(e)) } @@ -283,6 +302,12 @@ fn real_main() -> Result<()> { .help("timeout value as nanoseconds or using human-readable suffixes (0 [forever], 99ns, 30us, 2ms, 5s, 7m, etc)") .value_name("human-time"), ) + .arg( + Arg::new("vm") + .long("vm") + .help("boot a pod vm for testing") + .value_name("HYPERVISOR"), + ) ) .subcommand( Command::new("generate-cid") diff --git a/src/tools/agent-ctl/src/rpc.rs b/src/tools/agent-ctl/src/rpc.rs index fd9ad96936..b39fbaea7c 100644 --- a/src/tools/agent-ctl/src/rpc.rs +++ b/src/tools/agent-ctl/src/rpc.rs @@ -11,7 +11,7 @@ use slog::{o, Logger}; use crate::client::client; use crate::types::Config; -pub fn run(logger: &Logger, cfg: &Config, commands: Vec<&str>) -> Result<()> { +pub fn run(logger: &Logger, cfg: &mut Config, commands: Vec<&str>) -> Result<()> { // Maintain the global logger for the duration of the ttRPC comms let _guard = slog_scope::set_global_logger(logger.new(o!("subsystem" => "rpc"))); diff --git a/src/tools/agent-ctl/src/types.rs b/src/tools/agent-ctl/src/types.rs index e1d343b89a..198a8a274a 100644 --- a/src/tools/agent-ctl/src/types.rs +++ b/src/tools/agent-ctl/src/types.rs @@ -19,6 +19,7 @@ pub struct Config { pub hybrid_vsock: bool, pub ignore_errors: bool, pub no_auto_values: bool, + pub hypervisor_name: String, } // CopyFile input struct diff --git a/src/tools/agent-ctl/src/vm/mod.rs b/src/tools/agent-ctl/src/vm/mod.rs new file mode 100644 index 0000000000..d49a97d3ad --- /dev/null +++ b/src/tools/agent-ctl/src/vm/mod.rs @@ -0,0 +1,56 @@ +// Copyright (c) 2024 Microsoft Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// +// Description: Boot UVM for testing container storages/volumes. + +use anyhow::{anyhow, Context, Result}; +use hypervisor::Hypervisor; +use kata_types::config::{hypervisor::HYPERVISOR_NAME_CH, hypervisor::HYPERVISOR_NAME_QEMU}; +use slog::info; +use std::sync::Arc; + +mod vm_ops; +mod vm_utils; + +lazy_static! { + pub(crate) static ref SUPPORTED_VMMS: Vec<&'static str> = + vec![HYPERVISOR_NAME_CH, HYPERVISOR_NAME_QEMU]; +} + +#[derive(Clone)] +pub struct TestVm { + pub hypervisor_name: String, + pub hypervisor_instance: Arc, + pub socket_addr: String, + pub hybrid_vsock: bool, +} + +// Helper method to boot a test pod VM +pub fn setup_vm(hypervisor_name: &str) -> Result { + info!( + sl!(), + "booting a pod vm using hypervisor:{:?}", hypervisor_name + ); + + if !SUPPORTED_VMMS.contains(&hypervisor_name) { + return Err(anyhow!("Unsupported hypervisor:{}", hypervisor_name)); + } + + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(vm_ops::boot_vm(hypervisor_name)) + .context("booting the test vm") +} + +// Helper method to stop a test pod VM +pub fn remove_vm(instance: TestVm) -> Result<()> { + info!(sl!(), "Stopping booted pod vm"); + + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(vm_ops::stop_vm(instance.hypervisor_instance)) + .context("stopping the test vm") +} diff --git a/src/tools/agent-ctl/src/vm/vm_ops.rs b/src/tools/agent-ctl/src/vm/vm_ops.rs new file mode 100644 index 0000000000..74a807bcd0 --- /dev/null +++ b/src/tools/agent-ctl/src/vm/vm_ops.rs @@ -0,0 +1,166 @@ +// Copyright (c) 2024 Microsoft Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// +// Description: Boot UVM for testing container storages/volumes. + +use crate::vm::{vm_utils, TestVm}; +use anyhow::{anyhow, Context, Result}; +use hypervisor::{ + ch::CloudHypervisor, + device::{ + device_manager::{do_handle_device, DeviceManager}, + DeviceConfig, + }, + qemu::Qemu, + BlockConfig, Hypervisor, VsockConfig, +}; +use kata_types::config::{ + hypervisor::register_hypervisor_plugin, hypervisor::TopologyConfigInfo, + hypervisor::HYPERVISOR_NAME_CH, hypervisor::HYPERVISOR_NAME_QEMU, CloudHypervisorConfig, + QemuConfig, +}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; + +// Clh specific configuration path +const CLH_CONFIG_PATH: &str = + "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-cloud-hypervisor.toml"; + +// qemu specific configuration path +const QEMU_CONFIG_PATH: &str = + "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-qemu-runtime-rs.toml"; + +const VM_NAME: &str = "agent-ctl-testvm"; +const VM_START_TIMEOUT: i32 = 10_000; + +// Boot the test vm. +// In summary, this method +// - parses hypervisor specific kata config file +// - loads hypervisor specific config +// - instantiates a hypervisor object +// - calls prepare_vm +// - instantiates device manager to handle devices +// - calls start_vm to boot pod vm +// - retrieves the agent ttrpc server socket address +pub(crate) async fn boot_vm(name: &str) -> Result { + let config_path; + let mut is_hybrid_vsock = false; + + // Register the hypervisor config plugin + match name { + HYPERVISOR_NAME_CH => { + register_hypervisor_plugin(HYPERVISOR_NAME_CH, Arc::new(CloudHypervisorConfig::new())); + config_path = CLH_CONFIG_PATH; + is_hybrid_vsock = true; + } + &_ => { + register_hypervisor_plugin(HYPERVISOR_NAME_QEMU, Arc::new(QemuConfig::new())); + config_path = QEMU_CONFIG_PATH; + } + }; + + // get the kata configuration toml + let toml_config = vm_utils::load_config(config_path)?; + + let hypervisor_config = toml_config + .hypervisor + .get(name) + .ok_or_else(|| anyhow!("Failed to get hypervisor config")) + .context("get hypervisor config")?; + + let hypervisor: Arc = match name { + HYPERVISOR_NAME_CH => { + let hyp_ch = Arc::new(CloudHypervisor::new()); + hyp_ch + .set_hypervisor_config(hypervisor_config.clone()) + .await; + hyp_ch + } + &_ => { + let hyp_qemu = Arc::new(Qemu::new()); + hyp_qemu + .set_hypervisor_config(hypervisor_config.clone()) + .await; + hyp_qemu + } + }; + + // prepare vm + // we do not pass any network namesapce since we dont want any + let empty_anno_map: HashMap = HashMap::new(); + hypervisor + .prepare_vm(VM_NAME, None, &empty_anno_map) + .await + .context(" prepare test vm")?; + + // instantiate device manager + let topo_config = TopologyConfigInfo::new(&toml_config); + let dev_manager = Arc::new(RwLock::new( + DeviceManager::new(hypervisor.clone(), topo_config.as_ref()) + .await + .context("failed to create device manager")?, + )); + + // For qemu, we need some additional device handling + // - vsock device + // - block device for rootfs if using image + if name.contains(HYPERVISOR_NAME_QEMU) { + add_vsock_device(dev_manager.clone()) + .await + .context("qemu::adding vsock device")?; + if !hypervisor_config.boot_info.image.is_empty() { + let blk_config = BlockConfig { + path_on_host: hypervisor_config.boot_info.image.clone(), + is_readonly: true, + driver_option: hypervisor_config.boot_info.vm_rootfs_driver.clone(), + ..Default::default() + }; + add_block_device(dev_manager.clone(), blk_config) + .await + .context("qemu: handle rootfs")?; + } + } + + // start vm + hypervisor + .start_vm(VM_START_TIMEOUT) + .await + .context("start pod vm")?; + + let agent_socket_addr = hypervisor + .get_agent_socket() + .await + .context("get agent socket path")?; + + // return the vm structure + Ok(TestVm { + hypervisor_name: name.to_string(), + hypervisor_instance: hypervisor, + socket_addr: agent_socket_addr, + hybrid_vsock: is_hybrid_vsock, + }) +} + +pub(crate) async fn stop_vm(instance: Arc) -> Result<()> { + instance.stop_vm().await.context("stopping pod vm") +} + +async fn add_block_device(dev_mgr: Arc>, cfg: BlockConfig) -> Result<()> { + do_handle_device(&dev_mgr, &DeviceConfig::BlockCfg(cfg)) + .await + .context("handle block device failed")?; + Ok(()) +} + +async fn add_vsock_device(dev_mgr: Arc>) -> Result<()> { + let vsock_config = VsockConfig { + guest_cid: libc::VMADDR_CID_ANY, + }; + + do_handle_device(&dev_mgr, &DeviceConfig::VsockCfg(vsock_config)) + .await + .context("handle vsock device failed")?; + Ok(()) +} diff --git a/src/tools/agent-ctl/src/vm/vm_utils.rs b/src/tools/agent-ctl/src/vm/vm_utils.rs new file mode 100644 index 0000000000..6020532ed6 --- /dev/null +++ b/src/tools/agent-ctl/src/vm/vm_utils.rs @@ -0,0 +1,53 @@ +// Copyright (c) 2025 Microsoft Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// +// Description: Boot UVM for testing container storages/volumes. + +use anyhow::{anyhow, Context, Result}; +use kata_types::config::TomlConfig; +use slog::info; + +// Helper function to parse a configuration file. +pub fn load_config(config_file: &str) -> Result { + info!(sl!(), "Load kata configuration file {}", config_file); + + let (mut toml_config, _) = TomlConfig::load_from_file(config_file) + .context("Failed to load kata configuration file")?; + + // Update the agent kernel params in hypervisor config + update_agent_kernel_params(&mut toml_config)?; + + // validate configuration and return the error + toml_config.validate()?; + + info!(sl!(), "parsed config content {:?}", &toml_config); + Ok(toml_config) +} + +pub fn to_kernel_string(key: String, val: String) -> Result { + if key.is_empty() && val.is_empty() { + Err(anyhow!("Empty key and value")) + } else if key.is_empty() { + Err(anyhow!("Empty key")) + } else if val.is_empty() { + Ok(key.to_string()) + } else { + Ok(format!("{}{}{}", key, "=", val)) + } +} + +fn update_agent_kernel_params(config: &mut TomlConfig) -> Result<()> { + let mut params = vec![]; + if let Ok(kv) = config.get_agent_kernel_params() { + for (k, v) in kv.into_iter() { + if let Ok(s) = to_kernel_string(k.to_owned(), v.to_owned()) { + params.push(s); + } + } + if let Some(h) = config.hypervisor.get_mut(&config.runtime.hypervisor_name) { + h.boot_info.add_kernel_params(params); + } + } + Ok(()) +} diff --git a/tests/functional/kata-agent-apis/api-tests/test_vm_GuestDetails.bats b/tests/functional/kata-agent-apis/api-tests/test_vm_GuestDetails.bats new file mode 100755 index 0000000000..74f9d750ce --- /dev/null +++ b/tests/functional/kata-agent-apis/api-tests/test_vm_GuestDetails.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# Copyright (c) 2024 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +load "${BATS_TEST_DIRNAME}/../../../common.bash" +load "${BATS_TEST_DIRNAME}/../setup_common.sh" + +setup_file() { + info "setup" + sudo rm qmp.sock console.sock || echo "No existing qmp.sock/console.sock" +} + +@test "Test GetGuestDetails: Boot qemu pod vm and run GetGuestDetails" { + info "Boot qemu vm, establish connection with agent inside the vm and send GetGuestDetails command" + local cmds=() + cmds+=("--vm qemu -c GetGuestDetails") + run_agent_ctl "${cmds[@]}" + sudo rm qmp.sock console.sock +} + +@test "Test GetGuestDetails: Boot cloud hypervisor pod vm and run GetGuestDetails" { + info "Boot cloud hypervisor vm, establish connection with agent inside the vm and send GetGuestDetails command" + local cmds=() + cmds+=("--vm cloud-hypervisor -c GetGuestDetails") + run_agent_ctl "${cmds[@]}" +} + +teardown_file() { + info "teardown" + sudo rm -r /run/kata/agent-ctl-testvm || echo "Failed to clean /run/kata/agent-ctl-testvm" + sudo rm -r /run/kata-containers/ || echo "Failed to clean /run/kata-containers" +}