diff --git a/tools/agent-ctl/Makefile b/tools/agent-ctl/Makefile index 2ae095a00b..930b522017 100644 --- a/tools/agent-ctl/Makefile +++ b/tools/agent-ctl/Makefile @@ -19,6 +19,7 @@ vendor: test: install: + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --target $(TRIPLE) --path . check: diff --git a/tools/agent-ctl/README.md b/tools/agent-ctl/README.md index 9fc4771df7..f4c42e508b 100644 --- a/tools/agent-ctl/README.md +++ b/tools/agent-ctl/README.md @@ -45,7 +45,7 @@ the agent protocol and the client and server implementations. | Agent Control (client) API calls | [`src/client.rs`](src/client.rs) | `agent_cmd_container_create()` | Agent Control tool function that calls the `CreateContainer` API. | | Agent (server) API implementations | [`rpc.rs`](../../src/agent/src/rpc.rs) | `create_container()` | Server function that implements the `CreateContainers` API. | -## Running the tool +## Run the tool ### Prerequisites @@ -62,22 +62,31 @@ $ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf ### Connect to a real Kata Container +The method used to connect to Kata Containers agent depends on the configured +hypervisor. Although by default the Kata Containers agent listens for API calls on a +VSOCK socket, the way that socket is exposed to the host depends on the +hypervisor. + +#### QEMU + +Since QEMU supports VSOCK sockets in the standard way, it is only necessary to +establish the VSOCK guest CID value to connect to the agent. + 1. Start a Kata Container 1. Establish the VSOCK guest CID number for the virtual machine: - Assuming you are running a single QEMU based Kata Container, you can look - at the program arguments to find the (randomly-generated) `guest-cid=` option - value: - ```sh - $ guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2) + $ guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1) ``` 1. Run the tool to connect to the agent: ```sh - $ cargo run -- -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails + # Default VSOCK port the agent listens on + $ agent_vsock_port=1024 + + $ cargo run -- -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:${agent_vsock_port}" -c Check -c GetGuestDetails ``` This examples makes two API calls: @@ -86,6 +95,75 @@ $ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf - It then runs `GetGuestDetails` to establish some details of the environment the agent is running in. +#### Cloud Hypervisor and Firecracker + +Cloud Hypervisor and Firecracker both use "hybrid VSOCK" which uses a local +UNIX socket rather than the host kernel to handle communication with the +guest. As such, you need to specify the path to the UNIX socket. + +Since the UNIX socket path is sandbox-specific, you need to run the +`kata-runtime env` command to determine the socket's "template path". This +path includes a `{ID}` tag that represents the real sandbox ID or name. + +Further, since the socket path is below the sandbox directory and since that +directory is `root` owned, it is necessary to run the tool as `root` when +using a Hybrid VSOCKS hypervisor. + +##### Determine socket path template value + +###### Configured hypervisor is Cloud Hypervisor + +```bash +$ socket_path_template=$(sudo kata-runtime env --json | jq '.Hypervisor.SocketPath') +$ echo "$socket_path_template" +"/run/vc/vm/{ID}/clh.sock" +``` + +###### Configured hypervisor is Firecracker + +```bash +$ socket_path_template=$(sudo kata-runtime env --json | jq '.Hypervisor.SocketPath') +$ echo "$socket_path_template" +"/run/vc/firecracker/{ID}/root/kata.hvsock" +``` + +> **Note:** +> +> Do not rely on the paths shown above: you should run the command yourself +> as these paths _may_ change. + +Once you have determined the template path, build and install the tool to make +it easier to run as the `root` user. + +##### Build and install + +```bash +# Install for user +$ make install + +# Install centrally +$ sudo install -o root -g root -m 0755 ~/.cargo/bin/kata-agent-ctl /usr/local/bin +``` + +1. Start a Kata Container + + Create a container called `foo`. + +1. Run the tool + + ```bash + # Name of container + $ sandbox_id="foo" + + # Create actual socket path + $ socket_path=$(echo "$socket_path_template" | sed "s/{ID}/${sandbox_id}/g" | tr -d '"') + + $ sudo kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "unix://${socket_path}" --hybrid-vsock -c Check -c GetGuestDetails + ``` + + > **Note:** The `socket_path_template` variable was set in the + > [Determine socket path template value](#determine-socket-path-template-value) section. + ### Run the tool and the agent in the same environment > **Warnings:** @@ -100,6 +178,8 @@ $ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf $ sudo KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket target/x86_64-unknown-linux-musl/release/kata-agent ``` + > **Note:** This example assumes an Intel x86-64 system. + 1. Run the tool in the same environment: ```sh diff --git a/tools/agent-ctl/src/client.rs b/tools/agent-ctl/src/client.rs index 9113639316..605219147c 100644 --- a/tools/agent-ctl/src/client.rs +++ b/tools/agent-ctl/src/client.rs @@ -17,6 +17,7 @@ use protocols::health_ttrpc::*; use slog::{debug, info}; use std::io; use std::io::Write; // XXX: for flush() +use std::io::{BufRead, BufReader}; use std::os::unix::io::{IntoRawFd, RawFd}; use std::os::unix::net::UnixStream; use std::thread::sleep; @@ -87,11 +88,6 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_sandbox_add_arp_neighbors, }, - AgentCmd { - name: "AddSwap", - st: ServiceType::Agent, - fp: agent_cmd_sandbox_add_swap, - }, AgentCmd { name: "Check", st: ServiceType::Health, @@ -372,7 +368,59 @@ fn client_create_vsock_fd(cid: libc::c_uint, port: u32) -> Result { Ok(fd) } -fn create_ttrpc_client(server_address: String) -> Result { +// Setup the existing stream by making a Hybrid VSOCK host-initiated +// connection request to the Hybrid VSOCK-capable hypervisor (CLH or FC), +// asking it to route the connection to the Kata Agent running inside the VM. +fn setup_hybrid_vsock(mut stream: &UnixStream, hybrid_vsock_port: u64) -> Result<()> { + // Challenge message sent to the Hybrid VSOCK capable hypervisor asking + // for a connection to a real VSOCK server running in the VM on the + // port specified as part of this message. + const CONNECT_CMD: &str = "CONNECT"; + + // Expected response message returned by the Hybrid VSOCK capable + // hypervisor informing the client that the CONNECT_CMD was successful. + const OK_CMD: &str = "OK"; + + // Contact the agent by dialing it's port number and + // waiting for the hybrid vsock hypervisor to route the call for us ;) + // + // See: https://github.com/firecracker-microvm/firecracker/blob/main/docs/vsock.md#host-initiated-connections + let msg = format!("{} {}\n", CONNECT_CMD, hybrid_vsock_port); + + stream.write_all(msg.as_bytes())?; + + // Now, see if we get the expected response + let stream_reader = stream.try_clone()?; + let mut reader = BufReader::new(&stream_reader); + + let mut msg = String::new(); + reader.read_line(&mut msg)?; + + if msg.starts_with(OK_CMD) { + let response = msg + .strip_prefix(OK_CMD) + .ok_or(format!("invalid response: {:?}", msg)) + .map_err(|e| anyhow!(e))? + .trim(); + + debug!(sl!(), "Hybrid VSOCK host-side port: {:?}", response); + } else { + return Err(anyhow!( + "failed to setup Hybrid VSOCK connection: response was: {:?}", + msg + )); + } + + // The Unix stream is now connected directly to the VSOCK socket + // the Kata agent is listening to in the VM. + Ok(()) +} + +fn create_ttrpc_client( + server_address: String, + hybrid_vsock_port: u64, + hybrid_vsock: bool, +) -> Result { if server_address == "" { return Err(anyhow!("server address cannot be blank")); } @@ -388,7 +436,7 @@ fn create_ttrpc_client(server_address: String) -> Result { let fd: RawFd = match scheme.as_str() { // Formats: // - // - "unix://absolute-path" (domain socket) + // - "unix://absolute-path" (domain socket, or hybrid vsock!) // (example: "unix:///tmp/domain.socket") // // - "unix://@absolute-path" (abstract socket) @@ -445,6 +493,10 @@ fn create_ttrpc_client(server_address: String) -> Result { } }; + if hybrid_vsock { + setup_hybrid_vsock(&stream, hybrid_vsock_port)? + } + stream.into_raw_fd() } } @@ -481,14 +533,22 @@ fn create_ttrpc_client(server_address: String) -> Result { Ok(ttrpc::client::Client::new(fd)) } -fn kata_service_agent(server_address: String) -> Result { - let ttrpc_client = create_ttrpc_client(server_address)?; +fn kata_service_agent( + server_address: String, + hybrid_vsock_port: u64, + hybrid_vsock: bool, +) -> Result { + let ttrpc_client = create_ttrpc_client(server_address, hybrid_vsock_port, hybrid_vsock)?; Ok(AgentServiceClient::new(ttrpc_client)) } -fn kata_service_health(server_address: String) -> Result { - let ttrpc_client = create_ttrpc_client(server_address)?; +fn kata_service_health( + server_address: String, + hybrid_vsock_port: u64, + hybrid_vsock: bool, +) -> Result { + let ttrpc_client = create_ttrpc_client(server_address, hybrid_vsock_port, hybrid_vsock)?; Ok(HealthClient::new(ttrpc_client)) } @@ -522,8 +582,17 @@ pub fn client(cfg: &Config, commands: Vec<&str>) -> Result<()> { // Create separate connections for each of the services provided // by the agent. - let client = kata_service_agent(cfg.server_address.clone())?; - let health = kata_service_health(cfg.server_address.clone())?; + let client = kata_service_agent( + cfg.server_address.clone(), + cfg.hybrid_vsock_port, + cfg.hybrid_vsock, + )?; + + let health = kata_service_health( + cfg.server_address.clone(), + cfg.hybrid_vsock_port, + cfg.hybrid_vsock, + )?; let mut options = Options::new(); @@ -1923,29 +1992,3 @@ fn get_repeat_count(cmdline: &str) -> i64 { Err(_) => return default_repeat_count, } } - -fn agent_cmd_sandbox_add_swap( - ctx: &Context, - client: &AgentServiceClient, - _health: &HealthClient, - _options: &mut Options, - _args: &str, -) -> Result<()> { - let req = AddSwapRequest::default(); - - let ctx = clone_context(ctx); - - debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); - - let reply = client - .add_swap(ctx, &req) - .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; - - // FIXME: Implement 'AddSwap' fully. - eprintln!("FIXME: 'AddSwap' not fully implemented"); - - info!(sl!(), "response received"; - "response" => format!("{:?}", reply)); - - Ok(()) -} diff --git a/tools/agent-ctl/src/main.rs b/tools/agent-ctl/src/main.rs index 313068e0c1..ab4a90e45c 100644 --- a/tools/agent-ctl/src/main.rs +++ b/tools/agent-ctl/src/main.rs @@ -6,6 +6,7 @@ #[macro_use] extern crate lazy_static; +use crate::types::Config; use anyhow::{anyhow, Result}; use clap::{crate_name, crate_version, App, Arg, SubCommand}; use std::io; @@ -39,6 +40,9 @@ const WARNING_TEXT: &str = r#"WARNING: irrevocably other parts of the system or even kill a running container or sandbox."#; +// The VSOCK port number the Kata agent uses to listen to API requests on. +const DEFAULT_KATA_AGENT_API_VSOCK_PORT: &str = "1024"; + fn make_examples_text(program_name: &str) -> String { let abstract_server_address = "unix://@/foo/bar/abstract.socket"; let bundle = "$bundle_dir"; @@ -47,6 +51,7 @@ fn make_examples_text(program_name: &str) -> String { let local_server_address = "unix:///tmp/local.socket"; let sandbox_id = "$sandbox_id"; let vsock_server_address = "vsock://3:1024"; + let hybrid_vsock_server_address = "unix:///run/vc/vm/foo/clh.sock"; format!( r#"EXAMPLES: @@ -55,6 +60,10 @@ fn make_examples_text(program_name: &str) -> String { $ {program} connect --server-address "{vsock_server_address}" --cmd Check +- Connect to the agent using a Hybrid VSOCK hypervisor (here Cloud Hypervisor): + + $ {program} connect --server-address "{hybrid_vsock_server_address}" --hybrid-vsock --cmd Check + - Connect to the agent using local sockets (when running in same environment as the agent): # Local socket @@ -109,6 +118,7 @@ fn make_examples_text(program_name: &str) -> String { program = program_name, sandbox_id = sandbox_id, vsock_server_address = vsock_server_address, + hybrid_vsock_server_address = hybrid_vsock_server_address, ) } @@ -124,7 +134,8 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { let server_address = args .value_of("server-address") .ok_or("need server adddress".to_string()) - .map_err(|e| anyhow!(e))?; + .map_err(|e| anyhow!(e))? + .to_string(); let mut commands: Vec<&str> = Vec::new(); @@ -149,17 +160,28 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { None => 0, }; - let bundle_dir = args.value_of("bundle-dir").unwrap_or(""); + let hybrid_vsock_port: u64 = args + .value_of("hybrid-vsock-port") + .ok_or("Need Hybrid VSOCK port number") + .map(|p| p.parse::().unwrap()) + .map_err(|e| anyhow!("VSOCK port number must be an integer: {:?}", e))?; - let result = rpc::run( - &logger, + let bundle_dir = args.value_of("bundle-dir").unwrap_or("").to_string(); + + let hybrid_vsock = args.is_present("hybrid-vsock"); + + let cfg = Config { server_address, bundle_dir, interactive, ignore_errors, timeout_nano, - commands, - ); + hybrid_vsock_port, + hybrid_vsock, + }; + + let result = rpc::run(&logger, &cfg, commands); + if result.is_err() { return result; } @@ -170,6 +192,11 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> { fn real_main() -> Result<()> { let name = crate_name!(); + let hybrid_vsock_port_help = format!( + "Kata agent VSOCK port number (only useful with --hybrid-vsock) [default: {}]", + DEFAULT_KATA_AGENT_API_VSOCK_PORT + ); + let app = App::new(name) .version(crate_version!()) .about(ABOUT_TEXT) @@ -209,6 +236,19 @@ fn real_main() -> Result<()> { .long("ignore-errors") .help("Don't exit on first error"), ) + .arg( + Arg::with_name("hybrid-vsock") + .long("hybrid-vsock") + .help("Treat a unix:// server address as a Hybrid VSOCK one"), + ) + .arg( + Arg::with_name("hybrid-vsock-port") + .long("hybrid-vsock-port") + .help(&hybrid_vsock_port_help) + .default_value(DEFAULT_KATA_AGENT_API_VSOCK_PORT) + .takes_value(true) + .value_name("PORT") + ) .arg( Arg::with_name("interactive") .short("i") diff --git a/tools/agent-ctl/src/rpc.rs b/tools/agent-ctl/src/rpc.rs index 988fe43d4a..fd9ad96936 100644 --- a/tools/agent-ctl/src/rpc.rs +++ b/tools/agent-ctl/src/rpc.rs @@ -11,25 +11,9 @@ use slog::{o, Logger}; use crate::client::client; use crate::types::Config; -pub fn run( - logger: &Logger, - server_address: &str, - bundle_dir: &str, - interactive: bool, - ignore_errors: bool, - timeout_nano: i64, - commands: Vec<&str>, -) -> Result<()> { - let cfg = Config { - server_address: server_address.to_string(), - bundle_dir: bundle_dir.to_string(), - timeout_nano: timeout_nano, - interactive: interactive, - ignore_errors: ignore_errors, - }; - +pub fn run(logger: &Logger, cfg: &Config, commands: Vec<&str>) -> Result<()> { // Maintain the global logger for the duration of the ttRPC comms let _guard = slog_scope::set_global_logger(logger.new(o!("subsystem" => "rpc"))); - client(&cfg, commands) + client(cfg, commands) } diff --git a/tools/agent-ctl/src/types.rs b/tools/agent-ctl/src/types.rs index 65ccfcb9a8..b5d8f31a5c 100644 --- a/tools/agent-ctl/src/types.rs +++ b/tools/agent-ctl/src/types.rs @@ -14,6 +14,8 @@ pub struct Config { pub server_address: String, pub bundle_dir: String, pub timeout_nano: i64, + pub hybrid_vsock_port: u64, pub interactive: bool, + pub hybrid_vsock: bool, pub ignore_errors: bool, }