agent-ctl: Update for Hybrid VSOCK

Allow the `agent-ctl` tool to connect to a Hybrid VSOCK hypervisor such
as Cloud Hypervisor or Firecracker.

Fixes: #2914.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>

tools/agent-ctl/Makefile

@@ -19,6 +19,7 @@ vendor:
 test:
 
 install:
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --target $(TRIPLE) --path .
 
 check:
 

tools/agent-ctl/README.md

@@ -45,7 +45,7 @@ the agent protocol and the client and server implementations.
 | Agent Control (client) API calls | [`src/client.rs`](src/client.rs) | `agent_cmd_container_create()` | Agent Control tool function that calls the `CreateContainer` API. |
 | Agent (server) API implementations | [`rpc.rs`](../../src/agent/src/rpc.rs) | `create_container()` | Server function that implements the `CreateContainers` API. |
 
-## Running the tool
+## Run the tool
 
 ### Prerequisites
 
@@ -62,22 +62,31 @@ $ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf
 
 ### Connect to a real Kata Container
 
+The method used to connect to Kata Containers agent depends on the configured
+hypervisor. Although by default the Kata Containers agent listens for API calls on a
+VSOCK socket, the way that socket is exposed to the host depends on the
+hypervisor.
+
+#### QEMU
+
+Since QEMU supports VSOCK sockets in the standard way, it is only necessary to
+establish the VSOCK guest CID value to connect to the agent.
+
 1. Start a Kata Container
 
 1. Establish the VSOCK guest CID number for the virtual machine:
 
-   Assuming you are running a single QEMU based Kata Container, you can look
-   at the program arguments to find the (randomly-generated) `guest-cid=` option
-   value:
-
    ```sh
-   $ guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2)
+   $ guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
    ```
 
 1. Run the tool to connect to the agent:
 
    ```sh
-   $ cargo run -- -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails
+   # Default VSOCK port the agent listens on
+   $ agent_vsock_port=1024
+
+   $ cargo run -- -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:${agent_vsock_port}" -c Check -c GetGuestDetails
    ```
 
    This examples makes two API calls:
@@ -86,6 +95,75 @@ $ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf
    - It then runs `GetGuestDetails` to establish some details of the
      environment the agent is running in.
 
+#### Cloud Hypervisor and Firecracker
+
+Cloud Hypervisor and Firecracker both use "hybrid VSOCK" which uses a local
+UNIX socket rather than the host kernel to handle communication with the
+guest. As such, you need to specify the path to the UNIX socket.
+
+Since the UNIX socket path is sandbox-specific, you need to run the
+`kata-runtime env` command to determine the socket's "template path". This
+path includes a `{ID}` tag that represents the real sandbox ID or name.
+
+Further, since the socket path is below the sandbox directory and since that
+directory is `root` owned, it is necessary to run the tool as `root` when
+using a Hybrid VSOCKS hypervisor.
+
+##### Determine socket path template value
+
+###### Configured hypervisor is Cloud Hypervisor
+
+```bash
+$ socket_path_template=$(sudo kata-runtime env --json | jq '.Hypervisor.SocketPath')
+$ echo "$socket_path_template"
+"/run/vc/vm/{ID}/clh.sock"
+```
+
+###### Configured hypervisor is Firecracker
+
+```bash
+$ socket_path_template=$(sudo kata-runtime env --json | jq '.Hypervisor.SocketPath')
+$ echo "$socket_path_template"
+"/run/vc/firecracker/{ID}/root/kata.hvsock"
+```
+
+> **Note:**
+>
+> Do not rely on the paths shown above: you should run the command yourself
+> as these paths _may_ change.
+
+Once you have determined the template path, build and install the tool to make
+it easier to run as the `root` user.
+
+##### Build and install
+
+```bash
+# Install for user
+$ make install
+
+# Install centrally
+$ sudo install -o root -g root -m 0755 ~/.cargo/bin/kata-agent-ctl /usr/local/bin
+```
+
+1. Start a Kata Container
+
+   Create a container called `foo`.
+
+1. Run the tool
+
+   ```bash
+   # Name of container
+   $ sandbox_id="foo"
+
+   # Create actual socket path
+   $ socket_path=$(echo "$socket_path_template" | sed "s/{ID}/${sandbox_id}/g" | tr -d '"')
+
+   $ sudo kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "unix://${socket_path}" --hybrid-vsock -c Check -c GetGuestDetails
+   ```
+
+   > **Note:** The `socket_path_template` variable was set in the
+   > [Determine socket path template value](#determine-socket-path-template-value) section.
+
 ### Run the tool and the agent in the same environment
 
 > **Warnings:**
@@ -100,6 +178,8 @@ $ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf
    $ sudo KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket target/x86_64-unknown-linux-musl/release/kata-agent
    ```
 
+   > **Note:** This example assumes an Intel x86-64 system.
+
 1. Run the tool in the same environment:
 
    ```sh

tools/agent-ctl/src/client.rs

@@ -17,6 +17,7 @@ use protocols::health_ttrpc::*;
 use slog::{debug, info};
 use std::io;
 use std::io::Write; // XXX: for flush()
+use std::io::{BufRead, BufReader};
 use std::os::unix::io::{IntoRawFd, RawFd};
 use std::os::unix::net::UnixStream;
 use std::thread::sleep;
@@ -87,11 +88,6 @@ static AGENT_CMDS: &'static [AgentCmd] = &[
         st: ServiceType::Agent,
         fp: agent_cmd_sandbox_add_arp_neighbors,
     },
-    AgentCmd {
-        name: "AddSwap",
-        st: ServiceType::Agent,
-        fp: agent_cmd_sandbox_add_swap,
-    },
     AgentCmd {
         name: "Check",
         st: ServiceType::Health,
@@ -372,7 +368,59 @@ fn client_create_vsock_fd(cid: libc::c_uint, port: u32) -> Result<RawFd> {
     Ok(fd)
 }
 
-fn create_ttrpc_client(server_address: String) -> Result<ttrpc::Client> {
+// Setup the existing stream by making a Hybrid VSOCK host-initiated
+// connection request to the Hybrid VSOCK-capable hypervisor (CLH or FC),
+// asking it to route the connection to the Kata Agent running inside the VM.
+fn setup_hybrid_vsock(mut stream: &UnixStream, hybrid_vsock_port: u64) -> Result<()> {
+    // Challenge message sent to the Hybrid VSOCK capable hypervisor asking
+    // for a connection to a real VSOCK server running in the VM on the
+    // port specified as part of this message.
+    const CONNECT_CMD: &str = "CONNECT";
+
+    // Expected response message returned by the Hybrid VSOCK capable
+    // hypervisor informing the client that the CONNECT_CMD was successful.
+    const OK_CMD: &str = "OK";
+
+    // Contact the agent by dialing it's port number and
+    // waiting for the hybrid vsock hypervisor to route the call for us ;)
+    //
+    // See: https://github.com/firecracker-microvm/firecracker/blob/main/docs/vsock.md#host-initiated-connections
+    let msg = format!("{} {}\n", CONNECT_CMD, hybrid_vsock_port);
+
+    stream.write_all(msg.as_bytes())?;
+
+    // Now, see if we get the expected response
+    let stream_reader = stream.try_clone()?;
+    let mut reader = BufReader::new(&stream_reader);
+
+    let mut msg = String::new();
+    reader.read_line(&mut msg)?;
+
+    if msg.starts_with(OK_CMD) {
+        let response = msg
+            .strip_prefix(OK_CMD)
+            .ok_or(format!("invalid response: {:?}", msg))
+            .map_err(|e| anyhow!(e))?
+            .trim();
+
+        debug!(sl!(), "Hybrid VSOCK host-side port: {:?}", response);
+    } else {
+        return Err(anyhow!(
+            "failed to setup Hybrid VSOCK connection: response was: {:?}",
+            msg
+        ));
+    }
+
+    // The Unix stream is now connected directly to the VSOCK socket
+    // the Kata agent is listening to in the VM.
+    Ok(())
+}
+
+fn create_ttrpc_client(
+    server_address: String,
+    hybrid_vsock_port: u64,
+    hybrid_vsock: bool,
+) -> Result<ttrpc::Client> {
     if server_address == "" {
         return Err(anyhow!("server address cannot be blank"));
     }
@@ -388,7 +436,7 @@ fn create_ttrpc_client(server_address: String) -> Result<ttrpc::Client> {
     let fd: RawFd = match scheme.as_str() {
         // Formats:
         //
-        // - "unix://absolute-path" (domain socket)
+        // - "unix://absolute-path" (domain socket, or hybrid vsock!)
         //   (example: "unix:///tmp/domain.socket")
         //
         // - "unix://@absolute-path" (abstract socket)
@@ -445,6 +493,10 @@ fn create_ttrpc_client(server_address: String) -> Result<ttrpc::Client> {
                     }
                 };
 
+                if hybrid_vsock {
+                    setup_hybrid_vsock(&stream, hybrid_vsock_port)?
+                }
+
                 stream.into_raw_fd()
             }
         }
@@ -481,14 +533,22 @@ fn create_ttrpc_client(server_address: String) -> Result<ttrpc::Client> {
     Ok(ttrpc::client::Client::new(fd))
 }
 
-fn kata_service_agent(server_address: String) -> Result<AgentServiceClient> {
+fn kata_service_agent(
-    let ttrpc_client = create_ttrpc_client(server_address)?;
+    server_address: String,
+    hybrid_vsock_port: u64,
+    hybrid_vsock: bool,
+) -> Result<AgentServiceClient> {
+    let ttrpc_client = create_ttrpc_client(server_address, hybrid_vsock_port, hybrid_vsock)?;
 
     Ok(AgentServiceClient::new(ttrpc_client))
 }
 
-fn kata_service_health(server_address: String) -> Result<HealthClient> {
+fn kata_service_health(
-    let ttrpc_client = create_ttrpc_client(server_address)?;
+    server_address: String,
+    hybrid_vsock_port: u64,
+    hybrid_vsock: bool,
+) -> Result<HealthClient> {
+    let ttrpc_client = create_ttrpc_client(server_address, hybrid_vsock_port, hybrid_vsock)?;
 
     Ok(HealthClient::new(ttrpc_client))
 }
@@ -522,8 +582,17 @@ pub fn client(cfg: &Config, commands: Vec<&str>) -> Result<()> {
 
     // Create separate connections for each of the services provided
     // by the agent.
-    let client = kata_service_agent(cfg.server_address.clone())?;
+    let client = kata_service_agent(
-    let health = kata_service_health(cfg.server_address.clone())?;
+        cfg.server_address.clone(),
+        cfg.hybrid_vsock_port,
+        cfg.hybrid_vsock,
+    )?;
+
+    let health = kata_service_health(
+        cfg.server_address.clone(),
+        cfg.hybrid_vsock_port,
+        cfg.hybrid_vsock,
+    )?;
 
     let mut options = Options::new();
 
@@ -1923,29 +1992,3 @@ fn get_repeat_count(cmdline: &str) -> i64 {
         Err(_) => return default_repeat_count,
     }
 }
-
-fn agent_cmd_sandbox_add_swap(
-    ctx: &Context,
-    client: &AgentServiceClient,
-    _health: &HealthClient,
-    _options: &mut Options,
-    _args: &str,
-) -> Result<()> {
-    let req = AddSwapRequest::default();
-
-    let ctx = clone_context(ctx);
-
-    debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
-
-    let reply = client
-        .add_swap(ctx, &req)
-        .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?;
-
-    // FIXME: Implement 'AddSwap' fully.
-    eprintln!("FIXME: 'AddSwap' not fully implemented");
-
-    info!(sl!(), "response received";
-        "response" => format!("{:?}", reply));
-
-    Ok(())
-}

tools/agent-ctl/src/main.rs

@@ -6,6 +6,7 @@
 #[macro_use]
 extern crate lazy_static;
 
+use crate::types::Config;
 use anyhow::{anyhow, Result};
 use clap::{crate_name, crate_version, App, Arg, SubCommand};
 use std::io;
@@ -39,6 +40,9 @@ const WARNING_TEXT: &str = r#"WARNING:
     irrevocably other parts of the system or even kill a running container or
     sandbox."#;
 
+// The VSOCK port number the Kata agent uses to listen to API requests on.
+const DEFAULT_KATA_AGENT_API_VSOCK_PORT: &str = "1024";
+
 fn make_examples_text(program_name: &str) -> String {
     let abstract_server_address = "unix://@/foo/bar/abstract.socket";
     let bundle = "$bundle_dir";
@@ -47,6 +51,7 @@ fn make_examples_text(program_name: &str) -> String {
     let local_server_address = "unix:///tmp/local.socket";
     let sandbox_id = "$sandbox_id";
     let vsock_server_address = "vsock://3:1024";
+    let hybrid_vsock_server_address = "unix:///run/vc/vm/foo/clh.sock";
 
     format!(
         r#"EXAMPLES:
@@ -55,6 +60,10 @@ fn make_examples_text(program_name: &str) -> String {
 
   $ {program} connect --server-address "{vsock_server_address}" --cmd Check
 
+- Connect to the agent using a Hybrid VSOCK hypervisor (here Cloud Hypervisor):
+
+  $ {program} connect --server-address "{hybrid_vsock_server_address}" --hybrid-vsock --cmd Check
+
 - Connect to the agent using local sockets (when running in same environment as the agent):
 
   # Local socket
@@ -109,6 +118,7 @@ fn make_examples_text(program_name: &str) -> String {
         program = program_name,
         sandbox_id = sandbox_id,
         vsock_server_address = vsock_server_address,
+        hybrid_vsock_server_address = hybrid_vsock_server_address,
     )
 }
 
@@ -124,7 +134,8 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> {
     let server_address = args
         .value_of("server-address")
         .ok_or("need server adddress".to_string())
-        .map_err(|e| anyhow!(e))?;
+        .map_err(|e| anyhow!(e))?
+        .to_string();
 
     let mut commands: Vec<&str> = Vec::new();
 
@@ -149,17 +160,28 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> {
         None => 0,
     };
 
-    let bundle_dir = args.value_of("bundle-dir").unwrap_or("");
+    let hybrid_vsock_port: u64 = args
+        .value_of("hybrid-vsock-port")
+        .ok_or("Need Hybrid VSOCK port number")
+        .map(|p| p.parse::<u64>().unwrap())
+        .map_err(|e| anyhow!("VSOCK port number must be an integer: {:?}", e))?;
 
-    let result = rpc::run(
+    let bundle_dir = args.value_of("bundle-dir").unwrap_or("").to_string();
-        &logger,
+
+    let hybrid_vsock = args.is_present("hybrid-vsock");
+
+    let cfg = Config {
         server_address,
         bundle_dir,
         interactive,
         ignore_errors,
         timeout_nano,
-        commands,
+        hybrid_vsock_port,
-    );
+        hybrid_vsock,
+    };
+
+    let result = rpc::run(&logger, &cfg, commands);
+
     if result.is_err() {
         return result;
     }
@@ -170,6 +192,11 @@ fn connect(name: &str, global_args: clap::ArgMatches) -> Result<()> {
 fn real_main() -> Result<()> {
     let name = crate_name!();
 
+    let hybrid_vsock_port_help = format!(
+        "Kata agent VSOCK port number (only useful with --hybrid-vsock) [default: {}]",
+        DEFAULT_KATA_AGENT_API_VSOCK_PORT
+    );
+
     let app = App::new(name)
         .version(crate_version!())
         .about(ABOUT_TEXT)
@@ -209,6 +236,19 @@ fn real_main() -> Result<()> {
                     .long("ignore-errors")
                     .help("Don't exit on first error"),
                     )
+                .arg(
+                    Arg::with_name("hybrid-vsock")
+                    .long("hybrid-vsock")
+                    .help("Treat a unix:// server address as a Hybrid VSOCK one"),
+                    )
+                .arg(
+                    Arg::with_name("hybrid-vsock-port")
+                    .long("hybrid-vsock-port")
+                    .help(&hybrid_vsock_port_help)
+                    .default_value(DEFAULT_KATA_AGENT_API_VSOCK_PORT)
+                    .takes_value(true)
+                    .value_name("PORT")
+                    )
                 .arg(
                     Arg::with_name("interactive")
                     .short("i")

tools/agent-ctl/src/rpc.rs

@@ -11,25 +11,9 @@ use slog::{o, Logger};
 use crate::client::client;
 use crate::types::Config;
 
-pub fn run(
+pub fn run(logger: &Logger, cfg: &Config, commands: Vec<&str>) -> Result<()> {
-    logger: &Logger,
-    server_address: &str,
-    bundle_dir: &str,
-    interactive: bool,
-    ignore_errors: bool,
-    timeout_nano: i64,
-    commands: Vec<&str>,
-) -> Result<()> {
-    let cfg = Config {
-        server_address: server_address.to_string(),
-        bundle_dir: bundle_dir.to_string(),
-        timeout_nano: timeout_nano,
-        interactive: interactive,
-        ignore_errors: ignore_errors,
-    };
-
     // Maintain the global logger for the duration of the ttRPC comms
     let _guard = slog_scope::set_global_logger(logger.new(o!("subsystem" => "rpc")));
 
-    client(&cfg, commands)
+    client(cfg, commands)
 }

tools/agent-ctl/src/types.rs

@@ -14,6 +14,8 @@ pub struct Config {
     pub server_address: String,
     pub bundle_dir: String,
     pub timeout_nano: i64,
+    pub hybrid_vsock_port: u64,
     pub interactive: bool,
+    pub hybrid_vsock: bool,
     pub ignore_errors: bool,
 }