agent: launch confidential-data-hub

confidential-data-hub depends attestation-agent, and
confidential-data-hab need to start before rpc server, so move the
function 'init_attestation_agent' from image_rpc.rs to main.rs and
launch confidential-data-hub after 'init_attestation_agent'.

Fixes: #7544

Signed-off-by: Biao Lu <biao.lu@intel.com>

src/agent/Cargo.lock

@@ -718,6 +718,26 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913"
 
+[[package]]
+name = "const_format"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48"
+dependencies = [
+ "const_format_proc_macros",
+]
+
+[[package]]
+name = "const_format_proc_macros"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -2103,6 +2123,7 @@ dependencies = [
  "cfg-if 1.0.0",
  "cgroups-rs",
  "clap",
+ "const_format",
  "futures",
  "image-rs",
  "ipnetwork",

src/agent/Cargo.toml

@@ -23,6 +23,7 @@ regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
+const_format = "0.2.30"
 url = "2.2.2"
 
 # Async helpers

src/agent/src/image_rpc.rs

@@ -9,8 +9,7 @@ use std::collections::HashMap;
 use std::env;
 use std::fs;
 use std::path::Path;
-use std::process::Command;
-use std::sync::atomic::{AtomicBool, AtomicU16, Ordering};
+use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
@@ -26,14 +25,6 @@ use crate::AGENT_CONFIG;
 // A marker to merge container spec for images pulled inside guest.
 const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
 
-const AA_PATH: &str = "/usr/local/bin/attestation-agent";
-
-const AA_KEYPROVIDER_URI: &str =
-    "unix:///run/confidential-containers/attestation-agent/keyprovider.sock";
-const AA_GETRESOURCE_URI: &str =
-    "unix:///run/confidential-containers/attestation-agent/getresource.sock";
-
-const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
 // kata rootfs is readonly, use tmpfs before CC storage is implemented.
 const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
 const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
@@ -51,7 +42,6 @@ fn sl() -> slog::Logger {
 
 #[derive(Clone)]
 pub struct ImageService {
-    attestation_agent_started: Arc<AtomicBool>,
     image_client: Arc<Mutex<ImageClient>>,
     images: Arc<Mutex<HashMap<String, String>>>,
     container_count: Arc<AtomicU16>,
@@ -75,7 +65,6 @@ impl ImageService {
         }
 
         Self {
-            attestation_agent_started: Arc::new(AtomicBool::new(false)),
             image_client: Arc::new(Mutex::new(image_client)),
             images: Arc::new(Mutex::new(HashMap::new())),
             container_count: Arc::new(AtomicU16::new(0)),
@@ -117,36 +106,6 @@ impl ImageService {
         Ok(())
     }
 
-    // If we fail to start the AA, ocicrypt won't be able to unwrap keys
-    // and container decryption will fail.
-    fn init_attestation_agent() -> Result<()> {
-        let config_path = OCICRYPT_CONFIG_PATH;
-
-        // The image will need to be encrypted using a keyprovider
-        // that has the same name (at least according to the config).
-        let ocicrypt_config = serde_json::json!({
-            "key-providers": {
-                "attestation-agent":{
-                    "ttrpc":AA_KEYPROVIDER_URI
-                }
-            }
-        });
-
-        fs::write(config_path, ocicrypt_config.to_string().as_bytes())?;
-
-        env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path);
-
-        // The Attestation Agent will run for the duration of the guest.
-        Command::new(AA_PATH)
-            .arg("--keyprovider_sock")
-            .arg(AA_KEYPROVIDER_URI)
-            .arg("--getresource_sock")
-            .arg(AA_GETRESOURCE_URI)
-            .spawn()?;
-
-        Ok(())
-    }
-
     /// Determines the container id (cid) to use for a given request.
     ///
     /// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
@@ -188,17 +147,6 @@ impl ImageService {
         }
 
         let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
-        if !aa_kbc_params.is_empty() {
-            match self.attestation_agent_started.compare_exchange_weak(
-                false,
-                true,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => Self::init_attestation_agent()?,
-                Err(_) => info!(sl(), "Attestation Agent already running"),
-            }
-        }
         // If the attestation-agent is being used, then enable the authenticated credentials support
         info!(
             sl(),

src/agent/src/main.rs

@@ -22,6 +22,7 @@ extern crate slog;
 use anyhow::{anyhow, Context, Result};
 use cfg_if::cfg_if;
 use clap::{AppSettings, Parser};
+use const_format::concatcp;
 use nix::fcntl::OFlag;
 use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
 use nix::unistd::{self, dup, Pid};
@@ -32,6 +33,7 @@ use std::os::unix::fs as unixfs;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
 use std::process::exit;
+use std::process::Command;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
@@ -84,6 +86,26 @@ cfg_if! {
 
 const NAME: &str = "kata-agent";
 
+const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
+const AA_PATH: &str = "/usr/local/bin/attestation-agent";
+const AA_UNIX_SOCKET_DIR: &str = "/run/confidential-containers/attestation-agent/";
+const UNIX_SOCKET_PREFIX: &str = "unix://";
+const AA_KEYPROVIDER_URI: &str =
+    concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "keyprovider.sock");
+const AA_GETRESOURCE_URI: &str =
+    concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "getresource.sock");
+const AA_ATTESTATION_SOCKET: &str = concatcp!(AA_UNIX_SOCKET_DIR, "attestation-agent.sock");
+const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SOCKET);
+
+const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
+
+cfg_if! {
+    if #[cfg(feature = "confidential-data-hub")] {
+        const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub";
+        const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock";
+    }
+}
+
 lazy_static! {
     static ref AGENT_CONFIG: AgentConfig =
         // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
@@ -345,6 +367,10 @@ async fn start_sandbox(
     let (tx, rx) = tokio::sync::oneshot::channel();
     sandbox.lock().await.sender = Some(tx);
 
+    if !config.aa_kbc_params.is_empty() {
+        init_attestation_agent(logger)?;
+    }
+
     // vsock:///dev/vsock, port
     let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
     server.start().await?;
@@ -355,6 +381,100 @@ async fn start_sandbox(
     Ok(())
 }
 
+// If we fail to start the AA, ocicrypt won't be able to unwrap keys
+// and container decryption will fail.
+fn init_attestation_agent(logger: &Logger) -> Result<()> {
+    let config_path = OCICRYPT_CONFIG_PATH;
+
+    // The image will need to be encrypted using a keyprovider
+    // that has the same name (at least according to the config).
+    let ocicrypt_config = serde_json::json!({
+        "key-providers": {
+            "attestation-agent":{
+                "ttrpc":AA_KEYPROVIDER_URI
+            }
+        }
+    });
+
+    fs::write(config_path, ocicrypt_config.to_string().as_bytes())?;
+
+    env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path);
+
+    // The Attestation Agent will run for the duration of the guest.
+    launch_process(
+        logger,
+        AA_PATH,
+        &vec![
+            "--keyprovider_sock",
+            AA_KEYPROVIDER_URI,
+            "--getresource_sock",
+            AA_GETRESOURCE_URI,
+            "--attestation_sock",
+            AA_ATTESTATION_URI,
+        ],
+        AA_ATTESTATION_SOCKET,
+        DEFAULT_LAUNCH_PROCESS_TIMEOUT,
+    )
+    .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?;
+
+    #[cfg(feature = "confidential-data-hub")]
+    {
+        if let Err(e) = launch_process(
+            logger,
+            CDH_PATH,
+            &vec![],
+            CDH_SOCKET,
+            DEFAULT_LAUNCH_PROCESS_TIMEOUT,
+        ) {
+            error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e);
+        }
+    }
+
+    Ok(())
+}
+
+fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
+    let p = Path::new(path);
+    let mut attempts = 0;
+    loop {
+        std::thread::sleep(std::time::Duration::from_secs(1));
+        if p.exists() {
+            return Ok(());
+        }
+        if attempts >= timeout_secs {
+            break;
+        }
+        attempts += 1;
+        info!(
+            logger,
+            "waiting for {} to exist (attempts={})", path, attempts
+        );
+    }
+
+    Err(anyhow!("wait for {} to exist timeout.", path))
+}
+
+fn launch_process(
+    logger: &Logger,
+    path: &str,
+    args: &Vec<&str>,
+    unix_socket_path: &str,
+    timeout_secs: i32,
+) -> Result<()> {
+    if !Path::new(path).exists() {
+        return Err(anyhow!("path {} does not exist.", path));
+    }
+    if !unix_socket_path.is_empty() && Path::new(unix_socket_path).exists() {
+        fs::remove_file(unix_socket_path)?;
+    }
+    Command::new(path).args(args).spawn()?;
+    if !unix_socket_path.is_empty() && timeout_secs > 0 {
+        wait_for_path_to_exist(logger, unix_socket_path, timeout_secs)?;
+    }
+
+    Ok(())
+}
+
 // init_agent_as_init will do the initializations such as setting up the rootfs
 // when this agent has been run as the init process.
 fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<()> {