From 4e3a1ebcaf6e45184df5a38ce4b663944334212b Mon Sep 17 00:00:00 2001 From: Biao Lu Date: Tue, 1 Aug 2023 22:24:52 +0800 Subject: [PATCH 01/13] protocols: add support sealed_secret To call CDH ttrpc API, 'unseal_secret' for 'sealed_secret', add protocol file and generate ttrpc code. Fixes: #7544 Signed-off-by: Biao Lu --- src/libs/protocols/Cargo.toml | 1 + src/libs/protocols/build.rs | 9 +++++++ src/libs/protocols/protos/sealed_secret.proto | 27 +++++++++++++++++++ src/libs/protocols/src/lib.rs | 7 +++++ 4 files changed, 44 insertions(+) create mode 100644 src/libs/protocols/protos/sealed_secret.proto diff --git a/src/libs/protocols/Cargo.toml b/src/libs/protocols/Cargo.toml index 9c0033d17e..eb20a9c772 100644 --- a/src/libs/protocols/Cargo.toml +++ b/src/libs/protocols/Cargo.toml @@ -9,6 +9,7 @@ license = "Apache-2.0" default = [] with-serde = [ "serde", "serde_json" ] async = ["ttrpc/async", "async-trait"] +sealed-secret = [] [dependencies] ttrpc = { version = "0.7.1" } diff --git a/src/libs/protocols/build.rs b/src/libs/protocols/build.rs index af0dc691ea..ac7fb5cbdc 100644 --- a/src/libs/protocols/build.rs +++ b/src/libs/protocols/build.rs @@ -204,6 +204,8 @@ fn real_main() -> Result<(), std::io::Error> { "protos/agent.proto", "protos/health.proto", "protos/image.proto", + #[cfg(feature = "sealed-secret")] + "protos/sealed_secret.proto", ], true, )?; @@ -211,6 +213,11 @@ fn real_main() -> Result<(), std::io::Error> { fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?; fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?; fs::rename("src/image_ttrpc.rs", "src/image_ttrpc_async.rs")?; + #[cfg(feature = "sealed-secret")] + fs::rename( + "src/sealed_secret_ttrpc.rs", + "src/sealed_secret_ttrpc_async.rs", + )?; } codegen( @@ -219,6 +226,8 @@ fn real_main() -> Result<(), std::io::Error> { "protos/agent.proto", "protos/health.proto", "protos/image.proto", + #[cfg(feature = "sealed-secret")] + "protos/sealed_secret.proto", ], false, )?; diff --git a/src/libs/protocols/protos/sealed_secret.proto b/src/libs/protocols/protos/sealed_secret.proto new file mode 100644 index 0000000000..cba1382873 --- /dev/null +++ b/src/libs/protocols/protos/sealed_secret.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package api; + +message UnsealSecretInput { + bytes secret = 1; +} + +message UnsealSecretOutput { + bytes plaintext = 1; +} + +message GetResourceRequest { + string ResourcePath = 1; +} + +message GetResourceResponse { + bytes Resource = 1; +} + +service SealedSecretService { + rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {}; +} + +service GetResourceService { + rpc GetResource(GetResourceRequest) returns (GetResourceResponse) {}; +} diff --git a/src/libs/protocols/src/lib.rs b/src/libs/protocols/src/lib.rs index 0fe254704e..71f16116b0 100644 --- a/src/libs/protocols/src/lib.rs +++ b/src/libs/protocols/src/lib.rs @@ -31,3 +31,10 @@ pub use serde_config::{ deserialize_enum_or_unknown, deserialize_message_field, serialize_enum_or_unknown, serialize_message_field, }; + +#[cfg(feature = "sealed-secret")] +pub mod sealed_secret; +#[cfg(feature = "sealed-secret")] +pub mod sealed_secret_ttrpc; +#[cfg(all(feature = "sealed-secret", feature = "async"))] +pub mod sealed_secret_ttrpc_async; From acd0a75efd111d75088200021f4a57ef237e4506 Mon Sep 17 00:00:00 2001 From: Biao Lu Date: Thu, 10 Aug 2023 14:08:31 +0800 Subject: [PATCH 02/13] agent: rootfs: add sealed-secret in Makefile When set SEALED_SECRET to "yes", the kata-agent is built with sealed-secret capability, default value is "no". Fixes: #7544 Signed-off-by: Biao Lu --- src/agent/Makefile | 7 +++++++ tools/osbuilder/rootfs-builder/rootfs.sh | 9 ++++++++- .../kata-deploy/local-build/kata-deploy-binaries.sh | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/agent/Makefile b/src/agent/Makefile index ba065b4d04..a3eb567059 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -33,6 +33,13 @@ ifeq ($(SECCOMP),yes) override EXTRA_RUSTFEATURES += seccomp endif +SEALED_SECRET ?= no + +# Enable sealed-secret feature of rust build +ifeq ($(SEALED_SECRET),yes) + override EXTRA_RUSTFEATURES += sealed-secret +endif + include ../../utils.mk ifeq ($(ARCH), ppc64le) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 92e1632419..538df325ce 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -28,6 +28,8 @@ LIBC=${LIBC:-musl} # The kata agent enables seccomp feature. # However, it is not enforced by default: you need to enable that in the main configuration file. SECCOMP=${SECCOMP:-"yes"} +# The kata agent enables sealed-secret feature. +SEALED_SECRET=${SEALED_SECRET:-"no"} SELINUX=${SELINUX:-"no"} lib_file="${script_dir}/../scripts/lib.sh" @@ -156,6 +158,10 @@ ROOTFS_DIR Path to the directory that is populated with the rootfs. SECCOMP When set to "no", the kata-agent is built without seccomp capability. Default value: "yes" +SEALED_SECRET When set to "yes", the kata-agent is built with sealed-secret + capability. + Default value: "no" + SELINUX When set to "yes", build the rootfs with the required packages to enable SELinux in the VM. Make sure the guest kernel is compiled with SELinux enabled. @@ -469,6 +475,7 @@ build_rootfs_distro() --env INSIDE_CONTAINER=1 \ --env AA_KBC="${AA_KBC}" \ --env SECCOMP="${SECCOMP}" \ + --env SEALED_SECRET="${SEALED_SECRET}" \ --env SELINUX="${SELINUX}" \ --env DEBUG="${DEBUG}" \ --env HOME="/root" \ @@ -630,7 +637,7 @@ EOF git checkout "${AGENT_VERSION}" && OK "git checkout successful" || die "checkout agent ${AGENT_VERSION} failed!" fi make clean - make LIBC=${LIBC} INIT=${AGENT_INIT} SECCOMP=${SECCOMP} + make LIBC=${LIBC} INIT=${AGENT_INIT} SECCOMP=${SECCOMP} SEALED_SECRET=${SEALED_SECRET} make install DESTDIR="${ROOTFS_DIR}" LIBC=${LIBC} INIT=${AGENT_INIT} if [ "${SECCOMP}" == "yes" ]; then rm -rf "${libseccomp_install_dir}" "${gperf_install_dir}" diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 8d1835bdad..afdb149861 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -228,6 +228,7 @@ install_cc_image() { export KATA_BUILD_CC=yes export MEASURED_ROOTFS=yes export DM_VERITY=yes + export SEALED_SECRET=yes variant="${1:-}" install_image "${variant}" From b4092023bf65ea5109f820e5eef582978c23b6e1 Mon Sep 17 00:00:00 2001 From: Biao Lu Date: Thu, 17 Aug 2023 17:07:48 +0800 Subject: [PATCH 03/13] osbuilder: add confidential-data-hub in rootfs Fixes: #7544 Signed-off-by: Biao Lu --- tools/osbuilder/rootfs-builder/rootfs.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 538df325ce..127f37dafa 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -30,6 +30,7 @@ LIBC=${LIBC:-musl} SECCOMP=${SECCOMP:-"yes"} # The kata agent enables sealed-secret feature. SEALED_SECRET=${SEALED_SECRET:-"no"} +CDH_RESOURCE_PROVIDER=${CDH_RESOURCE_PROVIDER:-"kbs"} SELINUX=${SELINUX:-"no"} lib_file="${script_dir}/../scripts/lib.sh" @@ -699,6 +700,11 @@ EOF make KBC=${AA_KBC} ttrpc=true make install DESTDIR="${ROOTFS_DIR}/usr/local/bin/" popd + + pushd guest-components/confidential-data-hub + make RESOURCE_PROVIDER=${CDH_RESOURCE_PROVIDER} + make install DESTDIR="${ROOTFS_DIR}/usr/local/bin/" + popd fi if [ "${KATA_BUILD_CC}" == "yes" ]; then From 5316839165e25c04639b22cd31e3608624ba3004 Mon Sep 17 00:00:00 2001 From: Biao Lu Date: Wed, 23 Aug 2023 14:26:48 +0800 Subject: [PATCH 04/13] agent: launch confidential-data-hub confidential-data-hub depends attestation-agent, and confidential-data-hab need to start before rpc server, so move the function 'init_attestation_agent' from image_rpc.rs to main.rs and launch confidential-data-hub after 'init_attestation_agent'. Fixes: #7544 Signed-off-by: Biao Lu --- src/agent/Cargo.lock | 21 +++++++ src/agent/Cargo.toml | 1 + src/agent/src/image_rpc.rs | 54 +---------------- src/agent/src/main.rs | 120 +++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 53 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 93d40e173a..fa8a6f392e 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -718,6 +718,26 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" +[[package]] +name = "const_format" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -2103,6 +2123,7 @@ dependencies = [ "cfg-if 1.0.0", "cgroups-rs", "clap", + "const_format", "futures", "image-rs", "ipnetwork", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index bb72a5fa9c..be57aaab6d 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -23,6 +23,7 @@ regex = "1.5.6" serial_test = "0.5.1" kata-sys-util = { path = "../libs/kata-sys-util" } kata-types = { path = "../libs/kata-types" } +const_format = "0.2.30" url = "2.2.2" # Async helpers diff --git a/src/agent/src/image_rpc.rs b/src/agent/src/image_rpc.rs index 411e1de091..1915a9f4fa 100644 --- a/src/agent/src/image_rpc.rs +++ b/src/agent/src/image_rpc.rs @@ -9,8 +9,7 @@ use std::collections::HashMap; use std::env; use std::fs; use std::path::Path; -use std::process::Command; -use std::sync::atomic::{AtomicBool, AtomicU16, Ordering}; +use std::sync::atomic::{AtomicU16, Ordering}; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; @@ -26,14 +25,6 @@ use crate::AGENT_CONFIG; // A marker to merge container spec for images pulled inside guest. const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name"; -const AA_PATH: &str = "/usr/local/bin/attestation-agent"; - -const AA_KEYPROVIDER_URI: &str = - "unix:///run/confidential-containers/attestation-agent/keyprovider.sock"; -const AA_GETRESOURCE_URI: &str = - "unix:///run/confidential-containers/attestation-agent/getresource.sock"; - -const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json"; // kata rootfs is readonly, use tmpfs before CC storage is implemented. const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/"; const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle"; @@ -51,7 +42,6 @@ fn sl() -> slog::Logger { #[derive(Clone)] pub struct ImageService { - attestation_agent_started: Arc, image_client: Arc>, images: Arc>>, container_count: Arc, @@ -75,7 +65,6 @@ impl ImageService { } Self { - attestation_agent_started: Arc::new(AtomicBool::new(false)), image_client: Arc::new(Mutex::new(image_client)), images: Arc::new(Mutex::new(HashMap::new())), container_count: Arc::new(AtomicU16::new(0)), @@ -117,36 +106,6 @@ impl ImageService { Ok(()) } - // If we fail to start the AA, ocicrypt won't be able to unwrap keys - // and container decryption will fail. - fn init_attestation_agent() -> Result<()> { - let config_path = OCICRYPT_CONFIG_PATH; - - // The image will need to be encrypted using a keyprovider - // that has the same name (at least according to the config). - let ocicrypt_config = serde_json::json!({ - "key-providers": { - "attestation-agent":{ - "ttrpc":AA_KEYPROVIDER_URI - } - } - }); - - fs::write(config_path, ocicrypt_config.to_string().as_bytes())?; - - env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path); - - // The Attestation Agent will run for the duration of the guest. - Command::new(AA_PATH) - .arg("--keyprovider_sock") - .arg(AA_KEYPROVIDER_URI) - .arg("--getresource_sock") - .arg(AA_GETRESOURCE_URI) - .spawn()?; - - Ok(()) - } - /// Determines the container id (cid) to use for a given request. /// /// If the request specifies a non-empty id, use it; otherwise derive it from the image path. @@ -188,17 +147,6 @@ impl ImageService { } let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params; - if !aa_kbc_params.is_empty() { - match self.attestation_agent_started.compare_exchange_weak( - false, - true, - Ordering::SeqCst, - Ordering::SeqCst, - ) { - Ok(_) => Self::init_attestation_agent()?, - Err(_) => info!(sl(), "Attestation Agent already running"), - } - } // If the attestation-agent is being used, then enable the authenticated credentials support info!( sl(), diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 7e59e2daab..6d63ab428f 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -22,6 +22,7 @@ extern crate slog; use anyhow::{anyhow, Context, Result}; use cfg_if::cfg_if; use clap::{AppSettings, Parser}; +use const_format::concatcp; use nix::fcntl::OFlag; use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr}; use nix::unistd::{self, dup, Pid}; @@ -32,6 +33,7 @@ use std::os::unix::fs as unixfs; use std::os::unix::io::AsRawFd; use std::path::Path; use std::process::exit; +use std::process::Command; use std::sync::Arc; use tracing::{instrument, span}; @@ -84,6 +86,26 @@ cfg_if! { const NAME: &str = "kata-agent"; +const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json"; +const AA_PATH: &str = "/usr/local/bin/attestation-agent"; +const AA_UNIX_SOCKET_DIR: &str = "/run/confidential-containers/attestation-agent/"; +const UNIX_SOCKET_PREFIX: &str = "unix://"; +const AA_KEYPROVIDER_URI: &str = + concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "keyprovider.sock"); +const AA_GETRESOURCE_URI: &str = + concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "getresource.sock"); +const AA_ATTESTATION_SOCKET: &str = concatcp!(AA_UNIX_SOCKET_DIR, "attestation-agent.sock"); +const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SOCKET); + +const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6; + +cfg_if! { + if #[cfg(feature = "confidential-data-hub")] { + const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub"; + const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock"; + } +} + lazy_static! { static ref AGENT_CONFIG: AgentConfig = // Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig @@ -345,6 +367,10 @@ async fn start_sandbox( let (tx, rx) = tokio::sync::oneshot::channel(); sandbox.lock().await.sender = Some(tx); + if !config.aa_kbc_params.is_empty() { + init_attestation_agent(logger)?; + } + // vsock:///dev/vsock, port let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?; server.start().await?; @@ -355,6 +381,100 @@ async fn start_sandbox( Ok(()) } +// If we fail to start the AA, ocicrypt won't be able to unwrap keys +// and container decryption will fail. +fn init_attestation_agent(logger: &Logger) -> Result<()> { + let config_path = OCICRYPT_CONFIG_PATH; + + // The image will need to be encrypted using a keyprovider + // that has the same name (at least according to the config). + let ocicrypt_config = serde_json::json!({ + "key-providers": { + "attestation-agent":{ + "ttrpc":AA_KEYPROVIDER_URI + } + } + }); + + fs::write(config_path, ocicrypt_config.to_string().as_bytes())?; + + env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path); + + // The Attestation Agent will run for the duration of the guest. + launch_process( + logger, + AA_PATH, + &vec![ + "--keyprovider_sock", + AA_KEYPROVIDER_URI, + "--getresource_sock", + AA_GETRESOURCE_URI, + "--attestation_sock", + AA_ATTESTATION_URI, + ], + AA_ATTESTATION_SOCKET, + DEFAULT_LAUNCH_PROCESS_TIMEOUT, + ) + .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?; + + #[cfg(feature = "confidential-data-hub")] + { + if let Err(e) = launch_process( + logger, + CDH_PATH, + &vec![], + CDH_SOCKET, + DEFAULT_LAUNCH_PROCESS_TIMEOUT, + ) { + error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e); + } + } + + Ok(()) +} + +fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> { + let p = Path::new(path); + let mut attempts = 0; + loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + if p.exists() { + return Ok(()); + } + if attempts >= timeout_secs { + break; + } + attempts += 1; + info!( + logger, + "waiting for {} to exist (attempts={})", path, attempts + ); + } + + Err(anyhow!("wait for {} to exist timeout.", path)) +} + +fn launch_process( + logger: &Logger, + path: &str, + args: &Vec<&str>, + unix_socket_path: &str, + timeout_secs: i32, +) -> Result<()> { + if !Path::new(path).exists() { + return Err(anyhow!("path {} does not exist.", path)); + } + if !unix_socket_path.is_empty() && Path::new(unix_socket_path).exists() { + fs::remove_file(unix_socket_path)?; + } + Command::new(path).args(args).spawn()?; + if !unix_socket_path.is_empty() && timeout_secs > 0 { + wait_for_path_to_exist(logger, unix_socket_path, timeout_secs)?; + } + + Ok(()) +} + // init_agent_as_init will do the initializations such as setting up the rootfs // when this agent has been run as the init process. fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<()> { From 75def881e5bc64576c6d8f94d8947816dd9406f3 Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Mon, 7 Aug 2023 19:07:03 +0800 Subject: [PATCH 05/13] agent: support sealed secret as env in kata Fixes: #7555 Signed-off-by: Linda Yu --- src/agent/Cargo.lock | 1 + src/agent/Cargo.toml | 1 + src/agent/src/cdh.rs | 66 +++++++++++++++++++++++++++++++++++++++++++ src/agent/src/main.rs | 2 ++ src/agent/src/rpc.rs | 33 ++++++++++++++++++++-- 5 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 src/agent/src/cdh.rs diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index fa8a6f392e..1bd9336229 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -2124,6 +2124,7 @@ dependencies = [ "cgroups-rs", "clap", "const_format", + "derivative", "futures", "image-rs", "ipnetwork", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index be57aaab6d..2feb4c7c0e 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -25,6 +25,7 @@ kata-sys-util = { path = "../libs/kata-sys-util" } kata-types = { path = "../libs/kata-types" } const_format = "0.2.30" url = "2.2.2" +derivative = "2.2.0" # Async helpers async-trait = "0.1.42" diff --git a/src/agent/src/cdh.rs b/src/agent/src/cdh.rs new file mode 100644 index 0000000000..c82acdebcd --- /dev/null +++ b/src/agent/src/cdh.rs @@ -0,0 +1,66 @@ +// Copyright (c) 2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// Confidential Data Hub client wrapper. +// Confidential Data Hub is a service running inside guest to provide resource related APIs. +// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub + +use anyhow::{anyhow, Result}; +use protocols::{ + sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient, +}; +const CDH_ADDR: &str = "unix:///run/confidential-containers/cdh.sock"; + +#[derive(Clone)] +pub struct CDHClient { + sealed_secret_client: SealedSecretServiceClient, +} + +impl CDHClient { + pub fn new() -> Result { + let c = ttrpc::asynchronous::Client::connect(CDH_ADDR)?; + let ssclient = sealed_secret_ttrpc_async::SealedSecretServiceClient::new(c); + Ok(CDHClient { + sealed_secret_client: ssclient, + }) + } + + pub async fn unseal_secret_async( + &self, + sealed: &str, + ) -> Result { + let secret = sealed + .strip_prefix("sealed.") + .ok_or(anyhow!("strip_prefix sealed. failed"))?; + let mut input = sealed_secret::UnsealSecretInput::new(); + input.set_secret(secret.into()); + let unseal = self + .sealed_secret_client + .unseal_secret( + ttrpc::context::with_timeout(50 * 1000 * 1000 * 1000), + &input, + ) + .await?; + Ok(unseal) + } + + pub async fn unseal_env(&self, env: &str) -> Result { + let (key, value) = env.split_once('=').unwrap(); + if value.starts_with("sealed.") { + let unsealed_value = self.unseal_secret_async(value).await; + match unsealed_value { + Ok(v) => { + let plain_env = + format!("{}={}", key, std::str::from_utf8(&v.plaintext).unwrap()); + return Ok(plain_env); + } + Err(e) => { + return Err(e); + } + }; + } + Ok((*env.to_owned()).to_string()) + } +} diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 6d63ab428f..0761e230f3 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -37,6 +37,8 @@ use std::process::Command; use std::sync::Arc; use tracing::{instrument, span}; +#[cfg(feature = "confidential-data-hub")] +mod cdh; mod config; mod console; mod device; diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 6db6bbcc0b..654edf1657 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -71,6 +71,7 @@ use crate::AGENT_CONFIG; use crate::trace_rpc_call; use crate::tracer::extract_carrier_from_ttrpc; +use derivative::Derivative; use opentelemetry::global; use tracing::span; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -89,6 +90,9 @@ use std::io::{BufRead, BufReader, Write}; use std::os::unix::fs::FileExt; use std::path::PathBuf; +#[cfg(feature = "sealed-secret")] +use crate::cdh::CDHClient; + pub const CONTAINER_BASE: &str = "/run/kata-containers"; const MODPROBE_PATH: &str = "/sbin/modprobe"; const CONFIG_JSON: &str = "config.json"; @@ -139,10 +143,14 @@ fn is_allowed(req: &impl MessageDyn) -> ttrpc::Result<()> { } } -#[derive(Clone, Debug)] +#[derive(Derivative)] +#[derivative(Clone, Debug)] pub struct AgentService { sandbox: Arc>, init_mode: bool, + #[derivative(Debug = "ignore")] + #[cfg(feature = "sealed-secret")] + cdh_client: Option, } // A container ID must match this regex: @@ -201,6 +209,25 @@ impl AgentService { // cannot predict everything from the caller. add_devices(&req.devices, &mut oci, &self.sandbox).await?; + if cfg!(feature = "sealed-secret") { + let process = oci + .process + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't contain process field"))?; + + for env in process.env.iter_mut() { + let client = self + .cdh_client + .as_ref() + .ok_or(anyhow!("get cdh_client failed"))?; + let unsealed_env = client + .unseal_env(env) + .await + .map_err(|e| anyhow!("unseal env failed: {:?}", e))?; + *env = unsealed_env.to_string(); + } + } + let linux = oci .linux .as_mut() @@ -1697,6 +1724,8 @@ pub async fn start( let agent_service = Box::new(AgentService { sandbox: s.clone(), init_mode, + #[cfg(feature = "sealed-secret")] + cdh_client: Some(CDHClient::new()?), }) as Box; let aservice = agent_ttrpc::create_agent_service(Arc::new(agent_service)); @@ -2226,7 +2255,6 @@ mod tests { async fn test_update_routes() { let logger = slog::Logger::root(slog::Discard, o!()); let sandbox = Sandbox::new(&logger).unwrap(); - let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, @@ -2244,7 +2272,6 @@ mod tests { async fn test_add_arp_neighbors() { let logger = slog::Logger::root(slog::Discard, o!()); let sandbox = Sandbox::new(&logger).unwrap(); - let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, From 9c02722d46c320eabe966f4f40b9a7766d1e7afd Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Mon, 7 Aug 2023 19:37:21 +0800 Subject: [PATCH 06/13] agent: unittest for sealed secret as env in kata Fixes: #7555 Signed-off-by: Linda Yu --- src/agent/src/cdh.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++ src/agent/src/rpc.rs | 10 ++++++ 2 files changed, 93 insertions(+) diff --git a/src/agent/src/cdh.rs b/src/agent/src/cdh.rs index c82acdebcd..c8301ae18f 100644 --- a/src/agent/src/cdh.rs +++ b/src/agent/src/cdh.rs @@ -63,4 +63,87 @@ impl CDHClient { } Ok((*env.to_owned()).to_string()) } +} /* end of impl CDHClient */ + +#[cfg(test)] +#[cfg(feature = "sealed-secret")] +mod tests { + use crate::cdh::CDHClient; + use crate::cdh::CDH_ADDR; + use anyhow::anyhow; + use async_trait::async_trait; + use protocols::{sealed_secret, sealed_secret_ttrpc_async}; + use std::sync::Arc; + use tokio::signal::unix::{signal, SignalKind}; + + struct TestService; + + #[async_trait] + impl sealed_secret_ttrpc_async::SealedSecretService for TestService { + async fn unseal_secret( + &self, + _ctx: &::ttrpc::asynchronous::TtrpcContext, + _req: sealed_secret::UnsealSecretInput, + ) -> ttrpc::error::Result { + let mut output = sealed_secret::UnsealSecretOutput::new(); + output.set_plaintext("unsealed".into()); + Ok(output) + } + } + + fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> { + let path = sock_addr + .strip_prefix("unix://") + .expect("socket address does not have the expected format."); + + if std::path::Path::new(path).exists() { + std::fs::remove_file(path)?; + } + + Ok(()) + } + + fn start_ttrpc_server() { + tokio::spawn(async move { + let ss = Box::new(TestService {}) + as Box; + let ss = Arc::new(ss); + let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss); + + remove_if_sock_exist(CDH_ADDR).unwrap(); + + let mut server = ttrpc::asynchronous::Server::new() + .bind(CDH_ADDR) + .unwrap() + .register_service(ss_service); + + server.start().await.unwrap(); + + let mut interrupt = signal(SignalKind::interrupt()).unwrap(); + tokio::select! { + _ = interrupt.recv() => { + server.shutdown().await.unwrap(); + } + }; + }); + } + + #[tokio::test] + async fn test_unseal_env() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let _guard = rt.enter(); + start_ttrpc_server(); + std::thread::sleep(std::time::Duration::from_secs(2)); + + let cc = Some(CDHClient::new().unwrap()); + let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap(); + let sealed_env = String::from("key=sealed.testdata"); + let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap(); + assert_eq!(unsealed_env, String::from("key=unsealed")); + let normal_env = String::from("key=testdata"); + let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap(); + assert_eq!(unchanged_env, String::from("key=testdata")); + + rt.shutdown_background(); + } } diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 654edf1657..457251fa18 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -2241,6 +2241,8 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + #[cfg(feature = "sealed-secret")] + cdh_client: None, }); let req = protocols::agent::UpdateInterfaceRequest::default(); @@ -2258,6 +2260,8 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + #[cfg(feature = "sealed-secret")] + cdh_client: None, }); let req = protocols::agent::UpdateRoutesRequest::default(); @@ -2275,6 +2279,8 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + #[cfg(feature = "sealed-secret")] + cdh_client: None, }); let req = protocols::agent::AddARPNeighborsRequest::default(); @@ -2409,6 +2415,8 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + #[cfg(feature = "sealed-secret")] + cdh_client: None, }); let result = agent_service @@ -2890,6 +2898,8 @@ OtherField:other let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + #[cfg(feature = "sealed-secret")] + cdh_client: None, }); let ctx = mk_ttrpc_context(); From c60adedf99939abdb997728b9e3521dadd647da5 Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Wed, 9 Aug 2023 12:37:31 +0800 Subject: [PATCH 07/13] agent: add feature for confidential data hub (cdh) Fixes: #7555 Signed-off-by: Linda Yu --- src/agent/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 2feb4c7c0e..876a60727d 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -94,7 +94,9 @@ members = ["rustjail"] lto = true [features] +confidential-data-hub = [] seccomp = ["rustjail/seccomp"] +sealed-secret = ["protocols/sealed-secret", "confidential-data-hub"] standard-oci-runtime = ["rustjail/standard-oci-runtime"] [[bin]] From d7873e525159ffa43735125297c69a0b4fb0c84f Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Wed, 9 Aug 2023 17:51:38 +0800 Subject: [PATCH 08/13] agent: support sealed secret as file in kata Fixes: #7555 Signed-off-by: Linda Yu --- src/agent/src/cdh.rs | 120 ++++++++++++++++++++--- src/agent/src/rpc.rs | 26 +++-- src/runtime/virtcontainers/kata_agent.go | 4 + 3 files changed, 130 insertions(+), 20 deletions(-) diff --git a/src/agent/src/cdh.rs b/src/agent/src/cdh.rs index c8301ae18f..b4a8824fce 100644 --- a/src/agent/src/cdh.rs +++ b/src/agent/src/cdh.rs @@ -8,23 +8,41 @@ // https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub use anyhow::{anyhow, Result}; +use oci::{Mount, Spec}; use protocols::{ sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient, }; +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; const CDH_ADDR: &str = "unix:///run/confidential-containers/cdh.sock"; +const SECRETS_DIR: &str = "/run/secrets/"; +const SEALED_SECRET_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000; + +// Convenience function to obtain the scope logger. +fn sl() -> slog::Logger { + slog_scope::logger() +} #[derive(Clone)] pub struct CDHClient { - sealed_secret_client: SealedSecretServiceClient, + sealed_secret_client: Option, } impl CDHClient { pub fn new() -> Result { - let c = ttrpc::asynchronous::Client::connect(CDH_ADDR)?; - let ssclient = sealed_secret_ttrpc_async::SealedSecretServiceClient::new(c); - Ok(CDHClient { - sealed_secret_client: ssclient, - }) + let c = ttrpc::asynchronous::Client::connect(CDH_ADDR); + match c { + Ok(v) => { + let ssclient = sealed_secret_ttrpc_async::SealedSecretServiceClient::new(v); + Ok(CDHClient { + sealed_secret_client: Some(ssclient), + }) + } + Err(_) => Ok(CDHClient { + sealed_secret_client: None, + }), + } } pub async fn unseal_secret_async( @@ -33,27 +51,25 @@ impl CDHClient { ) -> Result { let secret = sealed .strip_prefix("sealed.") - .ok_or(anyhow!("strip_prefix sealed. failed"))?; + .ok_or(anyhow!("strip_prefix \"sealed.\" failed"))?; let mut input = sealed_secret::UnsealSecretInput::new(); input.set_secret(secret.into()); let unseal = self .sealed_secret_client - .unseal_secret( - ttrpc::context::with_timeout(50 * 1000 * 1000 * 1000), - &input, - ) + .as_ref() + .ok_or(anyhow!("unwrap sealed_secret_client failed"))? + .unseal_secret(ttrpc::context::with_timeout(SEALED_SECRET_TIMEOUT), &input) .await?; Ok(unseal) } pub async fn unseal_env(&self, env: &str) -> Result { - let (key, value) = env.split_once('=').unwrap(); + let (key, value) = env.split_once('=').unwrap_or(("", "")); if value.starts_with("sealed.") { let unsealed_value = self.unseal_secret_async(value).await; match unsealed_value { Ok(v) => { - let plain_env = - format!("{}={}", key, std::str::from_utf8(&v.plaintext).unwrap()); + let plain_env = format!("{}={}", key, std::str::from_utf8(&v.plaintext)?); return Ok(plain_env); } Err(e) => { @@ -63,6 +79,81 @@ impl CDHClient { } Ok((*env.to_owned()).to_string()) } + + pub async fn unseal_file(&self, sealed_source_path: &String) -> Result<()> { + if !Path::new(sealed_source_path).exists() { + info!( + sl(), + "sealed source path {:?} does not exist", sealed_source_path + ); + return Ok(()); + } + + for entry in fs::read_dir(sealed_source_path)? { + let entry = entry?; + + if !entry.file_type()?.is_symlink() + && !fs::metadata(entry.path())?.file_type().is_file() + { + info!( + sl(), + "skipping sealed source entry {:?} because its file type is {:?}", + entry, + entry.file_type()? + ); + continue; + } + + let target_path = fs::canonicalize(&entry.path())?; + info!(sl(), "sealed source entry target path: {:?}", target_path); + if !target_path.is_file() { + info!(sl(), "sealed source is not a file: {:?}", target_path); + continue; + } + + let secret_name = entry.file_name(); + let contents = fs::read_to_string(&target_path)?; + if contents.starts_with("sealed.") { + info!(sl(), "sealed source entry found: {:?}", target_path); + let unsealed_filename = SECRETS_DIR.to_string() + + secret_name + .as_os_str() + .to_str() + .ok_or(anyhow!("create unsealed_filename failed"))?; + let unsealed_value = self.unseal_secret_async(&contents).await?; + fs::write(&unsealed_filename, unsealed_value.plaintext)?; + fs::remove_file(&entry.path())?; + symlink(unsealed_filename, &entry.path())?; + } + } + Ok(()) + } + + pub fn create_sealed_secret_mounts(&self, spec: &mut Spec) -> Result> { + let mut sealed_source_path: Vec = vec![]; + for m in spec.mounts.iter_mut() { + if let Some(unsealed_mount_point) = m.destination.strip_prefix("/sealed") { + info!( + sl(), + "sealed mount destination: {:?} source: {:?}", m.destination, m.source + ); + sealed_source_path.push(m.source.clone()); + m.destination = unsealed_mount_point.to_string(); + } + } + + if sealed_source_path.len() > 0 { + let sealed_mounts = Mount { + destination: SECRETS_DIR.to_string(), + r#type: "bind".to_string(), + source: SECRETS_DIR.to_string(), + options: vec!["bind".to_string()], + }; + spec.mounts.push(sealed_mounts); + } + fs::create_dir_all(SECRETS_DIR)?; + Ok(sealed_source_path) + } } /* end of impl CDHClient */ #[cfg(test)] @@ -145,5 +236,6 @@ mod tests { assert_eq!(unchanged_env, String::from("key=testdata")); rt.shutdown_background(); + std::thread::sleep(std::time::Duration::from_secs(2)); } } diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 457251fa18..2b7afbdee8 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -209,24 +209,27 @@ impl AgentService { // cannot predict everything from the caller. add_devices(&req.devices, &mut oci, &self.sandbox).await?; - if cfg!(feature = "sealed-secret") { + #[cfg(feature = "sealed-secret")] + let mut sealed_source_path = { let process = oci .process .as_mut() .ok_or_else(|| anyhow!("Spec didn't contain process field"))?; + let client = self + .cdh_client + .as_ref() + .ok_or(anyhow!("get cdh_client failed"))?; for env in process.env.iter_mut() { - let client = self - .cdh_client - .as_ref() - .ok_or(anyhow!("get cdh_client failed"))?; let unsealed_env = client .unseal_env(env) .await .map_err(|e| anyhow!("unseal env failed: {:?}", e))?; *env = unsealed_env.to_string(); } - } + + client.create_sealed_secret_mounts(&mut oci)? + }; let linux = oci .linux @@ -311,6 +314,17 @@ impl AgentService { return Err(anyhow!(nix::Error::EINVAL)); }; + #[cfg(feature = "sealed-secret")] + { + let client = self + .cdh_client + .as_ref() + .ok_or(anyhow!("get cdh_client failed"))?; + for source_path in sealed_source_path.iter_mut() { + client.unseal_file(source_path).await?; + } + } + // if starting container failed, we will do some rollback work // to ensure no resources are leaked. if let Err(err) = ctr.start(p).await { diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 65fff6dde4..0276c7ae91 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -911,6 +911,10 @@ func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts map[stri for index, m := range ociMounts { if guestMount, ok := guestMounts[m.Destination]; ok { k.Logger().Debugf("Replacing OCI mount (%s) source %s with %s", m.Destination, m.Source, guestMount.Source) + if strings.Contains(m.Source, "kubernetes.io~secret") { + ociMounts[index].Destination = "/sealed" + m.Destination + k.Logger().Debugf("Replacing OCI mount (%s) with new destination %s", m.Destination, ociMounts[index].Destination) + } ociMounts[index].Source = guestMount.Source } } From f1573b4747425dc2a01240b282dfa518269c06a3 Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Wed, 9 Aug 2023 18:24:34 +0800 Subject: [PATCH 09/13] agent: unittest for sealed secret as file in kata Fixes: #7555 Signed-off-by: Linda Yu --- src/agent/src/cdh.rs | 52 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/agent/src/cdh.rs b/src/agent/src/cdh.rs index b4a8824fce..c088c282fb 100644 --- a/src/agent/src/cdh.rs +++ b/src/agent/src/cdh.rs @@ -142,7 +142,7 @@ impl CDHClient { } } - if sealed_source_path.len() > 0 { + if !sealed_source_path.is_empty() { let sealed_mounts = Mount { destination: SECRETS_DIR.to_string(), r#type: "bind".to_string(), @@ -161,9 +161,14 @@ impl CDHClient { mod tests { use crate::cdh::CDHClient; use crate::cdh::CDH_ADDR; + use crate::cdh::SECRETS_DIR; use anyhow::anyhow; use async_trait::async_trait; use protocols::{sealed_secret, sealed_secret_ttrpc_async}; + use std::fs; + use std::fs::File; + use std::io::{Read, Write}; + use std::path::Path; use std::sync::Arc; use tokio::signal::unix::{signal, SignalKind}; @@ -227,7 +232,10 @@ mod tests { std::thread::sleep(std::time::Duration::from_secs(2)); let cc = Some(CDHClient::new().unwrap()); - let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap(); + let cdh_client = cc + .as_ref() + .ok_or(anyhow!("get confidential-data-hub client failed")) + .unwrap(); let sealed_env = String::from("key=sealed.testdata"); let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap(); assert_eq!(unsealed_env, String::from("key=unsealed")); @@ -238,4 +246,44 @@ mod tests { rt.shutdown_background(); std::thread::sleep(std::time::Duration::from_secs(2)); } + + #[tokio::test] + async fn test_unseal_file() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let _guard = rt.enter(); + start_ttrpc_server(); + std::thread::sleep(std::time::Duration::from_secs(2)); + + let cc = Some(CDHClient::new().unwrap()); + let cdh_client = cc + .as_ref() + .ok_or(anyhow!("get confidential-data-hub client failed")) + .unwrap(); + + fs::create_dir_all(SECRETS_DIR).unwrap(); + + let sealed_filename = "passwd"; + let mut sealed_file = File::create(sealed_filename).unwrap(); + let dir = String::from("."); + sealed_file.write_all(b"sealed.passwd").unwrap(); + cdh_client.unseal_file(&dir).await.unwrap(); + let unsealed_filename = SECRETS_DIR.to_string() + "/passwd"; + let mut unsealed_file = fs::File::open(unsealed_filename.clone()).unwrap(); + let mut contents = String::new(); + unsealed_file.read_to_string(&mut contents).unwrap(); + assert_eq!(contents, String::from("unsealed")); + fs::remove_file(sealed_filename).unwrap(); + fs::remove_file(unsealed_filename).unwrap(); + + let normal_filename = "passwd"; + let mut normal_file = File::create(normal_filename).unwrap(); + normal_file.write_all(b"passwd").unwrap(); + cdh_client.unseal_file(&dir).await.unwrap(); + let filename = SECRETS_DIR.to_string() + "/passwd"; + assert!(!Path::new(&filename).exists()); + fs::remove_file(normal_filename).unwrap(); + + rt.shutdown_background(); + std::thread::sleep(std::time::Duration::from_secs(2)); + } } From 212229df837cb26fef48068620f988f858381256 Mon Sep 17 00:00:00 2001 From: Linda Yu Date: Thu, 10 Aug 2023 15:07:00 +0800 Subject: [PATCH 10/13] runtime: add sealed secret configuration Fixes: #7555 Signed-off-by: Linda Yu --- docs/how-to/how-to-set-sandbox-config-kata.md | 1 + src/runtime/Makefile | 6 +++++- src/runtime/config/configuration-clh-tdx.toml.in | 5 +++++ src/runtime/config/configuration-clh.toml.in | 5 +++++ .../config/configuration-qemu-nvidia-gpu.toml.in | 5 +++++ src/runtime/config/configuration-qemu-se.toml.in | 5 +++++ src/runtime/config/configuration-qemu-sev.toml.in | 5 +++++ src/runtime/config/configuration-qemu-snp.toml.in | 7 ++++++- src/runtime/config/configuration-qemu-tdx.toml.in | 5 +++++ src/runtime/config/configuration-qemu.toml.in | 5 +++++ src/runtime/config/configuration-remote.toml.in | 5 +++++ src/runtime/pkg/katautils/config.go | 2 ++ src/runtime/pkg/oci/utils.go | 11 +++++++++++ src/runtime/pkg/oci/utils_test.go | 2 ++ src/runtime/virtcontainers/kata_agent.go | 14 ++++++++------ .../virtcontainers/pkg/annotations/annotations.go | 3 +++ src/runtime/virtcontainers/sandbox.go | 2 ++ .../local-build/kata-deploy-binaries.sh | 2 +- 18 files changed, 81 insertions(+), 9 deletions(-) diff --git a/docs/how-to/how-to-set-sandbox-config-kata.md b/docs/how-to/how-to-set-sandbox-config-kata.md index 30d4504590..6cad424b50 100644 --- a/docs/how-to/how-to-set-sandbox-config-kata.md +++ b/docs/how-to/how-to-set-sandbox-config-kata.md @@ -28,6 +28,7 @@ There are several kinds of Kata configurations and they are listed below. | `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup | | `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process | | `io.katacontainers.config.runtime.image_request_timeout` | `uint64` | the timeout for pulling an image within the guest in `seconds`, default is `60` | +| `io.katacontainers.config.runtime.sealed_secret_enabled` | `boolean` | enables the sealed secret feature, default is `false` | ## Agent Options | Key | Value Type | Comments | diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 6362c98aa7..77ca2bc124 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -278,6 +278,9 @@ DEFSERVICEOFFLOAD ?= false # Image Request Timeout in seconds DEFIMAGEREQUESTTIMEOUT ?= 60 +# Enable sealed secret or not +DEFSEALEDSECRETENABLED ?= false + # SEV & SEV-ES Guest Pre-Attestation DEFGUESTPREATTESTATION ?= false DEFGUESTPREATTESTATIONPROXY ?= localhost:44444 @@ -709,6 +712,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_TEE USER_VARS += DEFBINDMOUNTS USER_VARS += DEFSERVICEOFFLOAD USER_VARS += DEFIMAGEREQUESTTIMEOUT +USER_VARS += DEFSEALEDSECRETENABLED USER_VARS += DEFVFIOMODE USER_VARS += BUILDFLAGS USER_VARS += DEFSERVICEOFFLOAD @@ -1057,4 +1061,4 @@ ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS))) endif @printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR)) @printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR)) - @printf "\n" \ No newline at end of file + @printf "\n" diff --git a/src/runtime/config/configuration-clh-tdx.toml.in b/src/runtime/config/configuration-clh-tdx.toml.in index c705478345..3f59d99446 100644 --- a/src/runtime/config/configuration-clh-tdx.toml.in +++ b/src/runtime/config/configuration-clh-tdx.toml.in @@ -415,6 +415,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 826fc29d72..d572d9aed8 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -434,6 +434,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in index 116576ac36..8f17c54f3c 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in @@ -669,6 +669,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-qemu-se.toml.in b/src/runtime/config/configuration-qemu-se.toml.in index b59fc7653d..9474361d68 100644 --- a/src/runtime/config/configuration-qemu-se.toml.in +++ b/src/runtime/config/configuration-qemu-se.toml.in @@ -645,6 +645,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in index 557dfd67e3..fa1d2e90d6 100644 --- a/src/runtime/config/configuration-qemu-sev.toml.in +++ b/src/runtime/config/configuration-qemu-sev.toml.in @@ -649,6 +649,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index 7fecf527b7..4521204e9f 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -674,6 +674,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] @@ -705,4 +710,4 @@ image_request_timeout = @DEFIMAGEREQUESTTIMEOUT@ # # Keys can be remotely provisioned. The Kata agent fetches them from e.g. # a HTTPS URL: -#provision=https://my-key-broker.foo/tenant/ \ No newline at end of file +#provision=https://my-key-broker.foo/tenant/ diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index 182ff4e6c4..1368e153eb 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -662,6 +662,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index e0f92726e6..d10bd9cca5 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -709,6 +709,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/config/configuration-remote.toml.in b/src/runtime/config/configuration-remote.toml.in index 5a42096cc9..50492e3da8 100644 --- a/src/runtime/config/configuration-remote.toml.in +++ b/src/runtime/config/configuration-remote.toml.in @@ -289,6 +289,11 @@ experimental=@DEFAULTEXPFEATURES@ # (default: false) # enable_pprof = true +# Sealed secret feature configuration. +# If true, sealed secret feature will be enabled. +# (default: false) +sealed_secret_enabled = @DEFSEALEDSECRETENABLED@ + # WARNING: All the options in the following section have not been implemented yet. # This section was added as a placeholder. DO NOT USE IT! [image] diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index b8f955cbba..b24ddb27da 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -191,6 +191,7 @@ type runtime struct { StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` EnablePprof bool `toml:"enable_pprof"` DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` + SealedSecretEnabled bool `toml:"sealed_secret_enabled"` } type agent struct { @@ -1453,6 +1454,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat config.JaegerEndpoint = tomlConf.Runtime.JaegerEndpoint config.JaegerUser = tomlConf.Runtime.JaegerUser config.JaegerPassword = tomlConf.Runtime.JaegerPassword + config.SealedSecretEnabled = tomlConf.Runtime.SealedSecretEnabled config.ServiceOffload = tomlConf.Image.ServiceOffload config.ImageRequestTimeout = tomlConf.Image.ImageRequestTimeout for _, f := range tomlConf.Runtime.Experimental { diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index 2355ed2167..78a5a4cf17 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -161,6 +161,9 @@ type RuntimeConfig struct { // Image request timeout which, if provided, indicates the image request timeout // in the guest needed for the workload(s) ImageRequestTimeout uint64 + + // Sealed secret enabled configuration + SealedSecretEnabled bool } // AddKernelParam allows the addition of new kernel parameters to an existing @@ -918,6 +921,12 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r }); err != nil { return err } + + if err := newAnnotationConfiguration(ocispec, vcAnnotations.SealedSecretEnabled).setBool(func(SealedSecretEnabled bool) { + sbConfig.SealedSecretEnabled = SealedSecretEnabled + }); err != nil { + return err + } return nil } @@ -1033,6 +1042,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st ServiceOffload: runtime.ServiceOffload, ImageRequestTimeout: runtime.ImageRequestTimeout, + + SealedSecretEnabled: runtime.SealedSecretEnabled, } if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil { diff --git a/src/runtime/pkg/oci/utils_test.go b/src/runtime/pkg/oci/utils_test.go index f045eede80..c787f8b654 100644 --- a/src/runtime/pkg/oci/utils_test.go +++ b/src/runtime/pkg/oci/utils_test.go @@ -811,6 +811,7 @@ func TestAddRuntimeAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.DisableNewNetNs] = "true" ocispec.Annotations[vcAnnotations.InterNetworkModel] = "macvtap" ocispec.Annotations[vcAnnotations.ImageRequestTimeout] = "100" + ocispec.Annotations[vcAnnotations.SealedSecretEnabled] = "true" addAnnotations(ocispec, &config, runtimeConfig) assert.Equal(config.DisableGuestSeccomp, true) @@ -818,6 +819,7 @@ func TestAddRuntimeAnnotations(t *testing.T) { assert.Equal(config.NetworkConfig.DisableNewNetwork, true) assert.Equal(config.NetworkConfig.InterworkingModel, vc.NetXConnectMacVtapModel) assert.Equal(config.ImageRequestTimeout, uint64(100)) + assert.Equal(config.SealedSecretEnabled, true) } diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 0276c7ae91..469d8cfc67 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -905,16 +905,18 @@ func (k *kataAgent) stopSandbox(ctx context.Context, sandbox *Sandbox) error { return nil } -func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts map[string]Mount) error { +func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts map[string]Mount, sealedSecretEnabled bool) error { ociMounts := spec.Mounts for index, m := range ociMounts { if guestMount, ok := guestMounts[m.Destination]; ok { k.Logger().Debugf("Replacing OCI mount (%s) source %s with %s", m.Destination, m.Source, guestMount.Source) - if strings.Contains(m.Source, "kubernetes.io~secret") { - ociMounts[index].Destination = "/sealed" + m.Destination - k.Logger().Debugf("Replacing OCI mount (%s) with new destination %s", m.Destination, ociMounts[index].Destination) - } + if sealedSecretEnabled { + if strings.Contains(m.Source, "kubernetes.io~secret") { + ociMounts[index].Destination = "/sealed" + m.Destination + k.Logger().Debugf("Replacing OCI mount (%s) with new destination %s", m.Destination, ociMounts[index].Destination) + } + } ociMounts[index].Source = guestMount.Source } } @@ -1303,7 +1305,7 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co // We replace all OCI mount sources that match our container mount // with the right source path (The guest one). - if err = k.replaceOCIMountSource(ociSpec, sharedDirMounts); err != nil { + if err = k.replaceOCIMountSource(ociSpec, sharedDirMounts, sandbox.config.SealedSecretEnabled); err != nil { return nil, err } diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 8827d1aa8c..59a4a33713 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -291,6 +291,9 @@ const ( // ImageRequestTimeout is a sandbox annotaion that sets the image pull timeout in the guest. ImageRequestTimeout = kataAnnotRuntimePrefix + "image_request_timeout" + + // SealedSecretEnabled is a sandbox annotaion that enables the sealed secret feature. + SealedSecretEnabled = kataAnnotRuntimePrefix + "sealed_secret_enabled" ) // Agent related annotations diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index d055750b82..ce34a30515 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -164,6 +164,8 @@ type SandboxConfig struct { // Image request timeout which, if provided, indicates the image request timeout // in the guest needed for the workload(s) ImageRequestTimeout uint64 + // enable sealed secret feature + SealedSecretEnabled bool // SharePidNs sets all containers to share the same sandbox level pid namespace. SharePidNs bool // SystemdCgroup enables systemd cgroup support diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index afdb149861..4e9b6e219e 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -270,7 +270,7 @@ install_cc_shimv2() { export RUST_VERSION export REMOVE_VMM_CONFIGS="acrn fc" - extra_opts="DEFSERVICEOFFLOAD=true" + extra_opts="DEFSERVICEOFFLOAD=true DEFSEALEDSECRETENABLED=true" if [ "${MEASURED_ROOTFS}" == "yes" ]; then if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt") From 47c28923d7fba53b742650bd9cbfb828dddcce88 Mon Sep 17 00:00:00 2001 From: Biao lu Date: Wed, 30 Aug 2023 15:04:50 +0800 Subject: [PATCH 11/13] agent: Add config for api-server-rest Add configuration for 'rest api server'. Optional configurations are 'agent.rest_api=attestation' will enable attestation api 'agent.rest_api=resource' will enable resource api 'agent.rest_api=all' will enable all (attestation and resource) api Fixes: #7555 Signed-off-by: Biao lu --- src/agent/src/config.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index e9d1d75fdd..c454f642e6 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -27,6 +27,7 @@ const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size"; const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy"; const CONFIG_FILE: &str = "agent.config_file"; const AA_KBC_PARAMS: &str = "agent.aa_kbc_params"; +const REST_API_OPTION: &str = "agent.rest_api"; const HTTPS_PROXY: &str = "agent.https_proxy"; const NO_PROXY: &str = "agent.no_proxy"; const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity"; @@ -88,6 +89,7 @@ pub struct AgentConfig { pub supports_seccomp: bool, pub container_policy_path: String, pub aa_kbc_params: String, + pub rest_api: String, pub https_proxy: String, pub no_proxy: String, pub data_integrity: bool, @@ -112,6 +114,7 @@ pub struct AgentConfigBuilder { pub endpoints: Option, pub container_policy_path: Option, pub aa_kbc_params: Option, + pub rest_api: Option, pub https_proxy: Option, pub no_proxy: Option, pub data_integrity: Option, @@ -182,6 +185,7 @@ impl Default for AgentConfig { supports_seccomp: rpc::have_seccomp(), container_policy_path: String::from(""), aa_kbc_params: String::from(""), + rest_api: String::from(""), https_proxy: String::from(""), no_proxy: String::from(""), data_integrity: false, @@ -219,6 +223,7 @@ impl FromStr for AgentConfig { config_override!(agent_config_builder, agent_config, tracing); config_override!(agent_config_builder, agent_config, container_policy_path); config_override!(agent_config_builder, agent_config, aa_kbc_params); + config_override!(agent_config_builder, agent_config, rest_api); config_override!(agent_config_builder, agent_config, https_proxy); config_override!(agent_config_builder, agent_config, no_proxy); config_override!(agent_config_builder, agent_config, data_integrity); @@ -343,6 +348,7 @@ impl AgentConfig { ); parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value); + parse_cmdline_param!(param, REST_API_OPTION, config.rest_api, get_string_value); parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value); parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value); parse_cmdline_param!( @@ -588,6 +594,7 @@ mod tests { tracing: bool, container_policy_path: &'a str, aa_kbc_params: &'a str, + rest_api: &'a str, https_proxy: &'a str, no_proxy: &'a str, data_integrity: bool, @@ -612,6 +619,7 @@ mod tests { tracing: false, container_policy_path: "", aa_kbc_params: "", + rest_api: "", https_proxy: "", no_proxy: "", data_integrity: false, @@ -998,6 +1006,21 @@ mod tests { aa_kbc_params: "eaa_kbc::127.0.0.1:50000", ..Default::default() }, + TestData { + contents: "agent.rest_api=attestation", + rest_api: "attestation", + ..Default::default() + }, + TestData { + contents: "agent.rest_api=resource", + rest_api: "resource", + ..Default::default() + }, + TestData { + contents: "agent.rest_api=all", + rest_api: "all", + ..Default::default() + }, TestData { contents: "agent.https_proxy=http://proxy.url.com:81/", https_proxy: "http://proxy.url.com:81/", @@ -1161,6 +1184,7 @@ mod tests { msg ); assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg); + assert_eq!(d.rest_api, config.rest_api, "{}", msg); assert_eq!(d.https_proxy, config.https_proxy, "{}", msg); assert_eq!(d.no_proxy, config.no_proxy, "{}", msg); assert_eq!(d.data_integrity, config.data_integrity, "{}", msg); From e865359f4ed62a143b6a6d4c0f778d63407053a3 Mon Sep 17 00:00:00 2001 From: Biao Lu Date: Wed, 30 Aug 2023 15:32:12 +0800 Subject: [PATCH 12/13] osbuilder: add api-server-rest in rootfs Integrate api-server-rest into rootfs image. Fixes: #7555 Signed-off-by: Biao Lu --- tools/osbuilder/rootfs-builder/rootfs.sh | 5 +++++ versions.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 127f37dafa..ee7569d7ce 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -705,6 +705,11 @@ EOF make RESOURCE_PROVIDER=${CDH_RESOURCE_PROVIDER} make install DESTDIR="${ROOTFS_DIR}/usr/local/bin/" popd + + pushd guest-components/api-server-rest + cargo build --release --target-dir ./target + install -D -m0755 ./target/release/api-server-rest ${ROOTFS_DIR}/usr/local/bin/ + popd fi if [ "${KATA_BUILD_CC}" == "yes" ]; then diff --git a/versions.yaml b/versions.yaml index 65bb063141..e32ea79c8e 100644 --- a/versions.yaml +++ b/versions.yaml @@ -200,7 +200,7 @@ externals: attestation-agent: description: "Provide attested key unwrapping for image decryption" url: "https://github.com/confidential-containers/guest-components/" - version: "53ddd632424432077e95d3901deb64727be0b4c1" + version: "29086ca22583f0dcfa6548866e9bf2a5e07881e9" cni-plugins: description: "CNI network plugins" From 13943fb81bdeec7062831a7eac59ec970cc1872d Mon Sep 17 00:00:00 2001 From: Biao Lu Date: Wed, 30 Aug 2023 16:48:18 +0800 Subject: [PATCH 13/13] agent: launch api-server-rest If 'rest_api' is configured, start api-server-rest after attestation-agent and confidential-data-hub. Fixes: #7555 Signed-off-by: Biao Lu --- src/agent/src/main.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 0761e230f3..30fc32507e 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -105,6 +105,7 @@ cfg_if! { if #[cfg(feature = "confidential-data-hub")] { const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub"; const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock"; + const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest"; } } @@ -370,7 +371,7 @@ async fn start_sandbox( sandbox.lock().await.sender = Some(tx); if !config.aa_kbc_params.is_empty() { - init_attestation_agent(logger)?; + init_attestation_agent(logger, config)?; } // vsock:///dev/vsock, port @@ -385,7 +386,7 @@ async fn start_sandbox( // If we fail to start the AA, ocicrypt won't be able to unwrap keys // and container decryption will fail. -fn init_attestation_agent(logger: &Logger) -> Result<()> { +fn init_attestation_agent(logger: &Logger, _config: &AgentConfig) -> Result<()> { let config_path = OCICRYPT_CONFIG_PATH; // The image will need to be encrypted using a keyprovider @@ -429,6 +430,16 @@ fn init_attestation_agent(logger: &Logger) -> Result<()> { DEFAULT_LAUNCH_PROCESS_TIMEOUT, ) { error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e); + } else if !_config.rest_api.is_empty() { + if let Err(e) = launch_process( + logger, + API_SERVER_PATH, + &vec!["--features", &_config.rest_api], + "", + 0, + ) { + error!(logger, "launch_process {} failed: {:?}", API_SERVER_PATH, e); + } } }