github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-21 09:25:19 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7570 from LindaYu17/CCv0

Browse Source 
CC | add sealed secret support in Kata
This commit is contained in:
Fabiano Fidêncio 2023-09-07 09:46:33 +02:00 committed by GitHub
commit d70ed93173
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 680 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/how-to/how-to-set-sandbox-config-kata.md
View File

@ -28,6 +28,7 @@ There are several kinds of Kata configurations and they are listed below.
| `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
| `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
| `io.katacontainers.config.runtime.image_request_timeout` | `uint64` | the timeout for pulling an image within the guest in `seconds`, default is `60` |
| `io.katacontainers.config.runtime.sealed_secret_enabled` | `boolean` | enables the sealed secret feature, default is `false` |
## Agent Options
| Key | Value Type | Comments |

22
src/agent/Cargo.lock generated
View File

@ -718,6 +718,26 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913"
[[package]]
name = "const_format"
version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48"
dependencies = [
"const_format_proc_macros",
]
[[package]]
name = "const_format_proc_macros"
version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -2103,6 +2123,8 @@ dependencies = [
"cfg-if 1.0.0",
"cgroups-rs",
"clap",
"const_format",
"derivative",
"futures",
"image-rs",
"ipnetwork",

4
src/agent/Cargo.toml
View File

@ -23,7 +23,9 @@ regex = "1.5.6"
serial_test = "0.5.1"
kata-sys-util = { path = "../libs/kata-sys-util" }
kata-types = { path = "../libs/kata-types" }
const_format = "0.2.30"
url = "2.2.2"
derivative = "2.2.0"
# Async helpers
async-trait = "0.1.42"
@ -92,7 +94,9 @@ members = ["rustjail"]
lto = true
[features]
confidential-data-hub = []
seccomp = ["rustjail/seccomp"]
sealed-secret = ["protocols/sealed-secret", "confidential-data-hub"]
standard-oci-runtime = ["rustjail/standard-oci-runtime"]
[[bin]]

7
src/agent/Makefile
View File

@ -33,6 +33,13 @@ ifeq ($(SECCOMP),yes)
override EXTRA_RUSTFEATURES += seccomp
endif
SEALED_SECRET ?= no
# Enable sealed-secret feature of rust build
ifeq ($(SEALED_SECRET),yes)
override EXTRA_RUSTFEATURES += sealed-secret
endif
include ../../utils.mk
ifeq ($(ARCH), ppc64le)

289
src/agent/src/cdh.rs Normal file
View File

@ -0,0 +1,289 @@
// Copyright (c) 2023 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
// Confidential Data Hub client wrapper.
// Confidential Data Hub is a service running inside guest to provide resource related APIs.
// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
use anyhow::{anyhow, Result};
use oci::{Mount, Spec};
use protocols::{
sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient,
};
use std::fs;
use std::os::unix::fs::symlink;
use std::path::Path;
const CDH_ADDR: &str = "unix:///run/confidential-containers/cdh.sock";
const SECRETS_DIR: &str = "/run/secrets/";
const SEALED_SECRET_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000;
// Convenience function to obtain the scope logger.
fn sl() -> slog::Logger {
slog_scope::logger()
}
#[derive(Clone)]
pub struct CDHClient {
sealed_secret_client: Option<SealedSecretServiceClient>,
}
impl CDHClient {
pub fn new() -> Result<Self> {
let c = ttrpc::asynchronous::Client::connect(CDH_ADDR);
match c {
Ok(v) => {
let ssclient = sealed_secret_ttrpc_async::SealedSecretServiceClient::new(v);
Ok(CDHClient {
sealed_secret_client: Some(ssclient),
})
}
Err(_) => Ok(CDHClient {
sealed_secret_client: None,
}),
}
}
pub async fn unseal_secret_async(
&self,
sealed: &str,
) -> Result<sealed_secret::UnsealSecretOutput> {
let secret = sealed
.strip_prefix("sealed.")
.ok_or(anyhow!("strip_prefix \"sealed.\" failed"))?;
let mut input = sealed_secret::UnsealSecretInput::new();
input.set_secret(secret.into());
let unseal = self
.sealed_secret_client
.as_ref()
.ok_or(anyhow!("unwrap sealed_secret_client failed"))?
.unseal_secret(ttrpc::context::with_timeout(SEALED_SECRET_TIMEOUT), &input)
.await?;
Ok(unseal)
}
pub async fn unseal_env(&self, env: &str) -> Result<String> {
let (key, value) = env.split_once('=').unwrap_or(("", ""));
if value.starts_with("sealed.") {
let unsealed_value = self.unseal_secret_async(value).await;
match unsealed_value {
Ok(v) => {
let plain_env = format!("{}={}", key, std::str::from_utf8(&v.plaintext)?);
return Ok(plain_env);
}
Err(e) => {
return Err(e);
}
};
}
Ok((*env.to_owned()).to_string())
}
pub async fn unseal_file(&self, sealed_source_path: &String) -> Result<()> {
if !Path::new(sealed_source_path).exists() {
info!(
sl(),
"sealed source path {:?} does not exist", sealed_source_path
);
return Ok(());
}
for entry in fs::read_dir(sealed_source_path)? {
let entry = entry?;
if !entry.file_type()?.is_symlink()
&& !fs::metadata(entry.path())?.file_type().is_file()
{
info!(
sl(),
"skipping sealed source entry {:?} because its file type is {:?}",
entry,
entry.file_type()?
);
continue;
}
let target_path = fs::canonicalize(&entry.path())?;
info!(sl(), "sealed source entry target path: {:?}", target_path);
if !target_path.is_file() {
info!(sl(), "sealed source is not a file: {:?}", target_path);
continue;
}
let secret_name = entry.file_name();
let contents = fs::read_to_string(&target_path)?;
if contents.starts_with("sealed.") {
info!(sl(), "sealed source entry found: {:?}", target_path);
let unsealed_filename = SECRETS_DIR.to_string()
+ secret_name
.as_os_str()
.to_str()
.ok_or(anyhow!("create unsealed_filename failed"))?;
let unsealed_value = self.unseal_secret_async(&contents).await?;
fs::write(&unsealed_filename, unsealed_value.plaintext)?;
fs::remove_file(&entry.path())?;
symlink(unsealed_filename, &entry.path())?;
}
}
Ok(())
}
pub fn create_sealed_secret_mounts(&self, spec: &mut Spec) -> Result<Vec<String>> {
let mut sealed_source_path: Vec<String> = vec![];
for m in spec.mounts.iter_mut() {
if let Some(unsealed_mount_point) = m.destination.strip_prefix("/sealed") {
info!(
sl(),
"sealed mount destination: {:?} source: {:?}", m.destination, m.source
);
sealed_source_path.push(m.source.clone());
m.destination = unsealed_mount_point.to_string();
}
}
if !sealed_source_path.is_empty() {
let sealed_mounts = Mount {
destination: SECRETS_DIR.to_string(),
r#type: "bind".to_string(),
source: SECRETS_DIR.to_string(),
options: vec!["bind".to_string()],
};
spec.mounts.push(sealed_mounts);
}
fs::create_dir_all(SECRETS_DIR)?;
Ok(sealed_source_path)
}
} /* end of impl CDHClient */
#[cfg(test)]
#[cfg(feature = "sealed-secret")]
mod tests {
use crate::cdh::CDHClient;
use crate::cdh::CDH_ADDR;
use crate::cdh::SECRETS_DIR;
use anyhow::anyhow;
use async_trait::async_trait;
use protocols::{sealed_secret, sealed_secret_ttrpc_async};
use std::fs;
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use std::sync::Arc;
use tokio::signal::unix::{signal, SignalKind};
struct TestService;
#[async_trait]
impl sealed_secret_ttrpc_async::SealedSecretService for TestService {
async fn unseal_secret(
&self,
_ctx: &::ttrpc::asynchronous::TtrpcContext,
_req: sealed_secret::UnsealSecretInput,
) -> ttrpc::error::Result<sealed_secret::UnsealSecretOutput> {
let mut output = sealed_secret::UnsealSecretOutput::new();
output.set_plaintext("unsealed".into());
Ok(output)
}
}
fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> {
let path = sock_addr
.strip_prefix("unix://")
.expect("socket address does not have the expected format.");
if std::path::Path::new(path).exists() {
std::fs::remove_file(path)?;
}
Ok(())
}
fn start_ttrpc_server() {
tokio::spawn(async move {
let ss = Box::new(TestService {})
as Box<dyn sealed_secret_ttrpc_async::SealedSecretService + Send + Sync>;
let ss = Arc::new(ss);
let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss);
remove_if_sock_exist(CDH_ADDR).unwrap();
let mut server = ttrpc::asynchronous::Server::new()
.bind(CDH_ADDR)
.unwrap()
.register_service(ss_service);
server.start().await.unwrap();
let mut interrupt = signal(SignalKind::interrupt()).unwrap();
tokio::select! {
_ = interrupt.recv() => {
server.shutdown().await.unwrap();
}
};
});
}
#[tokio::test]
async fn test_unseal_env() {
let rt = tokio::runtime::Runtime::new().unwrap();
let _guard = rt.enter();
start_ttrpc_server();
std::thread::sleep(std::time::Duration::from_secs(2));
let cc = Some(CDHClient::new().unwrap());
let cdh_client = cc
.as_ref()
.ok_or(anyhow!("get confidential-data-hub client failed"))
.unwrap();
let sealed_env = String::from("key=sealed.testdata");
let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
assert_eq!(unsealed_env, String::from("key=unsealed"));
let normal_env = String::from("key=testdata");
let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
assert_eq!(unchanged_env, String::from("key=testdata"));
rt.shutdown_background();
std::thread::sleep(std::time::Duration::from_secs(2));
}
#[tokio::test]
async fn test_unseal_file() {
let rt = tokio::runtime::Runtime::new().unwrap();
let _guard = rt.enter();
start_ttrpc_server();
std::thread::sleep(std::time::Duration::from_secs(2));
let cc = Some(CDHClient::new().unwrap());
let cdh_client = cc
.as_ref()
.ok_or(anyhow!("get confidential-data-hub client failed"))
.unwrap();
fs::create_dir_all(SECRETS_DIR).unwrap();
let sealed_filename = "passwd";
let mut sealed_file = File::create(sealed_filename).unwrap();
let dir = String::from(".");
sealed_file.write_all(b"sealed.passwd").unwrap();
cdh_client.unseal_file(&dir).await.unwrap();
let unsealed_filename = SECRETS_DIR.to_string() + "/passwd";
let mut unsealed_file = fs::File::open(unsealed_filename.clone()).unwrap();
let mut contents = String::new();
unsealed_file.read_to_string(&mut contents).unwrap();
assert_eq!(contents, String::from("unsealed"));
fs::remove_file(sealed_filename).unwrap();
fs::remove_file(unsealed_filename).unwrap();
let normal_filename = "passwd";
let mut normal_file = File::create(normal_filename).unwrap();
normal_file.write_all(b"passwd").unwrap();
cdh_client.unseal_file(&dir).await.unwrap();
let filename = SECRETS_DIR.to_string() + "/passwd";
assert!(!Path::new(&filename).exists());
fs::remove_file(normal_filename).unwrap();
rt.shutdown_background();
std::thread::sleep(std::time::Duration::from_secs(2));
}
}

24
src/agent/src/config.rs
View File

@ -27,6 +27,7 @@ const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
const CONFIG_FILE: &str = "agent.config_file";
const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
const REST_API_OPTION: &str = "agent.rest_api";
const HTTPS_PROXY: &str = "agent.https_proxy";
const NO_PROXY: &str = "agent.no_proxy";
const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
@ -88,6 +89,7 @@ pub struct AgentConfig {
pub supports_seccomp: bool,
pub container_policy_path: String,
pub aa_kbc_params: String,
pub rest_api: String,
pub https_proxy: String,
pub no_proxy: String,
pub data_integrity: bool,
@ -112,6 +114,7 @@ pub struct AgentConfigBuilder {
pub endpoints: Option<EndpointsConfig>,
pub container_policy_path: Option<String>,
pub aa_kbc_params: Option<String>,
pub rest_api: Option<String>,
pub https_proxy: Option<String>,
pub no_proxy: Option<String>,
pub data_integrity: Option<bool>,
@ -182,6 +185,7 @@ impl Default for AgentConfig {
supports_seccomp: rpc::have_seccomp(),
container_policy_path: String::from(""),
aa_kbc_params: String::from(""),
rest_api: String::from(""),
https_proxy: String::from(""),
no_proxy: String::from(""),
data_integrity: false,
@ -219,6 +223,7 @@ impl FromStr for AgentConfig {
config_override!(agent_config_builder, agent_config, tracing);
config_override!(agent_config_builder, agent_config, container_policy_path);
config_override!(agent_config_builder, agent_config, aa_kbc_params);
config_override!(agent_config_builder, agent_config, rest_api);
config_override!(agent_config_builder, agent_config, https_proxy);
config_override!(agent_config_builder, agent_config, no_proxy);
config_override!(agent_config_builder, agent_config, data_integrity);
@ -343,6 +348,7 @@ impl AgentConfig {
);
parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
parse_cmdline_param!(param, REST_API_OPTION, config.rest_api, get_string_value);
parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
parse_cmdline_param!(
@ -588,6 +594,7 @@ mod tests {
tracing: bool,
container_policy_path: &'a str,
aa_kbc_params: &'a str,
rest_api: &'a str,
https_proxy: &'a str,
no_proxy: &'a str,
data_integrity: bool,
@ -612,6 +619,7 @@ mod tests {
tracing: false,
container_policy_path: "",
aa_kbc_params: "",
rest_api: "",
https_proxy: "",
no_proxy: "",
data_integrity: false,
@ -998,6 +1006,21 @@ mod tests {
aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
..Default::default()
},
TestData {
contents: "agent.rest_api=attestation",
rest_api: "attestation",
..Default::default()
},
TestData {
contents: "agent.rest_api=resource",
rest_api: "resource",
..Default::default()
},
TestData {
contents: "agent.rest_api=all",
rest_api: "all",
..Default::default()
},
TestData {
contents: "agent.https_proxy=http://proxy.url.com:81/",
https_proxy: "http://proxy.url.com:81/",
@ -1161,6 +1184,7 @@ mod tests {
msg
);
assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
assert_eq!(d.rest_api, config.rest_api, "{}", msg);
assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);

54
src/agent/src/image_rpc.rs
View File

@ -9,8 +9,7 @@ use std::collections::HashMap;
use std::env;
use std::fs;
use std::path::Path;
use std::process::Command;
use std::sync::atomic::{AtomicBool, AtomicU16, Ordering};
use std::sync::atomic::{AtomicU16, Ordering};
use std::sync::Arc;
use anyhow::{anyhow, Context, Result};
@ -26,14 +25,6 @@ use crate::AGENT_CONFIG;
// A marker to merge container spec for images pulled inside guest.
const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
const AA_PATH: &str = "/usr/local/bin/attestation-agent";
const AA_KEYPROVIDER_URI: &str =
"unix:///run/confidential-containers/attestation-agent/keyprovider.sock";
const AA_GETRESOURCE_URI: &str =
"unix:///run/confidential-containers/attestation-agent/getresource.sock";
const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
// kata rootfs is readonly, use tmpfs before CC storage is implemented.
const KATA_CC_IMAGE_WORK_DIR: &str = "/run/image/";
const KATA_CC_PAUSE_BUNDLE: &str = "/pause_bundle";
@ -51,7 +42,6 @@ fn sl() -> slog::Logger {
#[derive(Clone)]
pub struct ImageService {
attestation_agent_started: Arc<AtomicBool>,
image_client: Arc<Mutex<ImageClient>>,
images: Arc<Mutex<HashMap<String, String>>>,
container_count: Arc<AtomicU16>,
@ -75,7 +65,6 @@ impl ImageService {
}
Self {
attestation_agent_started: Arc::new(AtomicBool::new(false)),
image_client: Arc::new(Mutex::new(image_client)),
images: Arc::new(Mutex::new(HashMap::new())),
container_count: Arc::new(AtomicU16::new(0)),
@ -117,36 +106,6 @@ impl ImageService {
Ok(())
}
// If we fail to start the AA, ocicrypt won't be able to unwrap keys
// and container decryption will fail.
fn init_attestation_agent() -> Result<()> {
let config_path = OCICRYPT_CONFIG_PATH;
// The image will need to be encrypted using a keyprovider
// that has the same name (at least according to the config).
let ocicrypt_config = serde_json::json!({
"key-providers": {
"attestation-agent":{
"ttrpc":AA_KEYPROVIDER_URI
}
}
});
fs::write(config_path, ocicrypt_config.to_string().as_bytes())?;
env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path);
// The Attestation Agent will run for the duration of the guest.
Command::new(AA_PATH)
.arg("--keyprovider_sock")
.arg(AA_KEYPROVIDER_URI)
.arg("--getresource_sock")
.arg(AA_GETRESOURCE_URI)
.spawn()?;
Ok(())
}
/// Determines the container id (cid) to use for a given request.
///
/// If the request specifies a non-empty id, use it; otherwise derive it from the image path.
@ -188,17 +147,6 @@ impl ImageService {
}
let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
if !aa_kbc_params.is_empty() {
match self.attestation_agent_started.compare_exchange_weak(
false,
true,
Ordering::SeqCst,
Ordering::SeqCst,
) {
Ok(_) => Self::init_attestation_agent()?,
Err(_) => info!(sl(), "Attestation Agent already running"),
}
}
// If the attestation-agent is being used, then enable the authenticated credentials support
info!(
sl(),

133
src/agent/src/main.rs
View File

@ -22,6 +22,7 @@ extern crate slog;
use anyhow::{anyhow, Context, Result};
use cfg_if::cfg_if;
use clap::{AppSettings, Parser};
use const_format::concatcp;
use nix::fcntl::OFlag;
use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
use nix::unistd::{self, dup, Pid};
@ -32,9 +33,12 @@ use std::os::unix::fs as unixfs;
use std::os::unix::io::AsRawFd;
use std::path::Path;
use std::process::exit;
use std::process::Command;
use std::sync::Arc;
use tracing::{instrument, span};
#[cfg(feature = "confidential-data-hub")]
mod cdh;
mod config;
mod console;
mod device;
@ -84,6 +88,27 @@ cfg_if! {
const NAME: &str = "kata-agent";
const OCICRYPT_CONFIG_PATH: &str = "/tmp/ocicrypt_config.json";
const AA_PATH: &str = "/usr/local/bin/attestation-agent";
const AA_UNIX_SOCKET_DIR: &str = "/run/confidential-containers/attestation-agent/";
const UNIX_SOCKET_PREFIX: &str = "unix://";
const AA_KEYPROVIDER_URI: &str =
concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "keyprovider.sock");
const AA_GETRESOURCE_URI: &str =
concatcp!(UNIX_SOCKET_PREFIX, AA_UNIX_SOCKET_DIR, "getresource.sock");
const AA_ATTESTATION_SOCKET: &str = concatcp!(AA_UNIX_SOCKET_DIR, "attestation-agent.sock");
const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SOCKET);
const DEFAULT_LAUNCH_PROCESS_TIMEOUT: i32 = 6;
cfg_if! {
if #[cfg(feature = "confidential-data-hub")] {
const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub";
const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock";
const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
}
}
lazy_static! {
static ref AGENT_CONFIG: AgentConfig =
// Note: We can't do AgentOpts.parse() here to send through the processed arguments to AgentConfig
@ -345,6 +370,10 @@ async fn start_sandbox(
let (tx, rx) = tokio::sync::oneshot::channel();
sandbox.lock().await.sender = Some(tx);
if !config.aa_kbc_params.is_empty() {
init_attestation_agent(logger, config)?;
}
// vsock:///dev/vsock, port
let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
server.start().await?;
@ -355,6 +384,110 @@ async fn start_sandbox(
Ok(())
}
// If we fail to start the AA, ocicrypt won't be able to unwrap keys
// and container decryption will fail.
fn init_attestation_agent(logger: &Logger, _config: &AgentConfig) -> Result<()> {
let config_path = OCICRYPT_CONFIG_PATH;
// The image will need to be encrypted using a keyprovider
// that has the same name (at least according to the config).
let ocicrypt_config = serde_json::json!({
"key-providers": {
"attestation-agent":{
"ttrpc":AA_KEYPROVIDER_URI
}
}
});
fs::write(config_path, ocicrypt_config.to_string().as_bytes())?;
env::set_var("OCICRYPT_KEYPROVIDER_CONFIG", config_path);
// The Attestation Agent will run for the duration of the guest.
launch_process(
logger,
AA_PATH,
&vec![
"--keyprovider_sock",
AA_KEYPROVIDER_URI,
"--getresource_sock",
AA_GETRESOURCE_URI,
"--attestation_sock",
AA_ATTESTATION_URI,
],
AA_ATTESTATION_SOCKET,
DEFAULT_LAUNCH_PROCESS_TIMEOUT,
)
.map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?;
#[cfg(feature = "confidential-data-hub")]
{
if let Err(e) = launch_process(
logger,
CDH_PATH,
&vec![],
CDH_SOCKET,
DEFAULT_LAUNCH_PROCESS_TIMEOUT,
) {
error!(logger, "launch_process {} failed: {:?}", CDH_PATH, e);
} else if !_config.rest_api.is_empty() {
if let Err(e) = launch_process(
logger,
API_SERVER_PATH,
&vec!["--features", &_config.rest_api],
"",
0,
) {
error!(logger, "launch_process {} failed: {:?}", API_SERVER_PATH, e);
}
}
}
Ok(())
}
fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {
let p = Path::new(path);
let mut attempts = 0;
loop {
std::thread::sleep(std::time::Duration::from_secs(1));
if p.exists() {
return Ok(());
}
if attempts >= timeout_secs {
break;
}
attempts += 1;
info!(
logger,
"waiting for {} to exist (attempts={})", path, attempts
);
}
Err(anyhow!("wait for {} to exist timeout.", path))
}
fn launch_process(
logger: &Logger,
path: &str,
args: &Vec<&str>,
unix_socket_path: &str,
timeout_secs: i32,
) -> Result<()> {
if !Path::new(path).exists() {
return Err(anyhow!("path {} does not exist.", path));
}
if !unix_socket_path.is_empty() && Path::new(unix_socket_path).exists() {
fs::remove_file(unix_socket_path)?;
}
Command::new(path).args(args).spawn()?;
if !unix_socket_path.is_empty() && timeout_secs > 0 {
wait_for_path_to_exist(logger, unix_socket_path, timeout_secs)?;
}
Ok(())
}
// init_agent_as_init will do the initializations such as setting up the rootfs
// when this agent has been run as the init process.
fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result<()> {

57
src/agent/src/rpc.rs
View File

@ -71,6 +71,7 @@ use crate::AGENT_CONFIG;
use crate::trace_rpc_call;
use crate::tracer::extract_carrier_from_ttrpc;
use derivative::Derivative;
use opentelemetry::global;
use tracing::span;
use tracing_opentelemetry::OpenTelemetrySpanExt;
@ -89,6 +90,9 @@ use std::io::{BufRead, BufReader, Write};
use std::os::unix::fs::FileExt;
use std::path::PathBuf;
#[cfg(feature = "sealed-secret")]
use crate::cdh::CDHClient;
pub const CONTAINER_BASE: &str = "/run/kata-containers";
const MODPROBE_PATH: &str = "/sbin/modprobe";
const CONFIG_JSON: &str = "config.json";
@ -139,10 +143,14 @@ fn is_allowed(req: &impl MessageDyn) -> ttrpc::Result<()> {
}
}
#[derive(Clone, Debug)]
#[derive(Derivative)]
#[derivative(Clone, Debug)]
pub struct AgentService {
sandbox: Arc<Mutex<Sandbox>>,
init_mode: bool,
#[derivative(Debug = "ignore")]
#[cfg(feature = "sealed-secret")]
cdh_client: Option<CDHClient>,
}
// A container ID must match this regex:
@ -201,6 +209,28 @@ impl AgentService {
// cannot predict everything from the caller.
add_devices(&req.devices, &mut oci, &self.sandbox).await?;
#[cfg(feature = "sealed-secret")]
let mut sealed_source_path = {
let process = oci
.process
.as_mut()
.ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
let client = self
.cdh_client
.as_ref()
.ok_or(anyhow!("get cdh_client failed"))?;
for env in process.env.iter_mut() {
let unsealed_env = client
.unseal_env(env)
.await
.map_err(|e| anyhow!("unseal env failed: {:?}", e))?;
*env = unsealed_env.to_string();
}
client.create_sealed_secret_mounts(&mut oci)?
};
let linux = oci
.linux
.as_mut()
@ -284,6 +314,17 @@ impl AgentService {
return Err(anyhow!(nix::Error::EINVAL));
};
#[cfg(feature = "sealed-secret")]
{
let client = self
.cdh_client
.as_ref()
.ok_or(anyhow!("get cdh_client failed"))?;
for source_path in sealed_source_path.iter_mut() {
client.unseal_file(source_path).await?;
}
}
// if starting container failed, we will do some rollback work
// to ensure no resources are leaked.
if let Err(err) = ctr.start(p).await {
@ -1697,6 +1738,8 @@ pub async fn start(
let agent_service = Box::new(AgentService {
sandbox: s.clone(),
init_mode,
#[cfg(feature = "sealed-secret")]
cdh_client: Some(CDHClient::new()?),
}) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
let aservice = agent_ttrpc::create_agent_service(Arc::new(agent_service));
@ -2212,6 +2255,8 @@ mod tests {
let agent_service = Box::new(AgentService {
sandbox: Arc::new(Mutex::new(sandbox)),
init_mode: true,
#[cfg(feature = "sealed-secret")]
cdh_client: None,
});
let req = protocols::agent::UpdateInterfaceRequest::default();
@ -2226,10 +2271,11 @@ mod tests {
async fn test_update_routes() {
let logger = slog::Logger::root(slog::Discard, o!());
let sandbox = Sandbox::new(&logger).unwrap();
let agent_service = Box::new(AgentService {
sandbox: Arc::new(Mutex::new(sandbox)),
init_mode: true,
#[cfg(feature = "sealed-secret")]
cdh_client: None,
});
let req = protocols::agent::UpdateRoutesRequest::default();
@ -2244,10 +2290,11 @@ mod tests {
async fn test_add_arp_neighbors() {
let logger = slog::Logger::root(slog::Discard, o!());
let sandbox = Sandbox::new(&logger).unwrap();
let agent_service = Box::new(AgentService {
sandbox: Arc::new(Mutex::new(sandbox)),
init_mode: true,
#[cfg(feature = "sealed-secret")]
cdh_client: None,
});
let req = protocols::agent::AddARPNeighborsRequest::default();
@ -2382,6 +2429,8 @@ mod tests {
let agent_service = Box::new(AgentService {
sandbox: Arc::new(Mutex::new(sandbox)),
init_mode: true,
#[cfg(feature = "sealed-secret")]
cdh_client: None,
});
let result = agent_service
@ -2863,6 +2912,8 @@ OtherField:other
let agent_service = Box::new(AgentService {
sandbox: Arc::new(Mutex::new(sandbox)),
init_mode: true,
#[cfg(feature = "sealed-secret")]
cdh_client: None,
});
let ctx = mk_ttrpc_context();

1
src/libs/protocols/Cargo.toml
View File

@ -9,6 +9,7 @@ license = "Apache-2.0"
default = []
with-serde = [ "serde", "serde_json" ]
async = ["ttrpc/async", "async-trait"]
sealed-secret = []
[dependencies]
ttrpc = { version = "0.7.1" }

9
src/libs/protocols/build.rs
View File

@ -204,6 +204,8 @@ fn real_main() -> Result<(), std::io::Error> {
"protos/agent.proto",
"protos/health.proto",
"protos/image.proto",
#[cfg(feature = "sealed-secret")]
"protos/sealed_secret.proto",
],
true,
)?;
@ -211,6 +213,11 @@ fn real_main() -> Result<(), std::io::Error> {
fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?;
fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?;
fs::rename("src/image_ttrpc.rs", "src/image_ttrpc_async.rs")?;
#[cfg(feature = "sealed-secret")]
fs::rename(
"src/sealed_secret_ttrpc.rs",
"src/sealed_secret_ttrpc_async.rs",
)?;
}
codegen(
@ -219,6 +226,8 @@ fn real_main() -> Result<(), std::io::Error> {
"protos/agent.proto",
"protos/health.proto",
"protos/image.proto",
#[cfg(feature = "sealed-secret")]
"protos/sealed_secret.proto",
],
false,
)?;

27
src/libs/protocols/protos/sealed_secret.proto Normal file
View File

@ -0,0 +1,27 @@
syntax = "proto3";
package api;
message UnsealSecretInput {
bytes secret = 1;
}
message UnsealSecretOutput {
bytes plaintext = 1;
}
message GetResourceRequest {
string ResourcePath = 1;
}
message GetResourceResponse {
bytes Resource = 1;
}
service SealedSecretService {
rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {};
}
service GetResourceService {
rpc GetResource(GetResourceRequest) returns (GetResourceResponse) {};
}

7
src/libs/protocols/src/lib.rs
View File

@ -31,3 +31,10 @@ pub use serde_config::{
deserialize_enum_or_unknown, deserialize_message_field, serialize_enum_or_unknown,
serialize_message_field,
};
#[cfg(feature = "sealed-secret")]
pub mod sealed_secret;
#[cfg(feature = "sealed-secret")]
pub mod sealed_secret_ttrpc;
#[cfg(all(feature = "sealed-secret", feature = "async"))]
pub mod sealed_secret_ttrpc_async;

4
src/runtime/Makefile
View File

@ -278,6 +278,9 @@ DEFSERVICEOFFLOAD ?= false
# Image Request Timeout in seconds
DEFIMAGEREQUESTTIMEOUT ?= 60
# Enable sealed secret or not
DEFSEALEDSECRETENABLED ?= false
# SEV & SEV-ES Guest Pre-Attestation
DEFGUESTPREATTESTATION ?= false
DEFGUESTPREATTESTATIONPROXY ?= localhost:44444
@ -709,6 +712,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_TEE
USER_VARS += DEFBINDMOUNTS
USER_VARS += DEFSERVICEOFFLOAD
USER_VARS += DEFIMAGEREQUESTTIMEOUT
USER_VARS += DEFSEALEDSECRETENABLED
USER_VARS += DEFVFIOMODE
USER_VARS += BUILDFLAGS
USER_VARS += DEFSERVICEOFFLOAD

5
src/runtime/config/configuration-clh-tdx.toml.in
View File

@ -415,6 +415,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-clh.toml.in
View File

@ -434,6 +434,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-qemu-nvidia-gpu.toml.in
View File

@ -669,6 +669,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-qemu-se.toml.in
View File

@ -645,6 +645,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-qemu-sev.toml.in
View File

@ -649,6 +649,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-qemu-snp.toml.in
View File

@ -674,6 +674,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-qemu-tdx.toml.in
View File

@ -662,6 +662,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-qemu.toml.in
View File

@ -709,6 +709,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

5
src/runtime/config/configuration-remote.toml.in
View File

@ -289,6 +289,11 @@ experimental=@DEFAULTEXPFEATURES@
# (default: false)
# enable_pprof = true
# Sealed secret feature configuration.
# If true, sealed secret feature will be enabled.
# (default: false)
sealed_secret_enabled = @DEFSEALEDSECRETENABLED@
# WARNING: All the options in the following section have not been implemented yet.
# This section was added as a placeholder. DO NOT USE IT!
[image]

2
src/runtime/pkg/katautils/config.go
View File

@ -191,6 +191,7 @@ type runtime struct {
StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"`
EnablePprof bool `toml:"enable_pprof"`
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
SealedSecretEnabled bool `toml:"sealed_secret_enabled"`
}
type agent struct {
@ -1453,6 +1454,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
config.JaegerEndpoint = tomlConf.Runtime.JaegerEndpoint
config.JaegerUser = tomlConf.Runtime.JaegerUser
config.JaegerPassword = tomlConf.Runtime.JaegerPassword
config.SealedSecretEnabled = tomlConf.Runtime.SealedSecretEnabled
config.ServiceOffload = tomlConf.Image.ServiceOffload
config.ImageRequestTimeout = tomlConf.Image.ImageRequestTimeout
for _, f := range tomlConf.Runtime.Experimental {

11
src/runtime/pkg/oci/utils.go
View File

@ -161,6 +161,9 @@ type RuntimeConfig struct {
// Image request timeout which, if provided, indicates the image request timeout
// in the guest needed for the workload(s)
ImageRequestTimeout uint64
// Sealed secret enabled configuration
SealedSecretEnabled bool
}
// AddKernelParam allows the addition of new kernel parameters to an existing
@ -918,6 +921,12 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r
}); err != nil {
return err
}
if err := newAnnotationConfiguration(ocispec, vcAnnotations.SealedSecretEnabled).setBool(func(SealedSecretEnabled bool) {
sbConfig.SealedSecretEnabled = SealedSecretEnabled
}); err != nil {
return err
}
return nil
}
@ -1033,6 +1042,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
ServiceOffload: runtime.ServiceOffload,
ImageRequestTimeout: runtime.ImageRequestTimeout,
SealedSecretEnabled: runtime.SealedSecretEnabled,
}
if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {

2
src/runtime/pkg/oci/utils_test.go
View File

@ -811,6 +811,7 @@ func TestAddRuntimeAnnotations(t *testing.T) {
ocispec.Annotations[vcAnnotations.DisableNewNetNs] = "true"
ocispec.Annotations[vcAnnotations.InterNetworkModel] = "macvtap"
ocispec.Annotations[vcAnnotations.ImageRequestTimeout] = "100"
ocispec.Annotations[vcAnnotations.SealedSecretEnabled] = "true"
addAnnotations(ocispec, &config, runtimeConfig)
assert.Equal(config.DisableGuestSeccomp, true)
@ -818,6 +819,7 @@ func TestAddRuntimeAnnotations(t *testing.T) {
assert.Equal(config.NetworkConfig.DisableNewNetwork, true)
assert.Equal(config.NetworkConfig.InterworkingModel, vc.NetXConnectMacVtapModel)
assert.Equal(config.ImageRequestTimeout, uint64(100))
assert.Equal(config.SealedSecretEnabled, true)
}

10
src/runtime/virtcontainers/kata_agent.go
View File

@ -905,12 +905,18 @@ func (k *kataAgent) stopSandbox(ctx context.Context, sandbox *Sandbox) error {
return nil
}
func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts map[string]Mount) error {
func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts map[string]Mount, sealedSecretEnabled bool) error {
ociMounts := spec.Mounts
for index, m := range ociMounts {
if guestMount, ok := guestMounts[m.Destination]; ok {
k.Logger().Debugf("Replacing OCI mount (%s) source %s with %s", m.Destination, m.Source, guestMount.Source)
if sealedSecretEnabled {
if strings.Contains(m.Source, "kubernetes.io~secret") {
ociMounts[index].Destination = "/sealed" + m.Destination
k.Logger().Debugf("Replacing OCI mount (%s) with new destination %s", m.Destination, ociMounts[index].Destination)
}
}
ociMounts[index].Source = guestMount.Source
}
}
@ -1299,7 +1305,7 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
// We replace all OCI mount sources that match our container mount
// with the right source path (The guest one).
if err = k.replaceOCIMountSource(ociSpec, sharedDirMounts); err != nil {
if err = k.replaceOCIMountSource(ociSpec, sharedDirMounts, sandbox.config.SealedSecretEnabled); err != nil {
return nil, err
}

3
src/runtime/virtcontainers/pkg/annotations/annotations.go
View File

@ -291,6 +291,9 @@ const (
// ImageRequestTimeout is a sandbox annotaion that sets the image pull timeout in the guest.
ImageRequestTimeout = kataAnnotRuntimePrefix + "image_request_timeout"
// SealedSecretEnabled is a sandbox annotaion that enables the sealed secret feature.
SealedSecretEnabled = kataAnnotRuntimePrefix + "sealed_secret_enabled"
)
// Agent related annotations

2
src/runtime/virtcontainers/sandbox.go
View File

@ -164,6 +164,8 @@ type SandboxConfig struct {
// Image request timeout which, if provided, indicates the image request timeout
// in the guest needed for the workload(s)
ImageRequestTimeout uint64
// enable sealed secret feature
SealedSecretEnabled bool
// SharePidNs sets all containers to share the same sandbox level pid namespace.
SharePidNs bool
// SystemdCgroup enables systemd cgroup support

20
tools/osbuilder/rootfs-builder/rootfs.sh
View File

@ -28,6 +28,9 @@ LIBC=${LIBC:-musl}
# The kata agent enables seccomp feature.
# However, it is not enforced by default: you need to enable that in the main configuration file.
SECCOMP=${SECCOMP:-"yes"}
# The kata agent enables sealed-secret feature.
SEALED_SECRET=${SEALED_SECRET:-"no"}
CDH_RESOURCE_PROVIDER=${CDH_RESOURCE_PROVIDER:-"kbs"}
SELINUX=${SELINUX:-"no"}
lib_file="${script_dir}/../scripts/lib.sh"
@ -156,6 +159,10 @@ ROOTFS_DIR Path to the directory that is populated with the rootfs.
SECCOMP When set to "no", the kata-agent is built without seccomp capability.
Default value: "yes"
SEALED_SECRET When set to "yes", the kata-agent is built with sealed-secret
capability.
Default value: "no"
SELINUX When set to "yes", build the rootfs with the required packages to
enable SELinux in the VM.
Make sure the guest kernel is compiled with SELinux enabled.
@ -469,6 +476,7 @@ build_rootfs_distro()
--env INSIDE_CONTAINER=1 \
--env AA_KBC="${AA_KBC}" \
--env SECCOMP="${SECCOMP}" \
--env SEALED_SECRET="${SEALED_SECRET}" \
--env SELINUX="${SELINUX}" \
--env DEBUG="${DEBUG}" \
--env HOME="/root" \
@ -630,7 +638,7 @@ EOF
git checkout "${AGENT_VERSION}" && OK "git checkout successful" || die "checkout agent ${AGENT_VERSION} failed!"
fi
make clean
make LIBC=${LIBC} INIT=${AGENT_INIT} SECCOMP=${SECCOMP}
make LIBC=${LIBC} INIT=${AGENT_INIT} SECCOMP=${SECCOMP} SEALED_SECRET=${SEALED_SECRET}
make install DESTDIR="${ROOTFS_DIR}" LIBC=${LIBC} INIT=${AGENT_INIT}
if [ "${SECCOMP}" == "yes" ]; then
rm -rf "${libseccomp_install_dir}" "${gperf_install_dir}"
@ -692,6 +700,16 @@ EOF
make KBC=${AA_KBC} ttrpc=true
make install DESTDIR="${ROOTFS_DIR}/usr/local/bin/"
popd
pushd guest-components/confidential-data-hub
make RESOURCE_PROVIDER=${CDH_RESOURCE_PROVIDER}
make install DESTDIR="${ROOTFS_DIR}/usr/local/bin/"
popd
pushd guest-components/api-server-rest
cargo build --release --target-dir ./target
install -D -m0755 ./target/release/api-server-rest ${ROOTFS_DIR}/usr/local/bin/
popd
fi
if [ "${KATA_BUILD_CC}" == "yes" ]; then

3
tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
View File

@ -228,6 +228,7 @@ install_cc_image() {
export KATA_BUILD_CC=yes
export MEASURED_ROOTFS=yes
export DM_VERITY=yes
export SEALED_SECRET=yes
variant="${1:-}"
install_image "${variant}"
@ -269,7 +270,7 @@ install_cc_shimv2() {
export RUST_VERSION
export REMOVE_VMM_CONFIGS="acrn fc"
extra_opts="DEFSERVICEOFFLOAD=true"
extra_opts="DEFSERVICEOFFLOAD=true DEFSEALEDSECRETENABLED=true"
if [ "${MEASURED_ROOTFS}" == "yes" ]; then
if [ -f "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt" ]; then
root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder/root_hash_vanilla.txt")

2
versions.yaml
View File

@ -200,7 +200,7 @@ externals:
attestation-agent:
description: "Provide attested key unwrapping for image decryption"
url: "https://github.com/confidential-containers/guest-components/"
version: "53ddd632424432077e95d3901deb64727be0b4c1"
version: "29086ca22583f0dcfa6548866e9bf2a5e07881e9"
cni-plugins:
description: "CNI network plugins"