From effac686fc34f73b467d524b4a4d2bf314da259b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 10 Jun 2026 16:59:49 +0200 Subject: [PATCH] agent: resolve and launch CoCo guest components via extension manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the agent's handling of CoCo guest components data-driven so that a new extension bundle can be introduced without changing agent code. Add a new `extension` module defining a generic component manifest contract. When a CoCo extension image is mounted at /run/kata-extensions/coco/, the agent reads etc/kata-extensions/components.toml from it to discover: - path-only components (looked up by id under the `[paths]` table), used for the ocicrypt config and the pause bundle, and - launchable processes (`[[process]]` entries) describing the binary path, args, optional args, config file, environment, socket to wait on and timeout. `${var}` tokens in process fields are substituted from a fixed, documented context the agent builds at runtime (attestation socket/uri, config paths, rest-api features, initdata toml path, launch timeout, resolved ocicrypt config path). Referencing an unknown variable is a hard error, and manifest paths are validated against traversal, so the contract stays fail-closed. Processes are gated by a numeric `level` mapped from guest_components_procs (AttestationAgent < ConfidentialDataHub < ApiServerRest), preserving the existing implication semantics. When no extension is mounted, launch_plan() returns None and the agent uses a built-in plan that reproduces the legacy behaviour byte-for-byte (binaries under /usr/local/bin, /etc/ocicrypt_config.json, /pause_bundle). Monolithic / non-split confidential images are therefore unaffected. Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor --- src/agent/src/confidential_data_hub/image.rs | 21 +- src/agent/src/guest_extension_image.rs | 810 +++++++++++++++++++ src/agent/src/main.rs | 303 +++++-- 3 files changed, 1041 insertions(+), 93 deletions(-) create mode 100644 src/agent/src/guest_extension_image.rs diff --git a/src/agent/src/confidential_data_hub/image.rs b/src/agent/src/confidential_data_hub/image.rs index 2087b7a378..058befbecb 100644 --- a/src/agent/src/confidential_data_hub/image.rs +++ b/src/agent/src/confidential_data_hub/image.rs @@ -8,12 +8,13 @@ use safe_path::scoped_join; use std::collections::HashMap; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, Context, Result}; use kata_sys_util::validate::verify_id; use oci_spec::runtime as oci; +use crate::guest_extension_image; use crate::rpc::CONTAINER_BASE; use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL; @@ -23,6 +24,16 @@ pub const KATA_IMAGE_WORK_DIR: &str = "/run/kata-containers/image/"; const CONFIG_JSON: &str = "config.json"; const KATA_PAUSE_BUNDLE: &str = "/pause_bundle"; +// Resolve the pause bundle directory: when the CoCo extension image is mounted its +// manifest decides the location, otherwise the legacy rootfs path is used. +fn resolve_pause_bundle() -> Result { + guest_extension_image::resolve_component_path( + guest_extension_image::COCO_EXTENSION_NAME, + guest_extension_image::COCO_COMPONENT_PAUSE_BUNDLE, + KATA_PAUSE_BUNDLE, + ) +} + const K8S_CONTAINER_TYPE_KEYS: [&str; 2] = [ "io.kubernetes.cri.container-type", "io.kubernetes.cri-o.ContainerType", @@ -46,11 +57,11 @@ fn copy_if_not_exists(src: &Path, dst: &Path) -> Result<()> { /// get guest pause image process specification fn get_pause_image_process() -> Result { - let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE); + let guest_pause_bundle = resolve_pause_bundle()?; if !guest_pause_bundle.exists() { bail!("Pause image not present in rootfs"); } - let guest_pause_config = scoped_join(guest_pause_bundle, CONFIG_JSON)?; + let guest_pause_config = scoped_join(&guest_pause_bundle, CONFIG_JSON)?; let image_oci = oci::Spec::load(guest_pause_config.to_str().ok_or_else(|| { anyhow!( @@ -70,11 +81,11 @@ fn get_pause_image_process() -> Result { pub fn unpack_pause_image(cid: &str) -> Result { verify_id(cid).context("The guest pause image cid contains invalid characters.")?; - let guest_pause_bundle = Path::new(KATA_PAUSE_BUNDLE); + let guest_pause_bundle = resolve_pause_bundle()?; if !guest_pause_bundle.exists() { bail!("Pause image not present in rootfs"); } - let guest_pause_config = scoped_join(guest_pause_bundle, CONFIG_JSON)?; + let guest_pause_config = scoped_join(&guest_pause_bundle, CONFIG_JSON)?; info!(sl(), "use guest pause image cid {:?}", cid); let image_oci = oci::Spec::load(guest_pause_config.to_str().ok_or_else(|| { diff --git a/src/agent/src/guest_extension_image.rs b/src/agent/src/guest_extension_image.rs new file mode 100644 index 0000000000..6ea489eadf --- /dev/null +++ b/src/agent/src/guest_extension_image.rs @@ -0,0 +1,810 @@ +// Copyright (c) 2026 NVIDIA Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +//! Generic, data-driven contract for guest components shipped in extension images. +//! +//! An extension image is mounted read-only at `/run/kata-extensions//` and +//! ships a manifest (`etc/kata-extensions/components.toml`) that declares the +//! path-only components and launchable processes it provides. The agent consumes +//! that manifest and builds its launch plan from it, instead of carrying +//! hard-coded knowledge about any particular extension. +//! +//! This keeps the agent generic for anything that can be expressed purely in the +//! manifest: adding, removing or reordering processes, changing their args, env, +//! paths or `${var}` substitutions, or selecting a build variant all happen with +//! no agent change. Workflows that need the agent to actively do something new +//! (e.g. publish a new substitution variable into the context, or react to a +//! component in a bespoke way) still require agent changes -- the CoCo +//! guest-component launching built on top of this module is one such case. +//! +//! The manifest schema, the `${var}` substitution variables the agent publishes, +//! and the attester-variant / NVRC contract are documented in +//! `docs/design/proposals/composable-vm-images.md`, which is the source of +//! truth; this module must stay consistent with it. + +use anyhow::{anyhow, bail, Context, Result}; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +pub const COCO_COMPONENT_OCICRYPT_CONFIG: &str = "ocicrypt-config"; +pub const COCO_COMPONENT_PAUSE_BUNDLE: &str = "pause-bundle"; +pub const COCO_EXTENSION_NAME: &str = "coco"; + +const EXTENSIONS_ROOT: &str = "/run/kata-extensions"; +const COMPONENTS_MANIFEST_REL_PATH: &str = "etc/kata-extensions/components.toml"; + +#[derive(Debug, Deserialize)] +struct ExtensionManifest { + // Required: the manifest is a versioned contract, so a manifest without an + // explicit schema_version is rejected rather than silently assumed to be v1. + schema_version: u32, + #[serde(default)] + paths: HashMap, + #[serde(default)] + process: Vec, +} + +#[derive(Debug, Deserialize, Clone)] +struct RawProcessSpec { + id: String, + #[serde(default)] + level: u32, + // `path` is optional: a process either declares its `path` directly, or + // provides `variants` and lets `select` choose which one is active. + #[serde(default)] + path: Option, + #[serde(default)] + select: Option, + #[serde(default)] + variants: HashMap, + #[serde(default)] + args: Vec, + #[serde(default)] + optional_args: Vec, + #[serde(default)] + config: Option, + #[serde(default)] + env: HashMap, + #[serde(default)] + wait_socket: Option, + #[serde(default)] + timeout_secs: Option, +} + +#[derive(Debug, Deserialize, Clone)] +struct OptionalArgs { + when: String, + args: Vec, +} + +/// A selectable flavour of a process. The active variant contributes its +/// `path` (overriding the process-level `path`, if any) and merges its extra +/// `args` (appended) and `env` (overriding on key conflicts) onto the base +/// process definition. This lets a single extension image ship several flavours of +/// the same component (e.g. a stock attestation-agent and an NVIDIA-attester +/// build) and have the consumer pick one without forking the image. +#[derive(Debug, Deserialize, Clone, Default)] +struct Variant { + #[serde(default)] + path: Option, + #[serde(default)] + args: Vec, + #[serde(default)] + env: HashMap, +} + +/// Default variant name used when the `select` value is empty or unset. +const DEFAULT_VARIANT: &str = "default"; + +/// A fully resolved description of a process the agent should launch. Paths are +/// absolute, all `${var}` tokens have been substituted, and the gating decision +/// has already been made. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LaunchSpec { + pub id: String, + pub path: PathBuf, + pub args: Vec, + pub config: Option, + pub env: Vec<(String, String)>, + pub wait_socket: Option, + pub timeout_secs: u64, +} + +fn validate_extension_name(extension_name: &str) -> Result<()> { + if extension_name.is_empty() { + bail!("invalid empty extension name"); + } + if extension_name.contains('/') + || Path::new(extension_name).components().any(|c| { + matches!( + c, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) + { + bail!("invalid extension name '{}'", extension_name); + } + Ok(()) +} + +fn extension_root(extensions_root: &Path, extension_name: &str) -> Result { + validate_extension_name(extension_name)?; + Ok(extensions_root.join(extension_name)) +} + +fn manifest_path(extensions_root: &Path, extension_name: &str) -> Result { + Ok(extension_root(extensions_root, extension_name)?.join(COMPONENTS_MANIFEST_REL_PATH)) +} + +/// Absolute mount root of an extension (e.g. `/run/kata-extensions/coco`). Exposed so +/// callers can publish it as a manifest substitution variable. +pub fn extension_mount_root(extension_name: &str) -> Result { + extension_root(Path::new(EXTENSIONS_ROOT), extension_name) +} + +/// Resolve a relative path declared inside a manifest against the extension root, +/// rejecting absolute paths and `..` traversal. +fn resolve_rel_path(root: &Path, rel_path: &str, what: &str) -> Result { + let rel_path = rel_path.trim(); + if rel_path.is_empty() { + bail!("empty path for {}", what); + } + if Path::new(rel_path) + .components() + .any(|c| matches!(c, Component::ParentDir | Component::Prefix(_))) + { + bail!("invalid path '{}' for {}", rel_path, what); + } + Ok(root.join(rel_path.trim_start_matches('/'))) +} + +fn load_manifest( + extensions_root: &Path, + extension_name: &str, +) -> Result> { + let root = extension_root(extensions_root, extension_name)?; + if !root.exists() { + // The extension is not mounted at all; callers fall back to legacy paths or + // their built-in defaults. + return Ok(None); + } + + let manifest_path = manifest_path(extensions_root, extension_name)?; + if !manifest_path.exists() { + // A mounted extension without a manifest is a packaging bug: fail closed so + // the misconfiguration is surfaced rather than silently ignored. + bail!( + "extension '{}' is mounted but manifest is missing at {}", + extension_name, + manifest_path.display() + ); + } + + let data = fs::read_to_string(&manifest_path).with_context(|| { + format!( + "failed to read extension manifest {}", + manifest_path.display() + ) + })?; + let manifest: ExtensionManifest = toml::from_str(&data).with_context(|| { + format!( + "failed to parse extension manifest {}", + manifest_path.display() + ) + })?; + + if manifest.schema_version != 1 { + bail!( + "unsupported extension manifest schema_version {} in {}", + manifest.schema_version, + manifest_path.display() + ); + } + + Ok(Some(manifest)) +} + +fn resolve_component_path_in_root( + extensions_root: &Path, + extension_name: &str, + component_id: &str, + legacy_path: &str, +) -> Result { + let manifest = match load_manifest(extensions_root, extension_name)? { + Some(manifest) => manifest, + None => return Ok(PathBuf::from(legacy_path)), + }; + + let rel_path = manifest.paths.get(component_id).ok_or_else(|| { + anyhow!( + "component '{}' not found in extension '{}' manifest", + component_id, + extension_name + ) + })?; + + let root = extension_root(extensions_root, extension_name)?; + resolve_rel_path( + &root, + rel_path, + &format!( + "component '{}' in extension '{}'", + component_id, extension_name + ), + ) +} + +/// Resolve the absolute path of a path-only component declared in the extension's +/// `[paths]` table. When the extension is not mounted, `legacy_path` is returned so +/// non-extension (e.g. monolithic) images keep working unchanged. +pub fn resolve_component_path( + extension_name: &str, + component_id: &str, + legacy_path: &str, +) -> Result { + resolve_component_path_in_root( + Path::new(EXTENSIONS_ROOT), + extension_name, + component_id, + legacy_path, + ) +} + +/// Replace every `${name}` token in `template` with the matching value from +/// `ctx`. An unknown variable is a hard error (fail-closed). +fn substitute(template: &str, ctx: &HashMap) -> Result { + let mut out = String::with_capacity(template.len()); + let bytes = template.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'$' && i + 1 < bytes.len() && bytes[i + 1] == b'{' { + let start = i + 2; + let end = template[start..] + .find('}') + .map(|p| start + p) + .ok_or_else(|| anyhow!("unterminated '${{' in '{}'", template))?; + let name = &template[start..end]; + let value = ctx + .get(name) + .ok_or_else(|| anyhow!("unknown substitution variable '{}'", name))?; + out.push_str(value); + i = end + 1; + } else { + // Push a full UTF-8 char to avoid splitting multi-byte sequences. + let ch = template[i..].chars().next().unwrap(); + out.push(ch); + i += ch.len_utf8(); + } + } + Ok(out) +} + +// Resolve the active variant (if any) for a process. Returns `None` when the +// process declares no variants. +fn select_variant<'a>( + raw: &'a RawProcessSpec, + ctx: &HashMap, +) -> Result> { + if raw.variants.is_empty() { + return Ok(None); + } + + let mut selector = match &raw.select { + Some(s) => substitute(s, ctx)?, + None => String::new(), + }; + if selector.is_empty() { + selector = DEFAULT_VARIANT.to_string(); + } + + let variant = raw.variants.get(&selector).ok_or_else(|| { + anyhow!( + "process '{}' has no variant '{}' (available: {})", + raw.id, + selector, + { + let mut names: Vec<&str> = raw.variants.keys().map(String::as_str).collect(); + names.sort_unstable(); + names.join(", ") + } + ) + })?; + + Ok(Some(variant)) +} + +fn build_launch_spec( + root: &Path, + raw: &RawProcessSpec, + ctx: &HashMap, +) -> Result { + let variant = select_variant(raw, ctx)?; + + let path_template = variant + .and_then(|v| v.path.as_deref()) + .or(raw.path.as_deref()) + .ok_or_else(|| { + anyhow!( + "process '{}' declares neither 'path' nor a variant path", + raw.id + ) + })?; + let path_template = substitute(path_template, ctx)?; + let path = resolve_rel_path(root, &path_template, &format!("process '{}'", raw.id))?; + + let mut args = Vec::with_capacity(raw.args.len()); + for a in &raw.args { + args.push(substitute(a, ctx)?); + } + for group in &raw.optional_args { + let enabled = ctx.get(&group.when).map(|v| !v.is_empty()).unwrap_or(false); + if enabled { + for a in &group.args { + args.push(substitute(a, ctx)?); + } + } + } + if let Some(v) = variant { + for a in &v.args { + args.push(substitute(a, ctx)?); + } + } + + let config = match &raw.config { + Some(c) => { + let c = substitute(c, ctx)?; + if c.is_empty() { + None + } else { + Some(c) + } + } + None => None, + }; + + // Base env first, then variant env (variant overrides on key conflict). + let mut env_map: HashMap = HashMap::new(); + for (k, v) in &raw.env { + env_map.insert(k.clone(), substitute(v, ctx)?); + } + if let Some(var) = variant { + for (k, v) in &var.env { + env_map.insert(k.clone(), substitute(v, ctx)?); + } + } + let mut env: Vec<(String, String)> = env_map.into_iter().collect(); + // Deterministic order keeps behaviour reproducible and testable. + env.sort(); + + let wait_socket = match &raw.wait_socket { + Some(s) => { + let s = substitute(s, ctx)?; + if s.is_empty() { + None + } else { + Some(s) + } + } + None => None, + }; + + let timeout_secs = match &raw.timeout_secs { + Some(t) => { + let t = substitute(t, ctx)?; + if t.is_empty() { + 0 + } else { + t.parse::().with_context(|| { + format!("invalid timeout_secs '{}' for process '{}'", t, raw.id) + })? + } + } + None => 0, + }; + + Ok(LaunchSpec { + id: raw.id.clone(), + path, + args, + config, + env, + wait_socket, + timeout_secs, + }) +} + +fn launch_plan_in_root( + extensions_root: &Path, + extension_name: &str, + max_level: u32, + ctx: &HashMap, +) -> Result>> { + let manifest = match load_manifest(extensions_root, extension_name)? { + Some(manifest) => manifest, + None => return Ok(None), + }; + + let root = extension_root(extensions_root, extension_name)?; + let mut specs = Vec::new(); + for raw in &manifest.process { + if raw.level == 0 || raw.level > max_level { + continue; + } + specs.push(build_launch_spec(&root, raw, ctx)?); + } + Ok(Some(specs)) +} + +/// Build the ordered list of processes the agent should launch for an extension, +/// gated by `max_level` and with all `${var}` tokens substituted from `ctx`. +/// +/// Returns `Ok(None)` when the extension is not mounted, signalling the caller to +/// use its built-in default launch behaviour. +pub fn launch_plan( + extension_name: &str, + max_level: u32, + ctx: &HashMap, +) -> Result>> { + launch_plan_in_root(Path::new(EXTENSIONS_ROOT), extension_name, max_level, ctx) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn write_manifest(root: &Path, extension_name: &str, content: &str) { + let manifest_dir = root.join(extension_name).join("etc/kata-extensions"); + fs::create_dir_all(&manifest_dir).unwrap(); + fs::write(manifest_dir.join("components.toml"), content).unwrap(); + } + + fn ctx() -> HashMap { + let mut c = HashMap::new(); + c.insert("aa_attestation_uri".into(), "unix:///run/aa.sock".into()); + c.insert("aa_attestation_socket".into(), "/run/aa.sock".into()); + c.insert("aa_config_path".into(), "/run/initdata/aa.toml".into()); + c.insert("cdh_config_path".into(), "/run/initdata/cdh.toml".into()); + c.insert("cdh_socket".into(), "/run/cdh.sock".into()); + c.insert( + "ocicrypt_config_path".into(), + "/run/kata-extensions/coco/etc/ocicrypt_config.json".into(), + ); + c.insert("rest_api_features".into(), "all".into()); + c.insert("launch_process_timeout".into(), "6".into()); + c.insert("initdata_toml_path".into(), "".into()); + c.insert("attester_variant".into(), "".into()); + c.insert("extension_root".into(), "/run/kata-extensions/coco".into()); + c + } + + const COCO_MANIFEST: &str = r#" +schema_version = 1 + +[paths] +"ocicrypt-config" = "etc/ocicrypt_config.json" +"pause-bundle" = "pause_bundle" + +[[process]] +id = "attestation-agent" +level = 1 +args = ["--attestation_sock", "${aa_attestation_uri}"] +optional_args = [{ when = "initdata_toml_path", args = ["--initdata-toml", "${initdata_toml_path}"] }] +config = "${aa_config_path}" +wait_socket = "${aa_attestation_socket}" +timeout_secs = "${launch_process_timeout}" +select = "${attester_variant}" + + [process.variants.default] + path = "usr/local/bin/attestation-agent" + + [process.variants.nvidia] + path = "usr/local/bin/attestation-agent-nv" + env = { LD_LIBRARY_PATH = "${extension_root}/usr/local/lib:/run/kata-extensions/gpu/usr/lib" } + +[[process]] +id = "confidential-data-hub" +level = 2 +path = "usr/local/bin/confidential-data-hub" +config = "${cdh_config_path}" +env = { OCICRYPT_KEYPROVIDER_CONFIG = "${ocicrypt_config_path}", PATH = "${extension_root}/usr/sbin:/bin:/sbin:/usr/bin:/usr/sbin" } +wait_socket = "${cdh_socket}" +timeout_secs = "${launch_process_timeout}" + +[[process]] +id = "api-server-rest" +level = 3 +path = "usr/local/bin/api-server-rest" +args = ["--features", "${rest_api_features}"] +timeout_secs = "0" +"#; + + #[test] + fn falls_back_to_legacy_when_extension_not_mounted() { + let dir = tempdir().unwrap(); + let legacy = "/usr/local/bin/attestation-agent"; + let got = resolve_component_path_in_root( + dir.path(), + COCO_EXTENSION_NAME, + COCO_COMPONENT_OCICRYPT_CONFIG, + legacy, + ) + .unwrap(); + assert_eq!(got, PathBuf::from(legacy)); + } + + #[test] + fn launch_plan_is_none_when_extension_not_mounted() { + let dir = tempdir().unwrap(); + let got = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 3, &ctx()).unwrap(); + assert!(got.is_none()); + } + + #[test] + fn fails_when_extension_root_exists_but_manifest_missing() { + let dir = tempdir().unwrap(); + fs::create_dir_all(dir.path().join(COCO_EXTENSION_NAME)).unwrap(); + + let err = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 3, &ctx()).unwrap_err(); + assert!(err.to_string().contains("manifest is missing")); + } + + #[test] + fn resolves_component_from_manifest() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + let got = resolve_component_path_in_root( + dir.path(), + COCO_EXTENSION_NAME, + COCO_COMPONENT_PAUSE_BUNDLE, + "/legacy/pause_bundle", + ) + .unwrap(); + assert_eq!( + got, + dir.path().join(COCO_EXTENSION_NAME).join("pause_bundle") + ); + } + + #[test] + fn fails_when_component_is_missing_in_manifest() { + let dir = tempdir().unwrap(); + write_manifest( + dir.path(), + COCO_EXTENSION_NAME, + r#" +schema_version = 1 +[paths] +"pause-bundle" = "pause_bundle" +"#, + ); + let err = resolve_component_path_in_root( + dir.path(), + COCO_EXTENSION_NAME, + COCO_COMPONENT_OCICRYPT_CONFIG, + "/legacy", + ) + .unwrap_err(); + assert!(err.to_string().contains("not found")); + } + + #[test] + fn rejects_path_traversal_in_manifest() { + let dir = tempdir().unwrap(); + write_manifest( + dir.path(), + COCO_EXTENSION_NAME, + r#" +schema_version = 1 +[paths] +"ocicrypt-config" = "../outside" +"#, + ); + let err = resolve_component_path_in_root( + dir.path(), + COCO_EXTENSION_NAME, + COCO_COMPONENT_OCICRYPT_CONFIG, + "/legacy", + ) + .unwrap_err(); + assert!(err.to_string().contains("invalid path")); + } + + #[test] + fn builds_full_coco_launch_plan() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + let specs = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 3, &ctx()) + .unwrap() + .unwrap(); + + assert_eq!(specs.len(), 3); + + let root = dir.path().join(COCO_EXTENSION_NAME); + let aa = &specs[0]; + assert_eq!(aa.id, "attestation-agent"); + assert_eq!(aa.path, root.join("usr/local/bin/attestation-agent")); + assert_eq!(aa.args, vec!["--attestation_sock", "unix:///run/aa.sock"]); + assert_eq!(aa.config.as_deref(), Some("/run/initdata/aa.toml")); + assert_eq!(aa.wait_socket.as_deref(), Some("/run/aa.sock")); + assert_eq!(aa.timeout_secs, 6); + + let cdh = &specs[1]; + assert_eq!(cdh.id, "confidential-data-hub"); + // env is sorted by key, so OCICRYPT_KEYPROVIDER_CONFIG precedes PATH. + // PATH puts the extension's cryptsetup (usr/sbin) ahead of the base dirs + // that carry mke2fs/mkfs.ext4/dd, which CDH's secure_mount shells out to. + assert_eq!( + cdh.env, + vec![ + ( + "OCICRYPT_KEYPROVIDER_CONFIG".to_string(), + "/run/kata-extensions/coco/etc/ocicrypt_config.json".to_string() + ), + ( + "PATH".to_string(), + "/run/kata-extensions/coco/usr/sbin:/bin:/sbin:/usr/bin:/usr/sbin".to_string() + ) + ] + ); + + let api = &specs[2]; + assert_eq!(api.id, "api-server-rest"); + assert_eq!(api.args, vec!["--features", "all"]); + assert_eq!(api.timeout_secs, 0); + } + + #[test] + fn gating_limits_launched_processes() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + + let specs = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx()) + .unwrap() + .unwrap(); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].id, "attestation-agent"); + } + + #[test] + fn optional_args_included_when_variable_set() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + + let mut ctx = ctx(); + ctx.insert( + "initdata_toml_path".into(), + "/run/initdata/initdata.toml".into(), + ); + let specs = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx) + .unwrap() + .unwrap(); + assert_eq!( + specs[0].args, + vec![ + "--attestation_sock", + "unix:///run/aa.sock", + "--initdata-toml", + "/run/initdata/initdata.toml" + ] + ); + } + + #[test] + fn unknown_substitution_variable_fails() { + let dir = tempdir().unwrap(); + write_manifest( + dir.path(), + COCO_EXTENSION_NAME, + r#" +schema_version = 1 +[[process]] +id = "broken" +level = 1 +path = "usr/local/bin/broken" +args = ["--flag", "${does_not_exist}"] +"#, + ); + let err = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx()).unwrap_err(); + assert!(err.to_string().contains("unknown substitution variable")); + } + + #[test] + fn unsupported_schema_version_fails() { + let dir = tempdir().unwrap(); + write_manifest( + dir.path(), + COCO_EXTENSION_NAME, + r#" +schema_version = 2 +[paths] +"pause-bundle" = "pause_bundle" +"#, + ); + let err = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 3, &ctx()).unwrap_err(); + assert!(err.to_string().contains("schema_version")); + } + + #[test] + fn default_variant_selected_when_selector_empty() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + + // ctx() leaves attester_variant empty -> "default". + let specs = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx()) + .unwrap() + .unwrap(); + let aa = &specs[0]; + assert_eq!( + aa.path, + dir.path() + .join(COCO_EXTENSION_NAME) + .join("usr/local/bin/attestation-agent") + ); + assert!(aa.env.is_empty()); + } + + #[test] + fn nvidia_variant_selects_nv_binary_and_merges_env() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + + let mut ctx = ctx(); + ctx.insert("attester_variant".into(), "nvidia".into()); + let specs = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx) + .unwrap() + .unwrap(); + let aa = &specs[0]; + assert_eq!( + aa.path, + dir.path() + .join(COCO_EXTENSION_NAME) + .join("usr/local/bin/attestation-agent-nv") + ); + assert_eq!( + aa.env, + vec![( + "LD_LIBRARY_PATH".to_string(), + "/run/kata-extensions/coco/usr/local/lib:/run/kata-extensions/gpu/usr/lib" + .to_string() + )] + ); + // Base args are preserved for the selected variant. + assert_eq!(aa.args, vec!["--attestation_sock", "unix:///run/aa.sock"]); + } + + #[test] + fn unknown_variant_fails() { + let dir = tempdir().unwrap(); + write_manifest(dir.path(), COCO_EXTENSION_NAME, COCO_MANIFEST); + + let mut ctx = ctx(); + ctx.insert("attester_variant".into(), "does-not-exist".into()); + let err = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx).unwrap_err(); + assert!(err.to_string().contains("has no variant 'does-not-exist'")); + } + + #[test] + fn process_without_path_or_variants_fails() { + let dir = tempdir().unwrap(); + write_manifest( + dir.path(), + COCO_EXTENSION_NAME, + r#" +schema_version = 1 +[[process]] +id = "broken" +level = 1 +args = ["--flag"] +"#, + ); + let err = launch_plan_in_root(dir.path(), COCO_EXTENSION_NAME, 1, &ctx()).unwrap_err(); + assert!(err + .to_string() + .contains("neither 'path' nor a variant path")); + } +} diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 50f396ebb3..a362396b6f 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -43,6 +43,7 @@ mod config; mod console; mod device; mod features; +mod guest_extension_image; mod initdata; mod linux_abi; mod metrics; @@ -96,6 +97,9 @@ const NAME: &str = "kata-agent"; const UNIX_SOCKET_PREFIX: &str = "unix://"; +// Legacy (non-extension) rootfs locations for the CoCo guest components. They are +// used to build the built-in launch plan when no CoCo extension image is mounted, +// keeping monolithic / non-confidential images working unchanged. const AA_PATH: &str = "/usr/local/bin/attestation-agent"; const AA_ATTESTATION_SOCKET: &str = "/run/confidential-containers/attestation-agent/attestation-agent.sock"; @@ -107,8 +111,6 @@ const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET); const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest"; -/// Path of ocicrypt config file. This is used by CDH when decrypting image. -/// TODO: remove this when we move the launch of CDH out of the kata-agent. const OCICRYPT_CONFIG_PATH: &str = "/etc/ocicrypt_config.json"; lazy_static! { @@ -412,13 +414,14 @@ async fn start_sandbox( let initdata_return_value = initdata::initialize_initdata(logger).await?; let gc_procs = config.guest_components_procs; - if !attestation_binaries_available(logger, &gc_procs) { + let launch_plan = build_coco_launch_plan(config, &initdata_return_value, gc_procs)?; + if !attestation_components_available(logger, &launch_plan) { warn!( logger, "attestation binaries requested for launch not available" ); } else { - init_attestation_components(logger, config, &initdata_return_value).await?; + init_attestation_components(logger, &launch_plan).await?; } // if policy is given via initdata, use it @@ -461,26 +464,192 @@ async fn start_sandbox( Ok(()) } -// Check if required attestation binaries are available on the rootfs. -fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs) -> bool { - let binaries = match procs { - GuestComponentsProcs::AttestationAgent => vec![AA_PATH], - GuestComponentsProcs::ConfidentialDataHub => vec![AA_PATH, CDH_PATH], - GuestComponentsProcs::ApiServerRest => vec![AA_PATH, CDH_PATH, API_SERVER_PATH], - _ => vec![], +// Map the requested guest-components level to the numeric gating level used by +// extension manifests. A process is launched only when its declared `level` is +// <= this value. The ordering mirrors the implications documented on +// `GuestComponentsProcs` (ApiServerRest implies CDH implies AttestationAgent). +fn guest_components_max_level(procs: GuestComponentsProcs) -> u32 { + match procs { + GuestComponentsProcs::None => 0, + GuestComponentsProcs::AttestationAgent => 1, + GuestComponentsProcs::ConfidentialDataHub => 2, + GuestComponentsProcs::ApiServerRest => 3, + } +} + +// Build the substitution context exposed to extension manifests. New extension bundles +// can rely on these variables without requiring agent code changes; introducing +// a brand new variable is the only case that needs touching the agent. +fn build_substitution_ctx( + config: &AgentConfig, + initdata_return_value: &Option, +) -> Result> { + let ocicrypt_config_path = guest_extension_image::resolve_component_path( + guest_extension_image::COCO_EXTENSION_NAME, + guest_extension_image::COCO_COMPONENT_OCICRYPT_CONFIG, + OCICRYPT_CONFIG_PATH, + )?; + + let initdata_toml_path = if initdata_return_value.is_some() { + initdata::INITDATA_TOML_PATH.to_string() + } else { + String::new() }; - for binary in binaries.iter() { - let exists = Path::new(binary) + + let extension_root = + guest_extension_image::extension_mount_root(guest_extension_image::COCO_EXTENSION_NAME)?; + + let mut ctx = std::collections::HashMap::new(); + ctx.insert( + "aa_attestation_uri".to_string(), + AA_ATTESTATION_URI.to_string(), + ); + ctx.insert( + "aa_attestation_socket".to_string(), + AA_ATTESTATION_SOCKET.to_string(), + ); + ctx.insert("aa_config_path".to_string(), AA_CONFIG_PATH.to_string()); + ctx.insert("cdh_config_path".to_string(), CDH_CONFIG_PATH.to_string()); + ctx.insert("cdh_socket".to_string(), CDH_SOCKET.to_string()); + ctx.insert( + "ocicrypt_config_path".to_string(), + ocicrypt_config_path.to_string_lossy().into_owned(), + ); + ctx.insert( + "rest_api_features".to_string(), + config.guest_components_rest_api.to_string(), + ); + ctx.insert( + "launch_process_timeout".to_string(), + config.launch_process_timeout.as_secs().to_string(), + ); + ctx.insert("initdata_toml_path".to_string(), initdata_toml_path); + ctx.insert( + "extension_root".to_string(), + extension_root.to_string_lossy().into_owned(), + ); + // The CoCo extension ships several attestation-agent flavours and selects one + // via the manifest's "attester_variant". The guest init (NVRC) owns that + // decision: with a GPU present it sets KATA_ATTESTER_VARIANT=nvidia so the + // NVIDIA-attester build launches (it emits the GPU evidence a KBS GPU + // policy requires). Absent that signal we fall back to the stock attester. + // Cross-component contract: the env var name and "nvidia" value are set by + // NVRC (src/kata_agent.rs, src/gpu.rs); keep them in sync. + let attester_variant = env::var("KATA_ATTESTER_VARIANT") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "default".to_string()); + ctx.insert("attester_variant".to_string(), attester_variant); + + Ok(ctx) +} + +// Built-in launch plan used when no CoCo extension image is mounted. It reproduces +// the legacy behaviour of launching the guest components from the rootfs +// (`/usr/local/bin/...`), so monolithic and non-confidential images are +// unaffected by the extension machinery. +fn builtin_coco_plan( + config: &AgentConfig, + initdata_return_value: &Option, + max_level: u32, +) -> Vec { + let mut plan = Vec::new(); + + if max_level >= 1 { + let mut args = vec![ + "--attestation_sock".to_string(), + AA_ATTESTATION_URI.to_string(), + ]; + if initdata_return_value.is_some() { + args.push("--initdata-toml".to_string()); + args.push(initdata::INITDATA_TOML_PATH.to_string()); + } + plan.push(guest_extension_image::LaunchSpec { + id: "attestation-agent".to_string(), + path: Path::new(AA_PATH).to_path_buf(), + args, + config: Some(AA_CONFIG_PATH.to_string()), + env: vec![], + wait_socket: Some(AA_ATTESTATION_SOCKET.to_string()), + timeout_secs: config.launch_process_timeout.as_secs(), + }); + } + + if max_level >= 2 { + plan.push(guest_extension_image::LaunchSpec { + id: "confidential-data-hub".to_string(), + path: Path::new(CDH_PATH).to_path_buf(), + args: vec![], + config: Some(CDH_CONFIG_PATH.to_string()), + env: vec![( + "OCICRYPT_KEYPROVIDER_CONFIG".to_string(), + OCICRYPT_CONFIG_PATH.to_string(), + )], + wait_socket: Some(CDH_SOCKET.to_string()), + timeout_secs: config.launch_process_timeout.as_secs(), + }); + } + + if max_level >= 3 { + plan.push(guest_extension_image::LaunchSpec { + id: "api-server-rest".to_string(), + path: Path::new(API_SERVER_PATH).to_path_buf(), + args: vec![ + "--features".to_string(), + config.guest_components_rest_api.to_string(), + ], + config: None, + env: vec![], + wait_socket: None, + timeout_secs: 0, + }); + } + + plan +} + +// Build the ordered launch plan for the guest components. When a CoCo extension +// image is mounted its manifest drives the plan (so new bundles need no agent +// changes); otherwise the built-in legacy plan is used. +fn build_coco_launch_plan( + config: &AgentConfig, + initdata_return_value: &Option, + procs: GuestComponentsProcs, +) -> Result> { + let max_level = guest_components_max_level(procs); + let ctx = build_substitution_ctx(config, initdata_return_value)?; + match guest_extension_image::launch_plan( + guest_extension_image::COCO_EXTENSION_NAME, + max_level, + &ctx, + )? { + Some(plan) => Ok(plan), + None => Ok(builtin_coco_plan(config, initdata_return_value, max_level)), + } +} + +// Check that every process in the launch plan is present on disk. A missing +// binary means the components were not provisioned (e.g. a non-confidential +// rootfs), in which case launching is skipped. +fn attestation_components_available( + logger: &Logger, + plan: &[guest_extension_image::LaunchSpec], +) -> bool { + for spec in plan { + let exists = spec + .path .try_exists() .unwrap_or_else(|error| match error.kind() { - ErrorKind::NotFound => { - warn!(logger, "{} not found", binary); - false - } - _ => panic!("Path existence check failed for '{}': {}", binary, error), + ErrorKind::NotFound => false, + _ => panic!( + "Path existence check failed for '{}': {}", + spec.path.display(), + error + ), }); if !exists { + warn!(logger, "{} not found", spec.path.display()); return false; } } @@ -489,76 +658,35 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs) async fn launch_guest_component_procs( logger: &Logger, - config: &AgentConfig, - initdata_return_value: &Option, + plan: &[guest_extension_image::LaunchSpec], ) -> Result<()> { - if config.guest_components_procs == GuestComponentsProcs::None { - return Ok(()); + for spec in plan { + let path = spec + .path + .to_str() + .ok_or_else(|| anyhow!("non-utf8 component path {}", spec.path.display()))?; + debug!(logger, "spawning extension component process {}", spec.id); + + let args: Vec<&str> = spec.args.iter().map(String::as_str).collect(); + let envs: Vec<(&str, &str)> = spec + .env + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + + launch_process( + logger, + path, + args, + spec.config.as_deref(), + spec.wait_socket.as_deref().unwrap_or(""), + spec.timeout_secs, + &envs, + ) + .await + .map_err(|e| anyhow!("launch_process {} failed: {:?}", path, e))?; } - debug!(logger, "spawning attestation-agent process {}", AA_PATH); - let mut aa_args = vec!["--attestation_sock", AA_ATTESTATION_URI]; - if initdata_return_value.is_some() { - aa_args.push("--initdata-toml"); - aa_args.push(initdata::INITDATA_TOML_PATH); - } - - launch_process( - logger, - AA_PATH, - aa_args, - Some(AA_CONFIG_PATH), - AA_ATTESTATION_SOCKET, - config.launch_process_timeout.as_secs(), - &[], - ) - .await - .map_err(|e| anyhow!("launch_process {} failed: {:?}", AA_PATH, e))?; - - // skip launch of confidential-data-hub and api-server-rest - if config.guest_components_procs == GuestComponentsProcs::AttestationAgent { - return Ok(()); - } - - debug!( - logger, - "spawning confidential-data-hub process {}", CDH_PATH - ); - - launch_process( - logger, - CDH_PATH, - vec![], - Some(CDH_CONFIG_PATH), - CDH_SOCKET, - config.launch_process_timeout.as_secs(), - &[("OCICRYPT_KEYPROVIDER_CONFIG", OCICRYPT_CONFIG_PATH)], - ) - .await - .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?; - - // skip launch of api-server-rest - if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub { - return Ok(()); - } - - let features = config.guest_components_rest_api; - debug!( - logger, - "spawning api-server-rest process {} --features {}", API_SERVER_PATH, features - ); - launch_process( - logger, - API_SERVER_PATH, - vec!["--features", &features.to_string()], - None, - "", - 0, - &[], - ) - .await - .map_err(|e| anyhow!("launch_process {} failed: {:?}", API_SERVER_PATH, e))?; - Ok(()) } @@ -568,10 +696,9 @@ async fn launch_guest_component_procs( // If the CDH is started, a CDH client will be instantiated and returned. async fn init_attestation_components( logger: &Logger, - config: &AgentConfig, - initdata_return_value: &Option, + plan: &[guest_extension_image::LaunchSpec], ) -> Result<()> { - launch_guest_component_procs(logger, config, initdata_return_value).await?; + launch_guest_component_procs(logger, plan).await?; // If a CDH socket exists, initialize the CDH client and enable ocicrypt match tokio::fs::metadata(CDH_SOCKET).await {