diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 745143bfbb..9cd4fe6cf1 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -2222,6 +2222,7 @@ dependencies = [ "cgroups-rs", "clap", "const_format", + "derivative", "futures", "image-rs", "ipnetwork", diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index a9eb53b95e..7cef491205 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -23,6 +23,7 @@ regex = "1.10.4" serial_test = "0.5.1" oci-distribution = "0.10.0" url = "2.5.0" +derivative = "2.2.0" kata-sys-util = { path = "../libs/kata-sys-util" } kata-types = { path = "../libs/kata-types" } safe-path = { path = "../libs/safe-path" } diff --git a/src/agent/src/cdh.rs b/src/agent/src/cdh.rs new file mode 100644 index 0000000000..7fe529e471 --- /dev/null +++ b/src/agent/src/cdh.rs @@ -0,0 +1,150 @@ +// Copyright (c) 2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// Confidential Data Hub client wrapper. +// Confidential Data Hub is a service running inside guest to provide resource related APIs. +// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub + +use anyhow::Result; +use derivative::Derivative; +use protocols::{ + sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient, +}; + +use crate::CDH_SOCKET_URI; + +// Nanoseconds +const CDH_UNSEAL_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000; +const SEALED_SECRET_PREFIX: &str = "sealed."; + +#[derive(Derivative)] +#[derivative(Clone, Debug)] +pub struct CDHClient { + #[derivative(Debug = "ignore")] + sealed_secret_client: SealedSecretServiceClient, +} + +impl CDHClient { + pub fn new() -> Result { + let client = ttrpc::asynchronous::Client::connect(CDH_SOCKET_URI)?; + let sealed_secret_client = + sealed_secret_ttrpc_async::SealedSecretServiceClient::new(client); + + Ok(CDHClient { + sealed_secret_client, + }) + } + + pub async fn unseal_secret_async(&self, sealed_secret: &str) -> Result> { + let mut input = sealed_secret::UnsealSecretInput::new(); + input.set_secret(sealed_secret.into()); + + let unsealed_secret = self + .sealed_secret_client + .unseal_secret(ttrpc::context::with_timeout(CDH_UNSEAL_TIMEOUT), &input) + .await?; + Ok(unsealed_secret.plaintext) + } + + pub async fn unseal_env(&self, env: &str) -> Result { + if let Some((key, value)) = env.split_once('=') { + if value.starts_with(SEALED_SECRET_PREFIX) { + let unsealed_value = self.unseal_secret_async(value).await?; + let unsealed_env = format!("{}={}", key, std::str::from_utf8(&unsealed_value)?); + + return Ok(unsealed_env); + } + } + + Ok((*env.to_owned()).to_string()) + } +} + +#[cfg(test)] +#[cfg(feature = "sealed-secret")] +mod tests { + use crate::cdh::CDHClient; + use crate::cdh::CDH_ADDR; + use anyhow::anyhow; + use async_trait::async_trait; + use protocols::{sealed_secret, sealed_secret_ttrpc_async}; + use std::sync::Arc; + use test_utils::skip_if_not_root; + use tokio::signal::unix::{signal, SignalKind}; + + struct TestService; + + #[async_trait] + impl sealed_secret_ttrpc_async::SealedSecretService for TestService { + async fn unseal_secret( + &self, + _ctx: &::ttrpc::asynchronous::TtrpcContext, + _req: sealed_secret::UnsealSecretInput, + ) -> ttrpc::error::Result { + let mut output = sealed_secret::UnsealSecretOutput::new(); + output.set_plaintext("unsealed".into()); + Ok(output) + } + } + + fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> { + let path = sock_addr + .strip_prefix("unix://") + .expect("socket address does not have the expected format."); + + if std::path::Path::new(path).exists() { + std::fs::remove_file(path)?; + } + + Ok(()) + } + + fn start_ttrpc_server() { + tokio::spawn(async move { + let ss = Box::new(TestService {}) + as Box; + let ss = Arc::new(ss); + let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss); + + remove_if_sock_exist(CDH_ADDR).unwrap(); + + let mut server = ttrpc::asynchronous::Server::new() + .bind(CDH_ADDR) + .unwrap() + .register_service(ss_service); + + server.start().await.unwrap(); + + let mut interrupt = signal(SignalKind::interrupt()).unwrap(); + tokio::select! { + _ = interrupt.recv() => { + server.shutdown().await.unwrap(); + } + }; + }); + } + + #[tokio::test] + async fn test_unseal_env() { + skip_if_not_root!(); + + let rt = tokio::runtime::Runtime::new().unwrap(); + let _guard = rt.enter(); + start_ttrpc_server(); + std::thread::sleep(std::time::Duration::from_secs(2)); + + let cc = Some(CDHClient::new().unwrap()); + let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap(); + let sealed_env = String::from("key=sealed.testdata"); + let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap(); + assert_eq!(unsealed_env, String::from("key=unsealed")); + let normal_env = String::from("key=testdata"); + let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap(); + assert_eq!(unchanged_env, String::from("key=testdata")); + + rt.shutdown_background(); + std::thread::sleep(std::time::Duration::from_secs(2)); + } +} diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 0450b1bcc3..299203b278 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -38,6 +38,7 @@ use std::process::Command; use std::sync::Arc; use tracing::{instrument, span}; +mod cdh; mod config; mod console; mod device; @@ -59,6 +60,7 @@ mod util; mod version; mod watcher; +use cdh::CDHClient; use config::GuestComponentsProcs; use mount::{cgroups_mount, general_mount}; use sandbox::Sandbox; @@ -104,6 +106,7 @@ const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SO const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub"; const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock"; +const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET); const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest"; @@ -403,6 +406,7 @@ async fn start_sandbox( let (tx, rx) = tokio::sync::oneshot::channel(); sandbox.lock().await.sender = Some(tx); + let mut cdh_client = None; let gc_procs = config.guest_components_procs; if gc_procs != GuestComponentsProcs::None { if !attestation_binaries_available(logger, &gc_procs) { @@ -411,12 +415,19 @@ async fn start_sandbox( "attestation binaries requested for launch not available" ); } else { - init_attestation_components(logger, config)?; + cdh_client = init_attestation_components(logger, config)?; } } // vsock:///dev/vsock, port - let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?; + let mut server = rpc::start( + sandbox.clone(), + config.server_addr.as_str(), + init_mode, + cdh_client, + ) + .await?; + server.start().await?; rx.await?; @@ -445,10 +456,11 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs) // Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs // and the corresponding procs are enabled in the agent configuration. the process will be // launched in the background and the function will return immediately. -fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> { +// If the CDH is started, a CDH client will be instantiated and returned. +fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result> { // skip launch of any guest-component if config.guest_components_procs == GuestComponentsProcs::None { - return Ok(()); + return Ok(None); } debug!(logger, "spawning attestation-agent process {}", AA_PATH); @@ -463,7 +475,7 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result< // skip launch of confidential-data-hub and api-server-rest if config.guest_components_procs == GuestComponentsProcs::AttestationAgent { - return Ok(()); + return Ok(None); } debug!( @@ -479,9 +491,11 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result< ) .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?; + let cdh_client = CDHClient::new().context("Failed to create CDH Client")?; + // skip launch of api-server-rest if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub { - return Ok(()); + return Ok(Some(cdh_client)); } let features = config.guest_components_rest_api; @@ -498,7 +512,7 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result< ) .map_err(|e| anyhow!("launch_process {} failed: {:?}", API_SERVER_PATH, e))?; - Ok(()) + Ok(Some(cdh_client)) } fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> { diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index cfafc5b21a..5c60f943a5 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -76,6 +76,8 @@ use crate::policy::{do_set_policy, is_allowed}; #[cfg(feature = "guest-pull")] use crate::image; +use crate::cdh::CDHClient; + use opentelemetry::global; use tracing::span; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -171,6 +173,7 @@ impl OptionToTtrpcResult for Option { pub struct AgentService { sandbox: Arc>, init_mode: bool, + cdh_client: Option, } impl AgentService { @@ -217,6 +220,22 @@ impl AgentService { // cannot predict everything from the caller. add_devices(&req.devices, &mut oci, &self.sandbox).await?; + if let Some(cdh) = self.cdh_client.as_ref() { + let process = oci + .process + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't contain process field"))?; + + for env in process.env.iter_mut() { + match cdh.unseal_env(env).await { + Ok(unsealed_env) => *env = unsealed_env.to_string(), + Err(e) => { + warn!(sl(), "Failed to unseal secret: {}", e) + } + } + } + } + // Both rootfs and volumes (invoked with --volume for instance) will // be processed the same way. The idea is to always mount any provided // storage to the specified MountPoint, so that it will match what's @@ -1596,10 +1615,12 @@ pub async fn start( s: Arc>, server_address: &str, init_mode: bool, + cdh_client: Option, ) -> Result { let agent_service = Box::new(AgentService { sandbox: s, init_mode, + cdh_client, }) as Box; let aservice = agent_ttrpc::create_agent_service(Arc::new(agent_service)); @@ -2150,6 +2171,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + cdh_client: None, }); let req = protocols::agent::UpdateInterfaceRequest::default(); @@ -2164,10 +2186,10 @@ mod tests { async fn test_update_routes() { let logger = slog::Logger::root(slog::Discard, o!()); let sandbox = Sandbox::new(&logger).unwrap(); - let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + cdh_client: None, }); let req = protocols::agent::UpdateRoutesRequest::default(); @@ -2182,10 +2204,10 @@ mod tests { async fn test_add_arp_neighbors() { let logger = slog::Logger::root(slog::Discard, o!()); let sandbox = Sandbox::new(&logger).unwrap(); - let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + cdh_client: None, }); let req = protocols::agent::AddARPNeighborsRequest::default(); @@ -2324,6 +2346,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + cdh_client: None, }); let result = agent_service @@ -2813,6 +2836,7 @@ OtherField:other let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), init_mode: true, + cdh_client: None, }); let ctx = mk_ttrpc_context(); diff --git a/src/libs/protocols/build.rs b/src/libs/protocols/build.rs index bc34c07a07..028bbd76d3 100644 --- a/src/libs/protocols/build.rs +++ b/src/libs/protocols/build.rs @@ -198,13 +198,34 @@ fn real_main() -> Result<(), std::io::Error> { // generate async #[cfg(feature = "async")] { - codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?; + + codegen( + "src", + &[ + "protos/agent.proto", + "protos/health.proto", + "protos/sealed_secret.proto", + ], + true, + )?; fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?; fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?; + fs::rename( + "src/sealed_secret_ttrpc.rs", + "src/sealed_secret_ttrpc_async.rs", + )?; } - codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?; + codegen( + "src", + &[ + "protos/agent.proto", + "protos/health.proto", + "protos/sealed_secret.proto", + ], + false, + )?; // There is a message named 'Box' in oci.proto // so there is a struct named 'Box', we should replace Box to ::std::boxed::Box diff --git a/src/libs/protocols/protos/sealed_secret.proto b/src/libs/protocols/protos/sealed_secret.proto new file mode 100644 index 0000000000..4e886ab2c4 --- /dev/null +++ b/src/libs/protocols/protos/sealed_secret.proto @@ -0,0 +1,21 @@ +// +// Copyright (c) 2024 IBM +// +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; + +package api; + +message UnsealSecretInput { + bytes secret = 1; +} + +message UnsealSecretOutput { + bytes plaintext = 1; +} + +service SealedSecretService { + rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {}; +} diff --git a/src/libs/protocols/src/lib.rs b/src/libs/protocols/src/lib.rs index 33f75ca0ea..9f2c244123 100644 --- a/src/libs/protocols/src/lib.rs +++ b/src/libs/protocols/src/lib.rs @@ -27,3 +27,9 @@ pub use serde_config::{ deserialize_enum_or_unknown, deserialize_message_field, serialize_enum_or_unknown, serialize_message_field, }; + +pub mod sealed_secret; +pub mod sealed_secret_ttrpc; + +#[cfg(feature = "async")] +pub mod sealed_secret_ttrpc_async; diff --git a/tests/integration/kubernetes/k8s-sealed-secret.bats b/tests/integration/kubernetes/k8s-sealed-secret.bats new file mode 100644 index 0000000000..749e53ec0d --- /dev/null +++ b/tests/integration/kubernetes/k8s-sealed-secret.bats @@ -0,0 +1,121 @@ +#!/usr/bin/env bats +# Copyright 2024 IBM Corporation +# Copyright 2024 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Test for Sealed Secret feature of CoCo +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/confidential_common.sh" +load "${BATS_TEST_DIRNAME}/confidential_kbs.sh" + +export KBS="${KBS:-false}" +export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +export AA_KBC="${AA_KBC:-cc_kbc}" + +setup() { + [ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}" + + if [ "${KBS}" = "false" ]; then + skip "Test skipped as KBS not setup" + fi + + setup_common + get_pod_config_dir + + export K8S_TEST_YAML="${pod_config_dir}/pod-sealed-secret.yaml" + # Schedule on a known node so that later it can print the system's logs for + # debugging. + set_node "$K8S_TEST_YAML" "$node" + + local CC_KBS_ADDR + export CC_KBS_ADDR=$(kbs_k8s_svc_http_addr) + kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params" + kernel_params_value="agent.guest_components_procs=confidential-data-hub" + + # For now we set aa_kbc_params via kernel cmdline + if [ "${AA_KBC}" = "cc_kbc" ]; then + kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${CC_KBS_ADDR}" + fi + set_metadata_annotation "${K8S_TEST_YAML}" \ + "${kernel_params_annotation}" \ + "${kernel_params_value}" + + # Setup k8s secret + kubectl delete secret sealed-secret --ignore-not-found + kubectl delete secret not-sealed-secret --ignore-not-found + + # Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault + # sealed.BASE64URL(UTF8(JWS Protected Header)) || '. + # || BASE64URL(JWS Payload) || '.' + # || BASE64URL(JWS Signature) + # test payload: + # { + # "version": "0.1.0", + # "type": "vault", + # "name": "kbs:///default/sealed-secret/test", + # "provider": "kbs", + # "provider_settings": {}, + # "annotations": {} + # } + kubectl create secret generic sealed-secret --from-literal='secret=sealed.fakejwsheader.ewogICAgInZlcnNpb24iOiAiMC4xLjAiLAogICAgInR5cGUiOiAidmF1bHQiLAogICAgIm5hbWUiOiAia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwKICAgICJwcm92aWRlciI6ICJrYnMiLAogICAgInByb3ZpZGVyX3NldHRpbmdzIjoge30sCiAgICAiYW5ub3RhdGlvbnMiOiB7fQp9Cg==.fakesignature' + + kubectl create secret generic not-sealed-secret --from-literal='secret=not_sealed_secret' + + if ! is_confidential_hardware; then + kbs_set_allow_all_resources + fi +} + +@test "Cannot Unseal Env Secrets with CDH without key" { + [ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}" + + if [ "${KBS}" = "false" ]; then + skip "Test skipped as KBS not setup" + fi + + k8s_create_pod "${K8S_TEST_YAML}" + + kubectl logs secret-test-pod-cc + kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret" + cmd="kubectl logs secret-test-pod-cc | grep -q \"PROTECTED_SECRET = unsealed_secret\"" + run $cmd + [ "$status" -eq 1 ] +} + + +@test "Unseal Env Secrets with CDH" { + [ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}" + + if [ "${KBS}" = "false" ]; then + skip "Test skipped as KBS not setup" + fi + + kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret" + k8s_create_pod "${K8S_TEST_YAML}" + + kubectl logs secret-test-pod-cc + kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret" + kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret" +} + +teardown() { + [ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}" + + if [ "${KBS}" = "false" ]; then + skip "Test skipped as KBS not setup" + fi + + [ -n "${pod_name:-}" ] && kubectl describe "pod/${pod_name}" || true + [ -n "${pod_config_dir:-}" ] && kubectl delete -f "${K8S_TEST_YAML}" || true + + kubectl delete secret sealed-secret --ignore-not-found + kubectl delete secret not-sealed-secret --ignore-not-found + + if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then + echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)" + print_node_journal "$node" "kata" --since "$node_start_time" || true + fi +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 9596098b41..112da943d1 100755 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -28,6 +28,7 @@ else "k8s-guest-pull-image.bats" \ "k8s-confidential-attestation.bats" \ "k8s-confidential.bats" \ + "k8s-sealed-secret.bats" \ "k8s-attach-handlers.bats" \ "k8s-caps.bats" \ "k8s-configmap.bats" \ diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-sealed-secret.yaml b/tests/integration/kubernetes/runtimeclass_workloads/pod-sealed-secret.yaml new file mode 100644 index 0000000000..cdb9ca27a0 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-sealed-secret.yaml @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Pod +metadata: + name: secret-test-pod-cc +spec: + runtimeClassName: kata + containers: + - name: busybox + image: quay.io/prometheus/busybox:latest + imagePullPolicy: Always + command: + - sh + - -c + - | + env + echo "PROTECTED_SECRET = $PROTECTED_SECRET" + echo "UNPROTECTED_SECRET = $UNPROTECTED_SECRET" + sleep 1000 + + # Expose secret data Containers through environment. + env: + - name: PROTECTED_SECRET + valueFrom: + secretKeyRef: + name: sealed-secret + key: secret + - name: UNPROTECTED_SECRET + valueFrom: + secretKeyRef: + name: not-sealed-secret + key: secret +