Merge pull request #9719 from fitzthum/sealed-secret

Support Confidential Sealed Secrets (as env vars)

src/agent/Cargo.lock

@@ -2222,6 +2222,7 @@ dependencies = [
  "cgroups-rs",
  "clap",
  "const_format",
+ "derivative",
  "futures",
  "image-rs",
  "ipnetwork",

src/agent/Cargo.toml

@@ -23,6 +23,7 @@ regex = "1.10.4"
 serial_test = "0.5.1"
 oci-distribution = "0.10.0"
 url = "2.5.0"
+derivative = "2.2.0"
 kata-sys-util = { path = "../libs/kata-sys-util" }
 kata-types = { path = "../libs/kata-types" }
 safe-path = { path = "../libs/safe-path" }

src/agent/src/cdh.rs

@@ -0,0 +1,150 @@
+// Copyright (c) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Confidential Data Hub client wrapper.
+// Confidential Data Hub is a service running inside guest to provide resource related APIs.
+// https://github.com/confidential-containers/guest-components/tree/main/confidential-data-hub
+
+use anyhow::Result;
+use derivative::Derivative;
+use protocols::{
+    sealed_secret, sealed_secret_ttrpc_async, sealed_secret_ttrpc_async::SealedSecretServiceClient,
+};
+
+use crate::CDH_SOCKET_URI;
+
+// Nanoseconds
+const CDH_UNSEAL_TIMEOUT: i64 = 50 * 1000 * 1000 * 1000;
+const SEALED_SECRET_PREFIX: &str = "sealed.";
+
+#[derive(Derivative)]
+#[derivative(Clone, Debug)]
+pub struct CDHClient {
+    #[derivative(Debug = "ignore")]
+    sealed_secret_client: SealedSecretServiceClient,
+}
+
+impl CDHClient {
+    pub fn new() -> Result<Self> {
+        let client = ttrpc::asynchronous::Client::connect(CDH_SOCKET_URI)?;
+        let sealed_secret_client =
+            sealed_secret_ttrpc_async::SealedSecretServiceClient::new(client);
+
+        Ok(CDHClient {
+            sealed_secret_client,
+        })
+    }
+
+    pub async fn unseal_secret_async(&self, sealed_secret: &str) -> Result<Vec<u8>> {
+        let mut input = sealed_secret::UnsealSecretInput::new();
+        input.set_secret(sealed_secret.into());
+
+        let unsealed_secret = self
+            .sealed_secret_client
+            .unseal_secret(ttrpc::context::with_timeout(CDH_UNSEAL_TIMEOUT), &input)
+            .await?;
+        Ok(unsealed_secret.plaintext)
+    }
+
+    pub async fn unseal_env(&self, env: &str) -> Result<String> {
+        if let Some((key, value)) = env.split_once('=') {
+            if value.starts_with(SEALED_SECRET_PREFIX) {
+                let unsealed_value = self.unseal_secret_async(value).await?;
+                let unsealed_env = format!("{}={}", key, std::str::from_utf8(&unsealed_value)?);
+
+                return Ok(unsealed_env);
+            }
+        }
+
+        Ok((*env.to_owned()).to_string())
+    }
+}
+
+#[cfg(test)]
+#[cfg(feature = "sealed-secret")]
+mod tests {
+    use crate::cdh::CDHClient;
+    use crate::cdh::CDH_ADDR;
+    use anyhow::anyhow;
+    use async_trait::async_trait;
+    use protocols::{sealed_secret, sealed_secret_ttrpc_async};
+    use std::sync::Arc;
+    use test_utils::skip_if_not_root;
+    use tokio::signal::unix::{signal, SignalKind};
+
+    struct TestService;
+
+    #[async_trait]
+    impl sealed_secret_ttrpc_async::SealedSecretService for TestService {
+        async fn unseal_secret(
+            &self,
+            _ctx: &::ttrpc::asynchronous::TtrpcContext,
+            _req: sealed_secret::UnsealSecretInput,
+        ) -> ttrpc::error::Result<sealed_secret::UnsealSecretOutput> {
+            let mut output = sealed_secret::UnsealSecretOutput::new();
+            output.set_plaintext("unsealed".into());
+            Ok(output)
+        }
+    }
+
+    fn remove_if_sock_exist(sock_addr: &str) -> std::io::Result<()> {
+        let path = sock_addr
+            .strip_prefix("unix://")
+            .expect("socket address does not have the expected format.");
+
+        if std::path::Path::new(path).exists() {
+            std::fs::remove_file(path)?;
+        }
+
+        Ok(())
+    }
+
+    fn start_ttrpc_server() {
+        tokio::spawn(async move {
+            let ss = Box::new(TestService {})
+                as Box<dyn sealed_secret_ttrpc_async::SealedSecretService + Send + Sync>;
+            let ss = Arc::new(ss);
+            let ss_service = sealed_secret_ttrpc_async::create_sealed_secret_service(ss);
+
+            remove_if_sock_exist(CDH_ADDR).unwrap();
+
+            let mut server = ttrpc::asynchronous::Server::new()
+                .bind(CDH_ADDR)
+                .unwrap()
+                .register_service(ss_service);
+
+            server.start().await.unwrap();
+
+            let mut interrupt = signal(SignalKind::interrupt()).unwrap();
+            tokio::select! {
+                _ = interrupt.recv() => {
+                    server.shutdown().await.unwrap();
+                }
+            };
+        });
+    }
+
+    #[tokio::test]
+    async fn test_unseal_env() {
+        skip_if_not_root!();
+
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let _guard = rt.enter();
+        start_ttrpc_server();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+
+        let cc = Some(CDHClient::new().unwrap());
+        let cdh_client = cc.as_ref().ok_or(anyhow!("get cdh_client failed")).unwrap();
+        let sealed_env = String::from("key=sealed.testdata");
+        let unsealed_env = cdh_client.unseal_env(&sealed_env).await.unwrap();
+        assert_eq!(unsealed_env, String::from("key=unsealed"));
+        let normal_env = String::from("key=testdata");
+        let unchanged_env = cdh_client.unseal_env(&normal_env).await.unwrap();
+        assert_eq!(unchanged_env, String::from("key=testdata"));
+
+        rt.shutdown_background();
+        std::thread::sleep(std::time::Duration::from_secs(2));
+    }
+}

src/agent/src/main.rs

@@ -38,6 +38,7 @@ use std::process::Command;
 use std::sync::Arc;
 use tracing::{instrument, span};
 
+mod cdh;
 mod config;
 mod console;
 mod device;
@@ -59,6 +60,7 @@ mod util;
 mod version;
 mod watcher;
 
+use cdh::CDHClient;
 use config::GuestComponentsProcs;
 use mount::{cgroups_mount, general_mount};
 use sandbox::Sandbox;
@@ -104,6 +106,7 @@ const AA_ATTESTATION_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, AA_ATTESTATION_SO
 
 const CDH_PATH: &str = "/usr/local/bin/confidential-data-hub";
 const CDH_SOCKET: &str = "/run/confidential-containers/cdh.sock";
+const CDH_SOCKET_URI: &str = concatcp!(UNIX_SOCKET_PREFIX, CDH_SOCKET);
 
 const API_SERVER_PATH: &str = "/usr/local/bin/api-server-rest";
 
@@ -403,6 +406,7 @@ async fn start_sandbox(
     let (tx, rx) = tokio::sync::oneshot::channel();
     sandbox.lock().await.sender = Some(tx);
 
+    let mut cdh_client = None;
     let gc_procs = config.guest_components_procs;
     if gc_procs != GuestComponentsProcs::None {
         if !attestation_binaries_available(logger, &gc_procs) {
@@ -411,12 +415,19 @@ async fn start_sandbox(
                 "attestation binaries requested for launch not available"
             );
         } else {
-            init_attestation_components(logger, config)?;
+            cdh_client = init_attestation_components(logger, config)?;
         }
     }
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode).await?;
+    let mut server = rpc::start(
+        sandbox.clone(),
+        config.server_addr.as_str(),
+        init_mode,
+        cdh_client,
+    )
+    .await?;
+
     server.start().await?;
 
     rx.await?;
@@ -445,10 +456,11 @@ fn attestation_binaries_available(logger: &Logger, procs: &GuestComponentsProcs)
 // Start-up attestation-agent, CDH and api-server-rest if they are packaged in the rootfs
 // and the corresponding procs are enabled in the agent configuration. the process will be
 // launched in the background and the function will return immediately.
-fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<()> {
+// If the CDH is started, a CDH client will be instantiated and returned.
+fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<Option<CDHClient>> {
     // skip launch of any guest-component
     if config.guest_components_procs == GuestComponentsProcs::None {
-        return Ok(());
+        return Ok(None);
     }
 
     debug!(logger, "spawning attestation-agent process {}", AA_PATH);
@@ -463,7 +475,7 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<
 
     // skip launch of confidential-data-hub and api-server-rest
     if config.guest_components_procs == GuestComponentsProcs::AttestationAgent {
-        return Ok(());
+        return Ok(None);
     }
 
     debug!(
@@ -479,9 +491,11 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", CDH_PATH, e))?;
 
+    let cdh_client = CDHClient::new().context("Failed to create CDH Client")?;
+
     // skip launch of api-server-rest
     if config.guest_components_procs == GuestComponentsProcs::ConfidentialDataHub {
-        return Ok(());
+        return Ok(Some(cdh_client));
     }
 
     let features = config.guest_components_rest_api;
@@ -498,7 +512,7 @@ fn init_attestation_components(logger: &Logger, config: &AgentConfig) -> Result<
     )
     .map_err(|e| anyhow!("launch_process {} failed: {:?}", API_SERVER_PATH, e))?;
 
-    Ok(())
+    Ok(Some(cdh_client))
 }
 
 fn wait_for_path_to_exist(logger: &Logger, path: &str, timeout_secs: i32) -> Result<()> {

src/agent/src/rpc.rs

@@ -76,6 +76,8 @@ use crate::policy::{do_set_policy, is_allowed};
 #[cfg(feature = "guest-pull")]
 use crate::image;
 
+use crate::cdh::CDHClient;
+
 use opentelemetry::global;
 use tracing::span;
 use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -171,6 +173,7 @@ impl<T> OptionToTtrpcResult<T> for Option<T> {
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
     init_mode: bool,
+    cdh_client: Option<CDHClient>,
 }
 
 impl AgentService {
@@ -217,6 +220,22 @@ impl AgentService {
         // cannot predict everything from the caller.
         add_devices(&req.devices, &mut oci, &self.sandbox).await?;
 
+        if let Some(cdh) = self.cdh_client.as_ref() {
+            let process = oci
+                .process
+                .as_mut()
+                .ok_or_else(|| anyhow!("Spec didn't contain process field"))?;
+
+            for env in process.env.iter_mut() {
+                match cdh.unseal_env(env).await {
+                    Ok(unsealed_env) => *env = unsealed_env.to_string(),
+                    Err(e) => {
+                        warn!(sl(), "Failed to unseal secret: {}", e)
+                    }
+                }
+            }
+        }
+
         // Both rootfs and volumes (invoked with --volume for instance) will
         // be processed the same way. The idea is to always mount any provided
         // storage to the specified MountPoint, so that it will match what's
@@ -1596,10 +1615,12 @@ pub async fn start(
     s: Arc<Mutex<Sandbox>>,
     server_address: &str,
     init_mode: bool,
+    cdh_client: Option<CDHClient>,
 ) -> Result<TtrpcServer> {
     let agent_service = Box::new(AgentService {
         sandbox: s,
         init_mode,
+        cdh_client,
     }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
     let aservice = agent_ttrpc::create_agent_service(Arc::new(agent_service));
 
@@ -2150,6 +2171,7 @@ mod tests {
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let req = protocols::agent::UpdateInterfaceRequest::default();
@@ -2164,10 +2186,10 @@ mod tests {
     async fn test_update_routes() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
-
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let req = protocols::agent::UpdateRoutesRequest::default();
@@ -2182,10 +2204,10 @@ mod tests {
     async fn test_add_arp_neighbors() {
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
-
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let req = protocols::agent::AddARPNeighborsRequest::default();
@@ -2324,6 +2346,7 @@ mod tests {
             let agent_service = Box::new(AgentService {
                 sandbox: Arc::new(Mutex::new(sandbox)),
                 init_mode: true,
+                cdh_client: None,
             });
 
             let result = agent_service
@@ -2813,6 +2836,7 @@ OtherField:other
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
             init_mode: true,
+            cdh_client: None,
         });
 
         let ctx = mk_ttrpc_context();

src/libs/protocols/build.rs

@@ -198,13 +198,34 @@ fn real_main() -> Result<(), std::io::Error> {
     // generate async
     #[cfg(feature = "async")]
     {
-        codegen("src", &["protos/agent.proto", "protos/health.proto"], true)?;
+
+        codegen(
+            "src",
+            &[
+                "protos/agent.proto",
+                "protos/health.proto",
+                "protos/sealed_secret.proto",
+            ],
+            true,
+        )?;
 
         fs::rename("src/agent_ttrpc.rs", "src/agent_ttrpc_async.rs")?;
         fs::rename("src/health_ttrpc.rs", "src/health_ttrpc_async.rs")?;
+        fs::rename(
+            "src/sealed_secret_ttrpc.rs",
+            "src/sealed_secret_ttrpc_async.rs",
+        )?;
     }
 
-    codegen("src", &["protos/agent.proto", "protos/health.proto"], false)?;
+    codegen(
+        "src",
+        &[
+            "protos/agent.proto",
+            "protos/health.proto",
+            "protos/sealed_secret.proto",
+        ],
+        false,
+    )?;
 
     // There is a message named 'Box' in oci.proto
     // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>

src/libs/protocols/protos/sealed_secret.proto

@@ -0,0 +1,21 @@
+//
+// Copyright (c) 2024 IBM
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package api;
+
+message UnsealSecretInput {
+    bytes secret = 1;
+}
+
+message UnsealSecretOutput {
+    bytes plaintext = 1;
+}
+
+service SealedSecretService {
+    rpc UnsealSecret(UnsealSecretInput) returns (UnsealSecretOutput) {};
+}

src/libs/protocols/src/lib.rs

@@ -27,3 +27,9 @@ pub use serde_config::{
     deserialize_enum_or_unknown, deserialize_message_field, serialize_enum_or_unknown,
     serialize_message_field,
 };
+
+pub mod sealed_secret;
+pub mod sealed_secret_ttrpc;
+
+#[cfg(feature = "async")]
+pub mod sealed_secret_ttrpc_async;

tests/integration/kubernetes/k8s-sealed-secret.bats

@@ -0,0 +1,121 @@
+#!/usr/bin/env bats
+# Copyright 2024 IBM Corporation
+# Copyright 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Test for Sealed Secret feature of CoCo
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/confidential_common.sh"
+load "${BATS_TEST_DIRNAME}/confidential_kbs.sh"
+
+export KBS="${KBS:-false}"
+export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
+export AA_KBC="${AA_KBC:-cc_kbc}"
+
+setup() {
+	[ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}"
+
+	if [ "${KBS}" = "false" ]; then
+		skip "Test skipped as KBS not setup"
+	fi
+
+	setup_common
+	get_pod_config_dir
+
+	export K8S_TEST_YAML="${pod_config_dir}/pod-sealed-secret.yaml"
+	# Schedule on a known node so that later it can print the system's logs for
+	# debugging.
+	set_node "$K8S_TEST_YAML" "$node"
+
+	local CC_KBS_ADDR
+	export CC_KBS_ADDR=$(kbs_k8s_svc_http_addr)
+	kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params"
+	kernel_params_value="agent.guest_components_procs=confidential-data-hub"
+
+	# For now we set aa_kbc_params via kernel cmdline
+	if [ "${AA_KBC}" = "cc_kbc" ]; then
+		kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${CC_KBS_ADDR}"
+	fi
+	set_metadata_annotation "${K8S_TEST_YAML}" \
+		"${kernel_params_annotation}" \
+		"${kernel_params_value}"
+
+	# Setup k8s secret
+	kubectl delete secret sealed-secret --ignore-not-found
+	kubectl delete secret not-sealed-secret --ignore-not-found
+
+	# Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault
+	# sealed.BASE64URL(UTF8(JWS Protected Header)) || '.
+	# || BASE64URL(JWS Payload) || '.'
+	# || BASE64URL(JWS Signature)
+	# test payload:
+	# {
+	# "version": "0.1.0",
+	# "type": "vault",
+	# "name": "kbs:///default/sealed-secret/test",
+	# "provider": "kbs",
+	# "provider_settings": {},
+	# "annotations": {}
+	# }
+	kubectl create secret generic sealed-secret --from-literal='secret=sealed.fakejwsheader.ewogICAgInZlcnNpb24iOiAiMC4xLjAiLAogICAgInR5cGUiOiAidmF1bHQiLAogICAgIm5hbWUiOiAia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwKICAgICJwcm92aWRlciI6ICJrYnMiLAogICAgInByb3ZpZGVyX3NldHRpbmdzIjoge30sCiAgICAiYW5ub3RhdGlvbnMiOiB7fQp9Cg==.fakesignature'
+
+	kubectl create secret generic not-sealed-secret --from-literal='secret=not_sealed_secret'
+
+	if ! is_confidential_hardware; then
+		kbs_set_allow_all_resources
+	fi
+}
+
+@test "Cannot Unseal Env Secrets with CDH without key" {
+	[ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}"
+
+	if [ "${KBS}" = "false" ]; then
+		skip "Test skipped as KBS not setup"
+	fi
+
+	k8s_create_pod "${K8S_TEST_YAML}"
+
+	kubectl logs secret-test-pod-cc
+	kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
+	cmd="kubectl logs secret-test-pod-cc | grep -q \"PROTECTED_SECRET = unsealed_secret\""
+	run $cmd
+	[ "$status" -eq 1 ]
+}
+
+
+@test "Unseal Env Secrets with CDH" {
+	[ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}"
+
+	if [ "${KBS}" = "false" ]; then
+		skip "Test skipped as KBS not setup"
+	fi
+
+	kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret"
+	k8s_create_pod "${K8S_TEST_YAML}"
+
+	kubectl logs secret-test-pod-cc
+	kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
+	kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret"
+}
+
+teardown() {
+	[ "${KATA_HYPERVISOR}" = "qemu-coco-dev" ] || skip "Test not ready yet for ${KATA_HYPERVISOR}"
+
+	if [ "${KBS}" = "false" ]; then
+		skip "Test skipped as KBS not setup"
+	fi
+
+	[ -n "${pod_name:-}" ] && kubectl describe "pod/${pod_name}" || true
+	[ -n "${pod_config_dir:-}" ] && kubectl delete -f "${K8S_TEST_YAML}" || true
+
+	kubectl delete secret sealed-secret --ignore-not-found
+	kubectl delete secret not-sealed-secret --ignore-not-found
+
+	if [[ -n "${node_start_time}:-}" && -z "$BATS_TEST_COMPLETED" ]]; then
+		echo "DEBUG: system logs of node '$node' since test start time ($node_start_time)"
+		print_node_journal "$node" "kata" --since "$node_start_time" || true
+	fi
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -28,6 +28,7 @@ else
 		"k8s-guest-pull-image.bats" \
 		"k8s-confidential-attestation.bats" \
 		"k8s-confidential.bats" \
+		"k8s-sealed-secret.bats" \
 		"k8s-attach-handlers.bats" \
 		"k8s-caps.bats" \
 		"k8s-configmap.bats" \

tests/integration/kubernetes/runtimeclass_workloads/pod-sealed-secret.yaml

@@ -0,0 +1,36 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: secret-test-pod-cc
+spec:
+  runtimeClassName: kata
+  containers:
+  - name: busybox
+    image: quay.io/prometheus/busybox:latest
+    imagePullPolicy: Always
+    command:
+    - sh
+    - -c
+    - |
+      env
+      echo "PROTECTED_SECRET = $PROTECTED_SECRET"
+      echo "UNPROTECTED_SECRET = $UNPROTECTED_SECRET"
+      sleep 1000
+
+    # Expose secret data Containers through environment.
+    env:
+    - name: PROTECTED_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: sealed-secret
+          key: secret
+    - name: UNPROTECTED_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: not-sealed-secret
+          key: secret
+