From 5bbc0abb81ccb9684c88449fbacc4b960da58223 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Thu, 19 Feb 2026 21:10:45 -0800 Subject: [PATCH] tests: use pre-created, signed sealed secrets With signature support for sealed secret, use pre-created signed sealed secrets and provision the signing public key to the KBS. Add instructions for re-creating these signed secrets. Improve k8s-sealed-secrets.bats by reducing repeated kubectl logs calls. A test run showed a SIGPIPE error one one of the grep-logs while the printouts of the initial kubectl logs invocation showed that the expected values were actually in the logs. Signed-off-by: Manuel Huber --- .../kubernetes/confidential_common.sh | 54 +++++++++++++++++++ .../kubernetes/k8s-nvidia-nim.bats | 40 ++------------ .../kubernetes/k8s-sealed-secret.bats | 42 ++++++--------- 3 files changed, 75 insertions(+), 61 deletions(-) diff --git a/tests/integration/kubernetes/confidential_common.sh b/tests/integration/kubernetes/confidential_common.sh index d6f5ec42b3..e487fadf5f 100644 --- a/tests/integration/kubernetes/confidential_common.sh +++ b/tests/integration/kubernetes/confidential_common.sh @@ -237,6 +237,60 @@ function create_coco_pod_yaml_with_annotations() { fi } +# Sealed secrets (signed JWS ES256). Pre-created with guest-components secret CLI; see +# https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md +# Tests provision the signing public key to KBS and use these pre-created sealed secret strings. +# +# To regenerate the signing key and sealed secrets: +# Install required dependencies, clone guest-components repository and change to guest-components/confidential-data-hub +# Create private and public JWK, for example: +# python3 -c " +# from jwcrypto import jwk +# k = jwk.JWK.generate(kty='EC', crv='P-256', alg='ES256', use='sig', kid='sealed-secret-test-key') +# with open('signing-key-private.jwk', 'w') as f: +# f.write(k.export_private()) +# with open('signing-key-public.jwk', 'w') as f: +# f.write(k.export_public()) +# print('Created signing-key-private.jwk and signing-key-public.jwk') +# " +# +# Build the secret CLI: +# cargo build -p confidential-data-hub --bin secret +# +# Create the sealed secret test secret: +# cargo run -p confidential-data-hub --bin secret -q -- seal \ +# --signing-kid "kbs:///default/signing-key/sealed-secret" \ +# --signing-jwk-path ./signing-key-private.jwk \ +# vault --resource-uri "kbs:///default/sealed-secret/test" --provider kbs +# +# Create the NIM test instruct secret: +# cargo run -p confidential-data-hub --bin secret -q -- seal \ +# --signing-kid "kbs:///default/signing-key/sealed-secret" \ +# --signing-jwk-path ./signing-key-private.jwk \ +# vault --resource-uri "kbs:///default/ngc-api-key/instruct" --provider kbs +# +# Create the NIM test embedqa secret: +# cargo run -p confidential-data-hub --bin secret -q -- seal \ +# --signing-kid "kbs:///default/signing-key/sealed-secret" \ +# --signing-jwk-path ./signing-key-private.jwk \ +# vault --resource-uri "kbs:///default/ngc-api-key/embedqa" --provider kbs +# +# Public JWK (no private key) used to verify the pre-created sealed secrets. Must match the key pair +# that was used to sign SEALED_SECRET_PRECREATED_*. +SEALED_SECRET_SIGNING_PUBLIC_JWK='{"alg":"ES256","crv":"P-256","kid":"sealed-secret-test-key","kty":"EC","use":"sig","x":"4jH376AuwTUCIx65AJ_56D7SZzWf7sGcEA7_Csq21UM","y":"rjdceysnSa5ZfzWOPGCURMUuHndxBAGUu4ISTIVN0yA"}' + +# Pre-created sealed secret for k8s-sealed-secret.bats (points to kbs:///default/sealed-secret/test) +export SEALED_SECRET_PRECREATED_TEST="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwicHJvdmlkZXIiOiJrYnMiLCJwcm92aWRlcl9zZXR0aW5ncyI6e30sImFubm90YXRpb25zIjp7fX0.ZI2fTv5ramHqHQa9DKBFD5hlJ_Mjf6cEIcpsNGshpyhEiKklML0abfH600TD7LAFHf53oDIJmEcVsDtJ20UafQ" + +# Pre-created sealed secrets for k8s-nvidia-nim.bats (point to kbs:///default/ngc-api-key/instruct and embedqa) +export SEALED_SECRET_PRECREATED_NIM_INSTRUCT="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvbmdjLWFwaS1rZXkvaW5zdHJ1Y3QiLCJwcm92aWRlciI6ImticyIsInByb3ZpZGVyX3NldHRpbmdzIjp7fSwiYW5ub3RhdGlvbnMiOnt9fQ.wpqvVFUaQymqgf54h70shZWDpk2NLW305wALz09YF0GKFBKBQiQB2sRwvn9Jk_rSju3YGLYxPO2Ub8qUbiMCuA" +export SEALED_SECRET_PRECREATED_NIM_EMBEDQA="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvbmdjLWFwaS1rZXkvZW1iZWRxYSIsInByb3ZpZGVyIjoia2JzIiwicHJvdmlkZXJfc2V0dGluZ3MiOnt9LCJhbm5vdGF0aW9ucyI6e319.4C1uqtVXi_qZT8vh_yZ4KpsRdgr2s4hU6ElKj18Hq1DJi_Iji61yuKsS6S1jWdb7drdoKKACvMD6RmCd85SJOQ" + +# Provision the signing public key to KBS so CDH can verify the pre-created sealed secrets. +function setup_sealed_secret_signing_public_key() { + kbs_set_resource "default" "signing-key" "sealed-secret" "${SEALED_SECRET_SIGNING_PUBLIC_JWK}" +} + function get_initdata_with_cdh_image_section() { CDH_IMAGE_SECTION=${1:-""} diff --git a/tests/integration/kubernetes/k8s-nvidia-nim.bats b/tests/integration/kubernetes/k8s-nvidia-nim.bats index 77db7859d3..46e7356dae 100644 --- a/tests/integration/kubernetes/k8s-nvidia-nim.bats +++ b/tests/integration/kubernetes/k8s-nvidia-nim.bats @@ -54,27 +54,8 @@ NGC_API_KEY_BASE64=$( ) export NGC_API_KEY_BASE64 -# Sealed secret format for TEE pods (vault type pointing to KBS resource) -# Format: sealed... -# IMPORTANT: JWS uses base64url encoding WITHOUT padding (no trailing '=') -# We use tr to convert standard base64 (+/) to base64url (-_) and remove padding (=) -# For vault type, header and signature can be placeholders since the payload -# contains the KBS resource path where the actual secret is stored. -# -# Vault type sealed secret payload for instruct pod: -# { -# "version": "0.1.0", -# "type": "vault", -# "name": "kbs:///default/ngc-api-key/instruct", -# "provider": "kbs", -# "provider_settings": {}, -# "annotations": {} -# } -NGC_API_KEY_SEALED_SECRET_INSTRUCT_PAYLOAD=$( - echo -n '{"version":"0.1.0","type":"vault","name":"kbs:///default/ngc-api-key/instruct","provider":"kbs","provider_settings":{},"annotations":{}}' | - base64 -w0 | tr '+/' '-_' | tr -d '=' -) -NGC_API_KEY_SEALED_SECRET_INSTRUCT="sealed.fakejwsheader.${NGC_API_KEY_SEALED_SECRET_INSTRUCT_PAYLOAD}.fakesignature" +# pre-created signed sealed secrets for TEE pods (from confidential_common.sh) +NGC_API_KEY_SEALED_SECRET_INSTRUCT="${SEALED_SECRET_PRECREATED_NIM_INSTRUCT}" export NGC_API_KEY_SEALED_SECRET_INSTRUCT # Base64 encode the sealed secret for use in Kubernetes Secret data field @@ -82,20 +63,7 @@ export NGC_API_KEY_SEALED_SECRET_INSTRUCT NGC_API_KEY_SEALED_SECRET_INSTRUCT_BASE64=$(echo -n "${NGC_API_KEY_SEALED_SECRET_INSTRUCT}" | base64 -w0) export NGC_API_KEY_SEALED_SECRET_INSTRUCT_BASE64 -# Vault type sealed secret payload for embedqa pod: -# { -# "version": "0.1.0", -# "type": "vault", -# "name": "kbs:///default/ngc-api-key/embedqa", -# "provider": "kbs", -# "provider_settings": {}, -# "annotations": {} -# } -NGC_API_KEY_SEALED_SECRET_EMBEDQA_PAYLOAD=$( - echo -n '{"version":"0.1.0","type":"vault","name":"kbs:///default/ngc-api-key/embedqa","provider":"kbs","provider_settings":{},"annotations":{}}' | - base64 -w0 | tr '+/' '-_' | tr -d '=' -) -NGC_API_KEY_SEALED_SECRET_EMBEDQA="sealed.fakejwsheader.${NGC_API_KEY_SEALED_SECRET_EMBEDQA_PAYLOAD}.fakesignature" +NGC_API_KEY_SEALED_SECRET_EMBEDQA="${SEALED_SECRET_PRECREATED_NIM_EMBEDQA}" export NGC_API_KEY_SEALED_SECRET_EMBEDQA NGC_API_KEY_SEALED_SECRET_EMBEDQA_BASE64=$(echo -n "${NGC_API_KEY_SEALED_SECRET_EMBEDQA}" | base64 -w0) @@ -223,6 +191,8 @@ setup_file() { if [ "${TEE}" = "true" ]; then setup_kbs_credentials + # provision signing public key to KBS so that CDH can verify pre-created, signed secret. + setup_sealed_secret_signing_public_key # Overwrite the empty default-initdata.toml with our CDH configuration. # This must happen AFTER create_tmp_policy_settings_dir() copies the empty # file and BEFORE auto_generate_policy() runs. diff --git a/tests/integration/kubernetes/k8s-sealed-secret.bats b/tests/integration/kubernetes/k8s-sealed-secret.bats index ae91fe84c1..c5165ee665 100644 --- a/tests/integration/kubernetes/k8s-sealed-secret.bats +++ b/tests/integration/kubernetes/k8s-sealed-secret.bats @@ -48,25 +48,13 @@ setup() { "${kernel_params_annotation}" \ "${kernel_params_value}" + # provision signing public key to KBS so that CDH can verify pre-created, signed secret. + setup_sealed_secret_signing_public_key + # Setup k8s secret kubectl delete secret sealed-secret --ignore-not-found kubectl delete secret not-sealed-secret --ignore-not-found - - # Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault - # sealed.BASE64URL(UTF8(JWS Protected Header)) || '. - # || BASE64URL(JWS Payload) || '.' - # || BASE64URL(JWS Signature) - # test payload: - # { - # "version": "0.1.0", - # "type": "vault", - # "name": "kbs:///default/sealed-secret/test", - # "provider": "kbs", - # "provider_settings": {}, - # "annotations": {} - # } - kubectl create secret generic sealed-secret --from-literal='secret=sealed.fakejwsheader.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwicHJvdmlkZXIiOiJrYnMiLCJwcm92aWRlcl9zZXR0aW5ncyI6e30sImFubm90YXRpb25zIjp7fX0.fakesignature' - + kubectl create secret generic sealed-secret --from-literal="secret=${SEALED_SECRET_PRECREATED_TEST}" kubectl create secret generic not-sealed-secret --from-literal='secret=not_sealed_secret' if ! is_confidential_hardware; then @@ -79,10 +67,10 @@ setup() { @test "Cannot Unseal Env Secrets with CDH without key" { k8s_create_pod "${K8S_TEST_ENV_YAML}" - kubectl logs secret-test-pod-cc - kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret" - cmd="kubectl logs secret-test-pod-cc | grep -q \"PROTECTED_SECRET = unsealed_secret\"" - run $cmd + logs=$(kubectl logs secret-test-pod-cc) + echo "$logs" + grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs" + run grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs" [ "$status" -eq 1 ] } @@ -91,18 +79,20 @@ setup() { kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret" k8s_create_pod "${K8S_TEST_ENV_YAML}" - kubectl logs secret-test-pod-cc - kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret" - kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret" + logs=$(kubectl logs secret-test-pod-cc) + echo "$logs" + grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs" + grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs" } @test "Unseal File Secrets with CDH" { kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret" k8s_create_pod "${K8S_TEST_FILE_YAML}" - kubectl logs secret-test-pod-cc - kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret" - kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret" + logs=$(kubectl logs secret-test-pod-cc) + echo "$logs" + grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs" + grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs" } teardown() {