mirror of
https://github.com/kata-containers/kata-containers.git
synced 2026-03-17 18:22:14 +00:00
tests: use pre-created, signed sealed secrets
With signature support for sealed secret, use pre-created signed sealed secrets and provision the signing public key to the KBS. Add instructions for re-creating these signed secrets. Improve k8s-sealed-secrets.bats by reducing repeated kubectl logs calls. A test run showed a SIGPIPE error one one of the grep-logs while the printouts of the initial kubectl logs invocation showed that the expected values were actually in the logs. Signed-off-by: Manuel Huber <manuelh@nvidia.com>
This commit is contained in:
Manuel Huber
committed by
manuelh-dev
manuelh-dev
parent
a9b222f91e
commit
5bbc0abb81
@@ -237,6 +237,60 @@ function create_coco_pod_yaml_with_annotations() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Sealed secrets (signed JWS ES256). Pre-created with guest-components secret CLI; see
|
||||
# https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md
|
||||
# Tests provision the signing public key to KBS and use these pre-created sealed secret strings.
|
||||
#
|
||||
# To regenerate the signing key and sealed secrets:
|
||||
# Install required dependencies, clone guest-components repository and change to guest-components/confidential-data-hub
|
||||
# Create private and public JWK, for example:
|
||||
# python3 -c "
|
||||
# from jwcrypto import jwk
|
||||
# k = jwk.JWK.generate(kty='EC', crv='P-256', alg='ES256', use='sig', kid='sealed-secret-test-key')
|
||||
# with open('signing-key-private.jwk', 'w') as f:
|
||||
# f.write(k.export_private())
|
||||
# with open('signing-key-public.jwk', 'w') as f:
|
||||
# f.write(k.export_public())
|
||||
# print('Created signing-key-private.jwk and signing-key-public.jwk')
|
||||
# "
|
||||
#
|
||||
# Build the secret CLI:
|
||||
# cargo build -p confidential-data-hub --bin secret
|
||||
#
|
||||
# Create the sealed secret test secret:
|
||||
# cargo run -p confidential-data-hub --bin secret -q -- seal \
|
||||
# --signing-kid "kbs:///default/signing-key/sealed-secret" \
|
||||
# --signing-jwk-path ./signing-key-private.jwk \
|
||||
# vault --resource-uri "kbs:///default/sealed-secret/test" --provider kbs
|
||||
#
|
||||
# Create the NIM test instruct secret:
|
||||
# cargo run -p confidential-data-hub --bin secret -q -- seal \
|
||||
# --signing-kid "kbs:///default/signing-key/sealed-secret" \
|
||||
# --signing-jwk-path ./signing-key-private.jwk \
|
||||
# vault --resource-uri "kbs:///default/ngc-api-key/instruct" --provider kbs
|
||||
#
|
||||
# Create the NIM test embedqa secret:
|
||||
# cargo run -p confidential-data-hub --bin secret -q -- seal \
|
||||
# --signing-kid "kbs:///default/signing-key/sealed-secret" \
|
||||
# --signing-jwk-path ./signing-key-private.jwk \
|
||||
# vault --resource-uri "kbs:///default/ngc-api-key/embedqa" --provider kbs
|
||||
#
|
||||
# Public JWK (no private key) used to verify the pre-created sealed secrets. Must match the key pair
|
||||
# that was used to sign SEALED_SECRET_PRECREATED_*.
|
||||
SEALED_SECRET_SIGNING_PUBLIC_JWK='{"alg":"ES256","crv":"P-256","kid":"sealed-secret-test-key","kty":"EC","use":"sig","x":"4jH376AuwTUCIx65AJ_56D7SZzWf7sGcEA7_Csq21UM","y":"rjdceysnSa5ZfzWOPGCURMUuHndxBAGUu4ISTIVN0yA"}'
|
||||
|
||||
# Pre-created sealed secret for k8s-sealed-secret.bats (points to kbs:///default/sealed-secret/test)
|
||||
export SEALED_SECRET_PRECREATED_TEST="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwicHJvdmlkZXIiOiJrYnMiLCJwcm92aWRlcl9zZXR0aW5ncyI6e30sImFubm90YXRpb25zIjp7fX0.ZI2fTv5ramHqHQa9DKBFD5hlJ_Mjf6cEIcpsNGshpyhEiKklML0abfH600TD7LAFHf53oDIJmEcVsDtJ20UafQ"
|
||||
|
||||
# Pre-created sealed secrets for k8s-nvidia-nim.bats (point to kbs:///default/ngc-api-key/instruct and embedqa)
|
||||
export SEALED_SECRET_PRECREATED_NIM_INSTRUCT="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvbmdjLWFwaS1rZXkvaW5zdHJ1Y3QiLCJwcm92aWRlciI6ImticyIsInByb3ZpZGVyX3NldHRpbmdzIjp7fSwiYW5ub3RhdGlvbnMiOnt9fQ.wpqvVFUaQymqgf54h70shZWDpk2NLW305wALz09YF0GKFBKBQiQB2sRwvn9Jk_rSju3YGLYxPO2Ub8qUbiMCuA"
|
||||
export SEALED_SECRET_PRECREATED_NIM_EMBEDQA="sealed.eyJiNjQiOnRydWUsImFsZyI6IkVTMjU2Iiwia2lkIjoia2JzOi8vL2RlZmF1bHQvc2lnbmluZy1rZXkvc2VhbGVkLXNlY3JldCJ9.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvbmdjLWFwaS1rZXkvZW1iZWRxYSIsInByb3ZpZGVyIjoia2JzIiwicHJvdmlkZXJfc2V0dGluZ3MiOnt9LCJhbm5vdGF0aW9ucyI6e319.4C1uqtVXi_qZT8vh_yZ4KpsRdgr2s4hU6ElKj18Hq1DJi_Iji61yuKsS6S1jWdb7drdoKKACvMD6RmCd85SJOQ"
|
||||
|
||||
# Provision the signing public key to KBS so CDH can verify the pre-created sealed secrets.
|
||||
function setup_sealed_secret_signing_public_key() {
|
||||
kbs_set_resource "default" "signing-key" "sealed-secret" "${SEALED_SECRET_SIGNING_PUBLIC_JWK}"
|
||||
}
|
||||
|
||||
function get_initdata_with_cdh_image_section() {
|
||||
CDH_IMAGE_SECTION=${1:-""}
|
||||
|
||||
|
||||
@@ -54,27 +54,8 @@ NGC_API_KEY_BASE64=$(
|
||||
)
|
||||
export NGC_API_KEY_BASE64
|
||||
|
||||
# Sealed secret format for TEE pods (vault type pointing to KBS resource)
|
||||
# Format: sealed.<base64url JWS header>.<base64url payload>.<base64url signature>
|
||||
# IMPORTANT: JWS uses base64url encoding WITHOUT padding (no trailing '=')
|
||||
# We use tr to convert standard base64 (+/) to base64url (-_) and remove padding (=)
|
||||
# For vault type, header and signature can be placeholders since the payload
|
||||
# contains the KBS resource path where the actual secret is stored.
|
||||
#
|
||||
# Vault type sealed secret payload for instruct pod:
|
||||
# {
|
||||
# "version": "0.1.0",
|
||||
# "type": "vault",
|
||||
# "name": "kbs:///default/ngc-api-key/instruct",
|
||||
# "provider": "kbs",
|
||||
# "provider_settings": {},
|
||||
# "annotations": {}
|
||||
# }
|
||||
NGC_API_KEY_SEALED_SECRET_INSTRUCT_PAYLOAD=$(
|
||||
echo -n '{"version":"0.1.0","type":"vault","name":"kbs:///default/ngc-api-key/instruct","provider":"kbs","provider_settings":{},"annotations":{}}' |
|
||||
base64 -w0 | tr '+/' '-_' | tr -d '='
|
||||
)
|
||||
NGC_API_KEY_SEALED_SECRET_INSTRUCT="sealed.fakejwsheader.${NGC_API_KEY_SEALED_SECRET_INSTRUCT_PAYLOAD}.fakesignature"
|
||||
# pre-created signed sealed secrets for TEE pods (from confidential_common.sh)
|
||||
NGC_API_KEY_SEALED_SECRET_INSTRUCT="${SEALED_SECRET_PRECREATED_NIM_INSTRUCT}"
|
||||
export NGC_API_KEY_SEALED_SECRET_INSTRUCT
|
||||
|
||||
# Base64 encode the sealed secret for use in Kubernetes Secret data field
|
||||
@@ -82,20 +63,7 @@ export NGC_API_KEY_SEALED_SECRET_INSTRUCT
|
||||
NGC_API_KEY_SEALED_SECRET_INSTRUCT_BASE64=$(echo -n "${NGC_API_KEY_SEALED_SECRET_INSTRUCT}" | base64 -w0)
|
||||
export NGC_API_KEY_SEALED_SECRET_INSTRUCT_BASE64
|
||||
|
||||
# Vault type sealed secret payload for embedqa pod:
|
||||
# {
|
||||
# "version": "0.1.0",
|
||||
# "type": "vault",
|
||||
# "name": "kbs:///default/ngc-api-key/embedqa",
|
||||
# "provider": "kbs",
|
||||
# "provider_settings": {},
|
||||
# "annotations": {}
|
||||
# }
|
||||
NGC_API_KEY_SEALED_SECRET_EMBEDQA_PAYLOAD=$(
|
||||
echo -n '{"version":"0.1.0","type":"vault","name":"kbs:///default/ngc-api-key/embedqa","provider":"kbs","provider_settings":{},"annotations":{}}' |
|
||||
base64 -w0 | tr '+/' '-_' | tr -d '='
|
||||
)
|
||||
NGC_API_KEY_SEALED_SECRET_EMBEDQA="sealed.fakejwsheader.${NGC_API_KEY_SEALED_SECRET_EMBEDQA_PAYLOAD}.fakesignature"
|
||||
NGC_API_KEY_SEALED_SECRET_EMBEDQA="${SEALED_SECRET_PRECREATED_NIM_EMBEDQA}"
|
||||
export NGC_API_KEY_SEALED_SECRET_EMBEDQA
|
||||
|
||||
NGC_API_KEY_SEALED_SECRET_EMBEDQA_BASE64=$(echo -n "${NGC_API_KEY_SEALED_SECRET_EMBEDQA}" | base64 -w0)
|
||||
@@ -223,6 +191,8 @@ setup_file() {
|
||||
|
||||
if [ "${TEE}" = "true" ]; then
|
||||
setup_kbs_credentials
|
||||
# provision signing public key to KBS so that CDH can verify pre-created, signed secret.
|
||||
setup_sealed_secret_signing_public_key
|
||||
# Overwrite the empty default-initdata.toml with our CDH configuration.
|
||||
# This must happen AFTER create_tmp_policy_settings_dir() copies the empty
|
||||
# file and BEFORE auto_generate_policy() runs.
|
||||
|
||||
@@ -48,25 +48,13 @@ setup() {
|
||||
"${kernel_params_annotation}" \
|
||||
"${kernel_params_value}"
|
||||
|
||||
# provision signing public key to KBS so that CDH can verify pre-created, signed secret.
|
||||
setup_sealed_secret_signing_public_key
|
||||
|
||||
# Setup k8s secret
|
||||
kubectl delete secret sealed-secret --ignore-not-found
|
||||
kubectl delete secret not-sealed-secret --ignore-not-found
|
||||
|
||||
# Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault
|
||||
# sealed.BASE64URL(UTF8(JWS Protected Header)) || '.
|
||||
# || BASE64URL(JWS Payload) || '.'
|
||||
# || BASE64URL(JWS Signature)
|
||||
# test payload:
|
||||
# {
|
||||
# "version": "0.1.0",
|
||||
# "type": "vault",
|
||||
# "name": "kbs:///default/sealed-secret/test",
|
||||
# "provider": "kbs",
|
||||
# "provider_settings": {},
|
||||
# "annotations": {}
|
||||
# }
|
||||
kubectl create secret generic sealed-secret --from-literal='secret=sealed.fakejwsheader.eyJ2ZXJzaW9uIjoiMC4xLjAiLCJ0eXBlIjoidmF1bHQiLCJuYW1lIjoia2JzOi8vL2RlZmF1bHQvc2VhbGVkLXNlY3JldC90ZXN0IiwicHJvdmlkZXIiOiJrYnMiLCJwcm92aWRlcl9zZXR0aW5ncyI6e30sImFubm90YXRpb25zIjp7fX0.fakesignature'
|
||||
|
||||
kubectl create secret generic sealed-secret --from-literal="secret=${SEALED_SECRET_PRECREATED_TEST}"
|
||||
kubectl create secret generic not-sealed-secret --from-literal='secret=not_sealed_secret'
|
||||
|
||||
if ! is_confidential_hardware; then
|
||||
@@ -79,10 +67,10 @@ setup() {
|
||||
@test "Cannot Unseal Env Secrets with CDH without key" {
|
||||
k8s_create_pod "${K8S_TEST_ENV_YAML}"
|
||||
|
||||
kubectl logs secret-test-pod-cc
|
||||
kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
|
||||
cmd="kubectl logs secret-test-pod-cc | grep -q \"PROTECTED_SECRET = unsealed_secret\""
|
||||
run $cmd
|
||||
logs=$(kubectl logs secret-test-pod-cc)
|
||||
echo "$logs"
|
||||
grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs"
|
||||
run grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs"
|
||||
[ "$status" -eq 1 ]
|
||||
}
|
||||
|
||||
@@ -91,18 +79,20 @@ setup() {
|
||||
kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret"
|
||||
k8s_create_pod "${K8S_TEST_ENV_YAML}"
|
||||
|
||||
kubectl logs secret-test-pod-cc
|
||||
kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
|
||||
kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret"
|
||||
logs=$(kubectl logs secret-test-pod-cc)
|
||||
echo "$logs"
|
||||
grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs"
|
||||
grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs"
|
||||
}
|
||||
|
||||
@test "Unseal File Secrets with CDH" {
|
||||
kbs_set_resource "default" "sealed-secret" "test" "unsealed_secret"
|
||||
k8s_create_pod "${K8S_TEST_FILE_YAML}"
|
||||
|
||||
kubectl logs secret-test-pod-cc
|
||||
kubectl logs secret-test-pod-cc | grep -q "UNPROTECTED_SECRET = not_sealed_secret"
|
||||
kubectl logs secret-test-pod-cc | grep -q "PROTECTED_SECRET = unsealed_secret"
|
||||
logs=$(kubectl logs secret-test-pod-cc)
|
||||
echo "$logs"
|
||||
grep -q "UNPROTECTED_SECRET = not_sealed_secret" <<< "$logs"
|
||||
grep -q "PROTECTED_SECRET = unsealed_secret" <<< "$logs"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
|
||||
Reference in New Issue
Block a user