From de1e5bbde1ee0e8e3b5baa744da32c46c9213113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 5 May 2026 10:46:58 +0200 Subject: [PATCH] tests: deploy CoCo Trustee via Helm on all platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the kustomize-based Trustee deployment with the upstream Trustee Helm chart (deployment/helm-chart) for every platform, including TDX and s390x, which the previous iteration still left on kustomize. The chart deploys KBS, AS and RVPS as three separate images, so versions.yaml pins image_kbs/image_as/image_rvps (from the upstream staged-images registry) alongside the trustee revision, and bumps trustee to a build that carries the IBM SE material mounting support and the SHA-512 report-data change. guest-components is bumped in lockstep so IBM SE attestation agrees on the SHA-512 runtime-data digest. KBS runs with authorization_mode = "AuthenticatedAuthorization", so admin requests from kbs-client carry the bearer token minted by the chart's bootstrap job instead of relying on an anonymous, allow-all admin API. As reused CI runners may still carry a kbs-client from an older Trustee that predates the --admin-token-file flag, the client is rebuilt from the pinned sources unless the installed one already understands it. On the Intel TDX CI runner only the Attestation Service reaches the public Intel DCAP collateral endpoint, so it is the sole component that needs the corporate proxy. We set https_proxy on the AS alone: the KBS <-> AS <-> RVPS traffic is plain HTTP/gRPC and in-cluster, and routing it through the proxy breaks those connections. The Intel DCAP and NVIDIA verifiers are configured by the chart out of the box, so TDX and SNP need no extra verifier knobs. For IBM SE (s390x) the SE verifier now runs inside the gRPC Attestation Service, so the SE materials are mounted on the AS Pod (not KBS) through the chart's as.ibmse.* knobs, which create a node-local PV/PVC. The AS runs as a non-root user with CAP_DAC_OVERRIDE dropped, hence fsGroup=1000 and a group-readable hdr.bin so it can read the mounted materials. The chart's default CPU requests reserve more of the small single-node CI runners than the old single-Pod deployment did, starving test workloads that ask for a couple of Guaranteed CPUs. Trustee requests are shrunk (limits kept generous) so those Pods schedule again. Debug logging is enabled on all components (log_level=debug) to ease triaging attestation / resource-retrieval failures in CI, and the release is rolled out with 'helm upgrade --install' so re-runs on long-lived CI clusters reconcile in place. Signed-off-by: Fabiano Fidêncio Assisted-by: Cursor --- .../kubernetes/confidential_kbs.sh | 465 ++++++++++++------ tests/integration/kubernetes/gha-run.sh | 2 + .../k8s-confidential-attestation.bats | 17 +- tools/packaging/guest-image/lib_se.sh | 3 + versions.yaml | 12 +- 5 files changed, 336 insertions(+), 163 deletions(-) diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 9c64863238..a7bb4e7254 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -19,19 +19,36 @@ source "${kubernetes_dir}/../../../tools/packaging/guest-image/lib_se.sh" export PATH="${PATH}:/opt/kata/bin" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" -HTTPS_PROXY="${HTTPS_PROXY:-}" +# Only the Attestation Service reaches out to the public Intel DCAP collateral +# endpoint (api.trustedservices.intel.com) when verifying TDX quotes, so on the +# Intel TDX CI runner it needs the corporate HTTPS proxy. HTTP_PROXY is +# deliberately ignored: the KBS <-> AS <-> RVPS gRPC/HTTP traffic is in-cluster +# and routing it through the proxy breaks those connections. +HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}" # Where the trustee (includes kbs) sources will be cloned readonly COCO_TRUSTEE_DIR="/tmp/trustee" # Where the kbs sources will be cloned readonly COCO_KBS_DIR="${COCO_TRUSTEE_DIR}/kbs" +# The Helm chart directory inside the trustee repo +readonly COCO_HELM_CHART_DIR="${COCO_TRUSTEE_DIR}/deployment/helm-chart" +# The Helm release name +readonly HELM_RELEASE_NAME="trustee" +# The admin bearer-token file used for kbs-client authentication. The chart's +# bootstrap job mints this JWT (signed with the admin key) and stores it in the +# bootstrap secret; KBS runs with authorization_mode = "AuthenticatedAuthorization", +# so admin requests must carry it. +readonly KBS_ADMIN_TOKEN_FILE="${KBS_ADMIN_TOKEN_FILE:-/opt/trustee/install/kbs-admin-token}" +# The bootstrap secret name holding admin keys +readonly KBS_BOOTSTRAP_SECRET="trustee-bootstrap-user-keys" + # The k8s namespace where the kbs service is deployed -readonly KBS_NS="coco-tenant" -# The private key file used for CLI authentication -readonly KBS_PRIVATE_KEY="${KBS_PRIVATE_KEY:-/opt/trustee/install/kbs.key}" -# The kbs service name -readonly KBS_SVC_NAME="kbs" +readonly KBS_NS="coco-trustee" +# The kbs service name (Helm chart names it -kbs) +readonly KBS_SVC_NAME="trustee-kbs" +# The kbs service exposed via NodePort (Helm chart names it -nodeport) +readonly KBS_NODEPORT_SVC_NAME="trustee-kbs-nodeport" # The kbs ingress name -readonly KBS_INGRESS_NAME="kbs" +readonly KBS_INGRESS_NAME="trustee-kbs" # Workdir for installing snphost readonly SNPHOST_DIR="/tmp/snphost-workdir" @@ -110,19 +127,23 @@ kbs_set_resources_policy() { fi kbs-client --url "$(kbs_k8s_svc_http_addr)" config \ - --auth-private-key "${KBS_PRIVATE_KEY}" set-resource-policy \ + --admin-token-file "${KBS_ADMIN_TOKEN_FILE}" \ + set-resource-policy \ --policy-file "${file}" } -# Execute an admin command via the KBS client using the correct -# URI and admin authentication key. +# Execute an admin command via the KBS client. +# +# Admin requests are authenticated with the bearer token minted by the chart's +# bootstrap job (KBS runs with authorization_mode = "AuthenticatedAuthorization"). # # Parameters: # $1 - config command to run # kbs_config_command() { kbs-client --url "$(kbs_k8s_svc_http_addr)" config \ - --auth-private-key "${KBS_PRIVATE_KEY}" "$@" + --admin-token-file "${KBS_ADMIN_TOKEN_FILE}" \ + "$@" } # Set resource data in base64 encoded. @@ -215,14 +236,28 @@ kbs_set_resource_from_file() { path+="${tag}" kbs-client --url "$(kbs_k8s_svc_http_addr)" config \ - --auth-private-key "${KBS_PRIVATE_KEY}" set-resource \ + --admin-token-file "${KBS_ADMIN_TOKEN_FILE}" \ + set-resource \ --path "${path}" --resource-file "${file}" } -# Build and install the kbs-client binary, unless it is already present. +# Build and install the kbs-client binary, unless a compatible one is already +# present. +# +# Reused (self-hosted / baremetal) CI runners can carry a kbs-client left over +# from an older Trustee whose admin API predates the bearer-token flag this +# script relies on. Reusing it makes every 'config' command fail with +# "unexpected argument '--admin-token-file'". Only skip the (re)build when the +# installed client actually understands --admin-token-file; otherwise rebuild it +# from the pinned Trustee sources cloned by kbs_k8s_deploy(). # kbs_install_cli() { - command -v kbs-client >/dev/null && return + if command -v kbs-client >/dev/null; then + if kbs-client config --help 2>&1 | grep -q -- '--admin-token-file'; then + return + fi + echo "Existing kbs-client lacks --admin-token-file; rebuilding from ${COCO_KBS_DIR}" + fi source /etc/os-release || source /usr/lib/os-release case "${ID}" in @@ -315,46 +350,44 @@ ensure_snphost() { # Delete the kbs on Kubernetes # -# Note: assume the kbs sources were cloned to $COCO_TRUSTEE_DIR -# function kbs_k8s_delete() { - pushd "${COCO_KBS_DIR}" - if [[ "${KATA_HYPERVISOR}" = qemu-se* ]]; then - kubectl delete -k config/kubernetes/overlays/ibm-se - else - kubectl delete -k config/kubernetes/overlays/ - fi + helm uninstall "${HELM_RELEASE_NAME}" -n "${KBS_NS}" || true # Verify that KBS namespace resources were properly deleted - cmd="kubectl get all -n ${KBS_NS} 2>&1 | grep 'No resources found'" + local cmd="kubectl get all -n ${KBS_NS} 2>&1 | grep 'No resources found'" waitForProcess "120" "30" "${cmd}" - popd + + kubectl delete ns "${KBS_NS}" --ignore-not-found --wait=false } -# Deploy the kbs on Kubernetes +# Deploy the kbs on Kubernetes via the Trustee Helm chart. # # Parameters: -# $1 - apply the specificed ingress handler to expose the service externally +# $1 - ingress type to expose the service externally (nodeport|aks|"") +# +# Environment (optional): +# NVIDIA_VERIFIER_MODE - remote (default) | local: overrides the NVIDIA verifier +# type when KATA_HYPERVISOR matches *nvidia-gpu*. # function kbs_k8s_deploy() { - local image - local image_tag local ingress=${1:-} local repo + local version + local image_kbs + local image_as + local image_rvps local svc_host local timeout - local kbs_ip - local kbs_port - local version - # yq is needed by get_from_kata_deps ensure_yq + ensure_helm # Read from versions.yaml repo=$(get_from_kata_deps ".externals.coco-trustee.url") version=$(get_from_kata_deps ".externals.coco-trustee.version") - image=$(get_from_kata_deps ".externals.coco-trustee.image") - image_tag=$(get_from_kata_deps ".externals.coco-trustee.image_tag") + image_kbs=$(get_from_kata_deps ".externals.coco-trustee.image_kbs") + image_as=$(get_from_kata_deps ".externals.coco-trustee.image_as") + image_rvps=$(get_from_kata_deps ".externals.coco-trustee.image_rvps") # The ingress handler for AKS relies on the cluster's name which in turn # contain the HEAD commit of the kata-containers repository (supposedly the @@ -369,103 +402,265 @@ function kbs_k8s_deploy() { rm -rf "${COCO_TRUSTEE_DIR}" fi - echo "::group::Clone the kbs sources" + echo "::group::Clone the trustee sources" git clone --depth 1 "${repo}" "${COCO_TRUSTEE_DIR}" pushd "${COCO_TRUSTEE_DIR}" git fetch --depth=1 origin "${version}" - git checkout FETCH_HEAD -b kbs_$$ + git checkout FETCH_HEAD -b trustee_$$ popd echo "::endgroup::" - pushd "${COCO_KBS_DIR}/config/kubernetes/" + # Split image references into repository and tag. + # Format is "repo:tag" (e.g. "ghcr.io/confidential-containers/staged-images/kbs-grpc-as:") + local kbs_repo="${image_kbs%:*}" + local kbs_tag="${image_kbs##*:}" + local as_repo="${image_as%:*}" + local as_tag="${image_as##*:}" + local rvps_repo="${image_rvps%:*}" + local rvps_tag="${image_rvps##*:}" - # Tests should fill kbs resources later, however, the deployment - # expects at least one secret served at install time. - echo "somesecret" > overlays/key.bin + # Build Helm --set arguments for verifier configuration. + # These supplement the values file and avoid embedding YAML inside the heredoc. + # + # The AS auto-selects a verifier per evidence type; the chart configures the + # NVIDIA and Intel DCAP verifiers by default (DCAP already points at the + # public Intel collateral service), so SNP and TDX need no extra knobs. Only + # NVIDIA requires overriding the verifier type (Local vs Remote). + local helm_set_args=() - # For qemu-se* runtime, prepare the necessary resources - if [[ "${KATA_HYPERVISOR}" == qemu-se* ]]; then - mv overlays/key.bin overlays/ibm-se/key.bin - prepare_credentials_for_qemu_se - # SE_SKIP_CERTS_VERIFICATION should be set to true - # to skip the verification of the certificates - sed -i "s/false/true/g" overlays/ibm-se/patch.yaml + if [[ "${KATA_HYPERVISOR}" == *nvidia-gpu* ]]; then + local nvidia_verifier_type + nvidia_verifier_type="$(printf '%s' "${NVIDIA_VERIFIER_MODE:-remote}" | sed 's/./\u&/')" + helm_set_args+=(--set "as.verifier.nvidia.type=${nvidia_verifier_type}") fi - echo "::group::Update the kbs container image" - install_kustomize - pushd base - kustomize edit set image "kbs-container-image=${image}:${image_tag}" - popd + # Build Helm values override. + # + # The chart's default CPU/memory *requests* (kbs 250m, as 500m, rvps 100m) + # reserve ~850m of CPU across the three Trustee Pods. On the small + # single-node non-TEE CI runners that leaves too little allocatable CPU for + # the test workloads (some request 2 full CPUs with Guaranteed QoS), so they + # stay Pending with "Insufficient cpu". The builtin-AS kustomize deployment + # ran a single Pod and did not reserve this much. Shrink the requests (which + # is what the scheduler accounts for) to a minimum while leaving generous + # limits so the AS can still burst during TDX quote verification. + local values_file + values_file=$(mktemp -t trustee-helm-values-XXXXX.yaml) + cat > "${values_file}" <<-EOF + # Enable debug logging on all Trustee components (kbs, as, rvps) to help + # debug attestation / resource-retrieval failures in CI. + log_level: debug + kbs: + image: + repository: "${kbs_repo}" + tag: "${kbs_tag}" + service: + exposeLoadBalancer: false + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + as: + image: + repository: "${as_repo}" + tag: "${as_tag}" + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi + rvps: + image: + repository: "${rvps_repo}" + tag: "${rvps_tag}" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 512Mi + EOF + + # Handle ingress / nodeport + if [[ "${ingress}" = "nodeport" ]]; then + cat >> "${values_file}" <<-EOF + nodePort: + enabled: true + EOF + elif [[ "${ingress}" = "aks" ]]; then + echo "::group::Enable approuting (application routing) add-on" + enable_cluster_approuting "" + echo "::endgroup::" + + cat >> "${values_file}" <<-EOF + ingress: + enabled: true + className: "webapprouting.kubernetes.azure.com" + host: "" + EOF + fi + + # Proxy handling (Intel TDX CI): only the Attestation Service reaches out to + # the public Intel DCAP collateral endpoint (api.trustedservices.intel.com) + # when verifying TDX quotes, so it is the only component that needs the + # corporate proxy. We set https_proxy only: the KBS <-> AS <-> RVPS traffic is + # plain HTTP/gRPC and in-cluster, and routing it through the proxy (via + # HTTP_PROXY) breaks those connections. This mirrors the simple legacy setup, + # which set https_proxy on the (then builtin) AS and no no_proxy at all. + # Helm --set treats commas as key separators; escape them with \, for literal + # commas. + local as_env_idx=0 + if [[ -n "${HTTPS_PROXY}" ]]; then + local helm_https_proxy="${HTTPS_PROXY//,/\\,}" + helm_set_args+=(--set "as.extraEnvVars[${as_env_idx}].name=HTTPS_PROXY") + helm_set_args+=(--set "as.extraEnvVars[${as_env_idx}].value=${helm_https_proxy}") + as_env_idx=$((as_env_idx + 1)) + helm_set_args+=(--set "as.extraEnvVars[${as_env_idx}].name=https_proxy") + helm_set_args+=(--set "as.extraEnvVars[${as_env_idx}].value=${helm_https_proxy}") + as_env_idx=$((as_env_idx + 1)) + fi + + # Handle IBM SE (s390x): the SE verifier runs inside the gRPC Attestation + # Service, so the SE materials are mounted on the AS Pod (not KBS) via the + # chart's as.ibmse.* knobs, which create a node-local PV/PVC pointing at + # ${IBM_SE_CREDS_DIR} on the target node. + if [[ "${KATA_HYPERVISOR}" == qemu-se* ]]; then + local node_name + node_name=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') + prepare_credentials_for_qemu_se + helm_set_args+=(--set "as.ibmse.credsDir=${IBM_SE_CREDS_DIR:-}") + helm_set_args+=(--set "as.ibmse.nodeName=${node_name}") + # fsGroup must match the owning GID of the files under credsDir so the + # non-root AS container (CAP_DAC_OVERRIDE dropped) can read them. + helm_set_args+=(--set "as.podSecurityContext.fsGroup=1000") + # Development materials: skip the HKD certificate chain verification. + helm_set_args+=(--set "as.extraEnvVars[${as_env_idx}].name=SE_SKIP_CERTS_VERIFICATION") + helm_set_args+=(--set-string "as.extraEnvVars[${as_env_idx}].value=true") + as_env_idx=$((as_env_idx + 1)) + fi + + # Baremetal / self-hosted CI clusters keep the same Kubernetes API; a prior + # run may leave the Helm release secret behind (e.g. pending-install after a + # timeout), which would make a subsequent upgrade fail. Clean it up so the + # 'helm upgrade --install' below always starts from a known state. + if helm status "${HELM_RELEASE_NAME}" -n "${KBS_NS}" &>/dev/null; then + echo "Removing existing Helm release ${HELM_RELEASE_NAME} in namespace ${KBS_NS}" + helm uninstall "${HELM_RELEASE_NAME}" -n "${KBS_NS}" --wait --timeout 5m + fi + + # The chart declares a (condition-gated, disabled-by-default) postgresql + # dependency, which helm still requires to be present under charts/ before + # install. Fetch it from the versions pinned in the chart's Chart.lock. + echo "::group::Build Helm chart dependencies" + helm dependency build "${COCO_HELM_CHART_DIR}" echo "::endgroup::" - [[ -n "${ingress}" ]] && _handle_ingress "${ingress}" - echo "::group::Deploy the KBS" - ./deploy-kbs.sh + echo "::group::Deploy Trustee via Helm" + echo "Helm values override:" + cat "${values_file}" - # Set proxy env vars and enable debug logging on the KBS deployment. - # Using 'kubectl set env' avoids patching the trustee source tree. - # All vars are set in a single call to avoid triggering two rolling restarts. - local kbs_env_args=(RUST_LOG=debug) - is_tdx_hypervisor && [[ -n "${HTTPS_PROXY}" ]] && kbs_env_args+=(https_proxy="${HTTPS_PROXY}") - kubectl set env deployment/kbs -n "${KBS_NS}" "${kbs_env_args[@]}" - - # Check the private key used to install the KBS exist and save it in a - # well-known location. That's the access key used by the kbs-client. - local install_key="${PWD}/base/kbs.key" - if [[ ! -f "${install_key}" ]]; then - echo "ERROR: KBS private key not found at ${install_key}" + if ! helm upgrade --install "${HELM_RELEASE_NAME}" "${COCO_HELM_CHART_DIR}" \ + --namespace "${KBS_NS}" --create-namespace \ + -f "${values_file}" \ + "${helm_set_args[@]}" \ + --wait --timeout 5m --debug 2>&1; then + echo "ERROR: helm upgrade --install failed" + echo "::group::DEBUG - helm status" + helm status "${HELM_RELEASE_NAME}" -n "${KBS_NS}" 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - all resources" + kubectl -n "${KBS_NS}" get all 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - pods" + kubectl -n "${KBS_NS}" get pods -o wide 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - describe pods" + kubectl -n "${KBS_NS}" describe pods 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - services" + kubectl -n "${KBS_NS}" get svc -o wide 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - endpoints" + kubectl -n "${KBS_NS}" get endpoints 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - secrets" + kubectl -n "${KBS_NS}" get secrets 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - deployments" + kubectl -n "${KBS_NS}" get deployments -o wide 2>&1 || true + kubectl -n "${KBS_NS}" describe deployments 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - jobs" + kubectl -n "${KBS_NS}" get jobs 2>&1 || true + kubectl -n "${KBS_NS}" describe jobs 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - events" + kubectl -n "${KBS_NS}" get events --sort-by='.lastTimestamp' 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - kbs logs" + kubectl -n "${KBS_NS}" logs -l app=kbs --all-containers 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - as logs" + kubectl -n "${KBS_NS}" logs -l app=attestation-service --all-containers 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - rvps logs" + kubectl -n "${KBS_NS}" logs -l app=reference-value-provider-service 2>&1 || true + echo "::endgroup::" + echo "::group::DEBUG - node status" + kubectl get nodes -o wide 2>&1 || true + kubectl describe nodes 2>&1 || true + echo "::endgroup::" + rm -f "${values_file}" return 1 fi - sudo mkdir -p "$(dirname "${KBS_PRIVATE_KEY}")" - sudo cp -f "${install_key}" "${KBS_PRIVATE_KEY}" - popd + rm -f "${values_file}" - if ! waitForProcess "120" "10" "kubectl -n \"${KBS_NS}\" get pods | \ - grep -q '^kbs-.*Running.*'"; then - echo "ERROR: KBS service pod isn't running" - echo "::group::DEBUG - describe kbs deployments" - kubectl -n "${KBS_NS}" get deployments || true + # Extract the admin bearer token minted by the chart's bootstrap job so + # kbs-client can authenticate against the KBS admin API. + sudo mkdir -p "$(dirname "${KBS_ADMIN_TOKEN_FILE}")" + kubectl get secret "${KBS_BOOTSTRAP_SECRET}" -n "${KBS_NS}" \ + -o jsonpath='{.data.KBS_ADMIN_TOKEN}' | \ + base64 -d | sudo tee "${KBS_ADMIN_TOKEN_FILE}" > /dev/null + echo "::endgroup::" + + # Verify all three pods are running + echo "::group::Verify pods are running" + if ! waitForProcess "120" "10" \ + "kubectl -n ${KBS_NS} wait --for=condition=Ready pod -l app.kubernetes.io/instance=trustee --timeout=0 2>/dev/null"; then + echo "ERROR: Not all Trustee pods are running" + echo "::group::DEBUG - pods" + kubectl -n "${KBS_NS}" get pods || true echo "::endgroup::" - echo "::group::DEBUG - describe kbs pod" - kubectl -n "${KBS_NS}" describe pod -l app=kbs || true + echo "::group::DEBUG - describe pods" + kubectl -n "${KBS_NS}" describe pods || true echo "::endgroup::" echo "::group::DEBUG - kbs logs" kubectl -n "${KBS_NS}" logs -l app=kbs || true echo "::endgroup::" + echo "::group::DEBUG - as logs" + kubectl -n "${KBS_NS}" logs -l app=attestation-service || true + echo "::endgroup::" + echo "::group::DEBUG - rvps logs" + kubectl -n "${KBS_NS}" logs -l app=reference-value-provider-service || true + echo "::endgroup::" return 1 fi + echo "All Trustee pods are running" echo "::endgroup::" echo "::group::Post deploy actions" _post_deploy "${ingress}" echo "::endgroup::" - # By default, the KBS service is reachable within the cluster only, - # thus the following healthy checker should run from a pod. So start a - # debug pod where it will try to get a response from the service. The - # expected response is '404 Not Found' because it will request an endpoint - # that does not exist. - # - echo "::group::Check the service healthy" - kbs_ip=$(kubectl get -o jsonpath='{.spec.clusterIP}' svc "${KBS_SVC_NAME}" -n "${KBS_NS}" 2>/dev/null) - kbs_port=$(kubectl get -o jsonpath='{.spec.ports[0].port}' svc "${KBS_SVC_NAME}" -n "${KBS_NS}" 2>/dev/null) - - local pod=kbs-checker-$$ - kubectl run "${pod}" --image=quay.io/prometheus/busybox --restart=Never -- \ - sh -c "wget -O- --timeout=60 \"${kbs_ip}:${kbs_port}\" || true" - if ! waitForProcess "60" "10" "kubectl logs \"${pod}\" 2>/dev/null | grep -q \"404 Not Found\""; then - echo "ERROR: KBS service is not responding to requests" - echo "::group::DEBUG - kbs logs" - kubectl -n "${KBS_NS}" logs -l app=kbs || true - echo "::endgroup::" - kubectl delete pod "${pod}" - return 1 - fi - kubectl delete pod "${pod}" - echo "KBS service respond to requests" - echo "::endgroup::" + # The KBS readiness probe hits /healthz, so a Ready pod (verified + # above) already confirms the endpoint is working. if [[ -n "${ingress}" ]]; then echo "::group::Check the kbs service is exposed" @@ -479,9 +674,9 @@ function kbs_k8s_deploy() { # the host name will take a while to start resolving. timeout=350 echo "Trying to connect at ${svc_host}. Timeout=${timeout}" - if ! waitForProcess "${timeout}" "30" "curl -s -I \"${svc_host}\" | grep -q \"404 Not Found\""; then + if ! waitForProcess "${timeout}" "30" "curl -s \"${svc_host}/healthz\" -o /dev/null -w '%{http_code}' | grep -q '200'"; then echo "ERROR: service seems to not respond on ${svc_host} host" - curl -I "${svc_host}" + curl -I "${svc_host}/healthz" return 1 fi echo "KBS service respond to requests at ${svc_host}" @@ -504,10 +699,10 @@ kbs_k8s_svc_host() { sleep 5 done echo "${host}" - elif kubectl get svc "${KBS_SVC_NAME}" -n "${KBS_NS}" &>/dev/null; then - local host - host=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' -n "${KBS_NS}") - echo "${host}" + elif kubectl get svc "${KBS_NODEPORT_SVC_NAME}" -n "${KBS_NS}" &>/dev/null; then + local host + host=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') + echo "${host}" else kubectl get svc "${KBS_SVC_NAME}" -n "${KBS_NS}" \ -o jsonpath='{.spec.clusterIP}' 2>/dev/null @@ -521,8 +716,8 @@ kbs_k8s_svc_port() { if kubectl get ingress -n "${KBS_NS}" 2>/dev/null | grep -q kbs; then # Assume served on default HTTP port 80 echo "80" - elif kubectl get svc "${KBS_SVC_NAME}" -n "${KBS_NS}" &>/dev/null; then - kubectl get svc "${KBS_SVC_NAME}" -n "${KBS_NS}" -o jsonpath='{.spec.ports[0].nodePort}' + elif kubectl get svc "${KBS_NODEPORT_SVC_NAME}" -n "${KBS_NS}" &>/dev/null; then + kubectl get svc "${KBS_NODEPORT_SVC_NAME}" -n "${KBS_NS}" -o jsonpath='{.spec.ports[0].nodePort}' else kubectl get svc "${KBS_SVC_NAME}" -n "${KBS_NS}" \ -o jsonpath='{.spec.ports[0].port}' 2>/dev/null @@ -551,6 +746,12 @@ kbs_k8s_print_logs() { echo "::group::DEBUG - kbs logs since ${start_time}" kubectl -n "${KBS_NS}" logs -l app=kbs --since-time="${iso_start_time}" --timestamps=true || true echo "::endgroup::" + echo "::group::DEBUG - as logs since ${start_time}" + kubectl -n "${KBS_NS}" logs -l app=attestation-service --since-time="${iso_start_time}" --timestamps=true || true + echo "::endgroup::" + echo "::group::DEBUG - rvps logs since ${start_time}" + kubectl -n "${KBS_NS}" logs -l app=reference-value-provider-service --since-time="${iso_start_time}" --timestamps=true || true + echo "::endgroup::" } # Ensure rust is installed in the host. @@ -584,52 +785,6 @@ _ensure_rust() { fi } -# Choose the appropriated ingress handler. -# -# To add a new handler, create a function named as _handle_ingress_NAME where -# NAME is the handler name. This is enough for this method to pick up the right -# implementation. -# -_handle_ingress() { - local ingress="$1" - - type -a "_handle_ingress_${ingress}" &>/dev/null || { - echo "ERROR: ingress '${ingress}' handler not implemented"; - return 1; - } - - "_handle_ingress_${ingress}" -} - -# Implement the ingress handler for AKS. -# -_handle_ingress_aks() { - echo "::group::Enable approuting (application routing) add-on" - enable_cluster_approuting "" - echo "::endgroup::" - - pushd "${COCO_KBS_DIR}/config/kubernetes/overlays/" - - echo "::group::$(pwd)/ingress.yaml" - # We don't use a cluster DNS zone, instead get the ingress public IP, - # thus KBS_INGRESS_HOST is set empty. - KBS_INGRESS_CLASS="webapprouting.kubernetes.azure.com" \ - KBS_INGRESS_HOST="\"\"" \ - envsubst < ingress.yaml | tee ingress.yaml.tmp - echo "::endgroup::" - mv ingress.yaml.tmp ingress.yaml - - kustomize edit add resource ingress.yaml - popd -} - -# Implements the ingress handler for servernode -# -_handle_ingress_nodeport() { - # By exporting this variable the kbs deploy script will install the nodeport service - export DEPLOYMENT_DIR=nodeport -} - # Run further actions after the kbs was deployed, usually to apply further # configurations. # diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 528be36a30..f20969875e 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -169,6 +169,8 @@ function delete_coco_kbs() { # Environment variables: # KBS_INGRESS - (optional) specify the ingress implementation to expose the # service externally +# NVIDIA_VERIFIER_MODE - (optional) remote (default) | local: overrides the +# NVIDIA verifier type for nvidia-gpu hypervisors. # function deploy_coco_kbs() { kbs_k8s_deploy "${KBS_INGRESS}" diff --git a/tests/integration/kubernetes/k8s-confidential-attestation.bats b/tests/integration/kubernetes/k8s-confidential-attestation.bats index 86f5ff32af..34f8614a5f 100644 --- a/tests/integration/kubernetes/k8s-confidential-attestation.bats +++ b/tests/integration/kubernetes/k8s-confidential-attestation.bats @@ -80,16 +80,23 @@ setup() { # Check pod creation kubectl wait --for=condition=Ready --timeout="$timeout" pod "${pod_name}" - # Wait 5s for connecting with remote KBS - sleep 5 + # pod-attestable-gpu becomes Ready once curl is installed; CDH fetch and GPU + # attestation (e.g. remote NRAS) run afterward and routinely exceed a short + # fixed sleep. Poll logs until the resource appears (or timeout). + if is_confidential_gpu_hardware; then + waitForProcess "180" "5" "kubectl logs ${pod_name} 2>/dev/null | grep -q \"${test_key}\"" + waitForProcess "120" "5" "kubectl logs ${pod_name} 2>/dev/null | grep -iq 'Confidential Compute GPUs Ready state:[[:space:]]*ready'" + else + sleep 5 + fi - kubectl logs aa-test-cc - cmd="kubectl logs aa-test-cc | grep -q ${test_key}" + kubectl logs "${pod_name}" + cmd="kubectl logs ${pod_name} | grep -q ${test_key}" run bash -c "$cmd" [ "$status" -eq 0 ] if is_confidential_gpu_hardware; then - cmd="kubectl logs aa-test-cc | grep -iq 'Confidential Compute GPUs Ready state:[[:space:]]*ready'" + cmd="kubectl logs ${pod_name} | grep -iq 'Confidential Compute GPUs Ready state:[[:space:]]*ready'" run bash -c "$cmd" [ "$status" -eq 0 ] fi diff --git a/tools/packaging/guest-image/lib_se.sh b/tools/packaging/guest-image/lib_se.sh index e257ea6944..f8f9ba4d03 100755 --- a/tools/packaging/guest-image/lib_se.sh +++ b/tools/packaging/guest-image/lib_se.sh @@ -131,6 +131,9 @@ function repack_secure_image() { if [[ "${for_kbs}" == "true" ]]; then # Rename kata-containers-se.img to hdr.bin and clean up kernel and initrd mv "${build_dir}/hdr/kata-containers-se.img" "${build_dir}/hdr/hdr.bin" + # The Attestation Service reads hdr.bin as a non-root user via a group + # (fsGroup) mount, so make sure it is group/other readable. + chmod +r "${build_dir}/hdr/hdr.bin" rm -f "${build_dir}"/hdr/{vmlinuz.container,kata-containers-initrd-confidential.img} else # Clean up the build directory completely diff --git a/versions.yaml b/versions.yaml index 22af6490a7..07466d4675 100644 --- a/versions.yaml +++ b/versions.yaml @@ -292,17 +292,23 @@ externals: coco-guest-components: description: "Provides attested key unwrapping for image decryption" url: "https://github.com/confidential-containers/guest-components/" - version: "f1561038b9a58d309a3366cc8e25d8e6162424a0" + version: "740b92fffcb1bd164cdd238efd2913d1a296ec4b" toolchain: "1.90.0" coco-trustee: description: "Provides attestation and secret delivery components" url: "https://github.com/confidential-containers/trustee" - version: "d4e317620c4039c89779b725f74974d8f005da66" + version: "80a3c55e68e7d81fd31730ce9d49031f989e6c72" # image and image_tag must be in sync image: "ghcr.io/confidential-containers/staged-images/kbs" - image_tag: "d4e317620c4039c89779b725f74974d8f005da66" + image_tag: "80a3c55e68e7d81fd31730ce9d49031f989e6c72" toolchain: "1.90.0" + # Helm chart deploys three separate images (kbs, as, rvps) + # yamllint disable rule:line-length + image_kbs: "ghcr.io/confidential-containers/staged-images/kbs-grpc-as:80a3c55e68e7d81fd31730ce9d49031f989e6c72" + image_as: "ghcr.io/confidential-containers/staged-images/coco-as-grpc:80a3c55e68e7d81fd31730ce9d49031f989e6c72" + image_rvps: "ghcr.io/confidential-containers/staged-images/rvps:80a3c55e68e7d81fd31730ce9d49031f989e6c72" + # yamllint enable rule:line-length containerd: description: |