diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 12128e902..d91899abf 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -27,11 +27,19 @@ function _print_instance_type() { esac } +# Print the cluster name set by $AKS_NAME or generated out of runtime +# metadata (e.g. pull request number, commit SHA, etc). +# function _print_cluster_name() { - test_type="${1:-k8s}" + local test_type="${1:-k8s}" + local short_sha - short_sha="$(git rev-parse --short=12 HEAD)" - echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}" + if [ -n "${AKS_NAME:-}" ]; then + echo "$AKS_NAME" + else + short_sha="$(git rev-parse --short=12 HEAD)" + echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}" + fi } function _print_rg_name() { @@ -40,6 +48,21 @@ function _print_rg_name() { echo "${AZ_RG:-"kataCI-$(_print_cluster_name ${test_type})"}" } +# Enable the HTTP application routing add-on to AKS. +# Use with ingress to expose a service API externally. +# +function enable_cluster_http_application_routing() { + local test_type="${1:-k8s}" + local cluster_name + local rg + + rg="$(_print_rg_name "${test_type}")" + cluster_name="$(_print_cluster_name "${test_type}")" + + az aks enable-addons -g "$rg" -n "$cluster_name" \ + --addons http_application_routing +} + function install_azure_cli() { curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash # The aks-preview extension is required while the Mariner Kata host is in preview. @@ -94,6 +117,33 @@ function install_kubectl() { sudo az aks install-cli } +# Install the kustomize tool in /usr/local/bin if it doesn't exist on +# the system yet. +# +function install_kustomize() { + local arch + local checksum + local version + + if command -v kustomize >/dev/null; then + return + fi + + ensure_yq + version=$(get_from_kata_deps "externals.kustomize.version") + arch=$(arch_to_golang) + checksum=$(get_from_kata_deps "externals.kustomize.checksum.${arch}") + + local tarball="kustomize_${version}_linux_${arch}.tar.gz" + curl -Lf -o "$tarball" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${version}/${tarball}" + + local rc=0 + echo "${checksum} $tarball" | sha256sum -c || rc=$? + [ $rc -eq 0 ] && sudo tar -xvzf "${tarball}" -C /usr/local/bin || rc=$? + rm -f "$tarball" + [ $rc -eq 0 ] +} + function get_cluster_credentials() { test_type="${1:-k8s}" @@ -102,6 +152,24 @@ function get_cluster_credentials() { -n "$(_print_cluster_name ${test_type})" } + +# Get the AKS DNS zone name of HTTP application routing. +# +# Note: if the HTTP application routing add-on isn't installed in the cluster +# then it will return an empty string. +# +function get_cluster_specific_dns_zone() { + local test_type="${1:-k8s}" + local cluster_name + local rg + local q="addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName" + + rg="$(_print_rg_name "${test_type}")" + cluster_name="$(_print_cluster_name "${test_type}")" + + az aks show -g "$rg" -n "$cluster_name" --query "$q" | tr -d \" +} + function delete_cluster() { test_type="${1:-k8s}" local rg diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh new file mode 100644 index 000000000..259a55827 --- /dev/null +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash + +# Copyright (c) 2024 Red Hat +# +# SPDX-License-Identifier: Apache-2.0 +# +# Provides a library to deal with the CoCo KBS +# + +set -o errexit +set -o nounset +set -o pipefail + +kubernetes_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=1091 +source "${kubernetes_dir}/../../gha-run-k8s-common.sh" + +# Where the kbs sources will be cloned +readonly COCO_KBS_DIR="/tmp/kbs" +# The k8s namespace where the kbs service is deployed +readonly KBS_NS="coco-tenant" +# The kbs service name +readonly KBS_SVC_NAME="kbs" + +# Delete the kbs on Kubernetes +# +# Note: assume the kbs sources were cloned to $COCO_KBS_DIR +# +function kbs_k8s_delete() { + pushd "$COCO_KBS_DIR" + kubectl delete -k kbs/config/kubernetes/overlays + popd +} + +# Deploy the kbs on Kubernetes +# +# Parameters: +# $1 - apply the specificed ingress handler to expose the service externally +# +function kbs_k8s_deploy() { + local image + local image_tag + local ingress=${1:-} + local repo + local svc_host + local timeout + local kbs_ip + local kbs_port + local version + + # yq is needed by get_from_kata_deps + ensure_yq + + # Read from versions.yaml + repo=$(get_from_kata_deps "externals.coco-kbs.url") + version=$(get_from_kata_deps "externals.coco-kbs.version") + image=$(get_from_kata_deps "externals.coco-kbs.image") + image_tag=$(get_from_kata_deps "externals.coco-kbs.image_tag") + + # The ingress handler for AKS relies on the cluster's name which in turn + # contain the HEAD commit of the kata-containers repository (supposedly the + # current directory). It will be needed to save the cluster's name before + # it switches to the kbs repository and get a wrong HEAD commit. + if [ -z "${AKS_NAME:-}" ]; then + AKS_NAME=$(_print_cluster_name) + export AKS_NAME + fi + + if [ -d "$COCO_KBS_DIR" ]; then + rm -rf "$COCO_KBS_DIR" + fi + + echo "::group::Clone the kbs sources" + git clone --depth 1 "${repo}" "$COCO_KBS_DIR" + pushd "$COCO_KBS_DIR" + git fetch --depth=1 origin "${version}" + git checkout FETCH_HEAD -b kbs_$$ + echo "::endgroup::" + + pushd kbs/config/kubernetes/ + + # Tests should fill kbs resources later, however, the deployment + # expects at least one secret served at install time. + echo "somesecret" > overlays/key.bin + + echo "::group::Update the kbs container image" + install_kustomize + pushd base + kustomize edit set image "kbs-container-image=${image}:${image_tag}" + popd + echo "::endgroup::" + + [ -n "$ingress" ] && _handle_ingress "$ingress" + + echo "::group::Deploy the KBS" + ./deploy-kbs.sh + popd + popd + + if ! waitForProcess "120" "10" "kubectl -n \"$KBS_NS\" get pods | \ + grep -q '^kbs-.*Running.*'"; then + echo "ERROR: KBS service pod isn't running" + echo "::group::DEBUG - describe kbs deployments" + kubectl -n "$KBS_NS" get deployments || true + echo "::endgroup::" + echo "::group::DEBUG - describe kbs pod" + kubectl -n "$KBS_NS" describe pod -l app=kbs || true + echo "::endgroup::" + return 1 + fi + echo "::endgroup::" + + # By default, the KBS service is reachable within the cluster only, + # thus the following healthy checker should run from a pod. So start a + # debug pod where it will try to get a response from the service. The + # expected response is '404 Not Found' because it will request an endpoint + # that does not exist. + # + echo "::group::Check the service healthy" + kbs_ip=$(kubectl get -o jsonpath='{.spec.clusterIP}' svc "$KBS_SVC_NAME" -n "$KBS_NS" 2>/dev/null) + kbs_port=$(kubectl get -o jsonpath='{.spec.ports[0].port}' svc "$KBS_SVC_NAME" -n "$KBS_NS" 2>/dev/null) + local pod=kbs-checker-$$ + kubectl run "$pod" --image=quay.io/prometheus/busybox --restart=Never -- \ + sh -c "wget -O- --timeout=5 \"${kbs_ip}:${kbs_port}\" || true" + if ! waitForProcess "60" "10" "kubectl logs \"$pod\" 2>/dev/null | grep -q \"404 Not Found\""; then + echo "ERROR: KBS service is not responding to requests" + echo "::group::DEBUG - kbs logs" + kubectl -n "$KBS_NS" logs -l app=kbs || true + echo "::endgroup::" + kubectl delete pod "$pod" + return 1 + fi + kubectl delete pod "$pod" + echo "KBS service respond to requests" + echo "::endgroup::" + + if [ -n "$ingress" ]; then + echo "::group::Check the kbs service is exposed" + svc_host=$(kbs_k8s_svc_host) + if [ -z "$svc_host" ]; then + echo "ERROR: service host not found" + return 1 + fi + + # AZ DNS can take several minutes to update its records so that + # the host name will take a while to start resolving. + timeout=350 + echo "Trying to connect at $svc_host. Timeout=$timeout" + if ! waitForProcess "$timeout" "30" "curl -s -I \"$svc_host\" | grep -q \"404 Not Found\""; then + echo "ERROR: service seems to not respond on $svc_host host" + curl -I "$svc_host" + return 1 + fi + echo "KBS service respond to requests at $svc_host" + echo "::endgroup::" + fi +} + +# Return the kbs service host name in case ingress is configured +# otherwise the cluster IP. +# +kbs_k8s_svc_host() { + if kubectl get ingress -n "$KBS_NS" | grep -q kbs; then + kubectl get ingress kbs -n "$KBS_NS" \ + -o jsonpath='{.spec.rules[0].host}' 2>/dev/null + else + kubectl get svc kbs -n "$KBS_NS" \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null + fi +} + +# Choose the appropriated ingress handler. +# +# To add a new handler, create a function named as _handle_ingress_NAME where +# NAME is the handler name. This is enough for this method to pick up the right +# implementation. +# +_handle_ingress() { + local ingress="$1" + + type -a "_handle_ingress_$ingress" &>/dev/null || { + echo "ERROR: ingress '$ingress' handler not implemented"; + return 1; + } + + "_handle_ingress_$ingress" +} + +# Implement the ingress handler for AKS. +# +_handle_ingress_aks() { + local dns_zone + + dns_zone=$(get_cluster_specific_dns_zone "") + + # In case the DNS zone name is empty, the cluster might not have the HTTP + # application routing add-on. Let's try to enable it. + if [ -z "$dns_zone" ]; then + echo "::group::Enable HTTP application routing add-on" + enable_cluster_http_application_routing "" + echo "::endgroup::" + dns_zone=$(get_cluster_specific_dns_zone "") + fi + + if [ -z "$dns_zone" ]; then + echo "ERROR: the DNS zone name is nil, it cannot configure Ingress" + return 1 + fi + + pushd "$COCO_KBS_DIR/kbs/config/kubernetes/overlays" + + echo "::group::$(pwd)/ingress.yaml" + KBS_INGRESS_CLASS="addon-http-application-routing" \ + KBS_INGRESS_HOST="kbs.${dns_zone}" \ + envsubst < ingress.yaml | tee ingress.yaml.tmp + echo "::endgroup::" + mv ingress.yaml.tmp ingress.yaml + + kustomize edit add resource ingress.yaml + popd +} \ No newline at end of file diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 91b218ffb..ac183a1f0 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -13,6 +13,8 @@ DEBUG="${DEBUG:-}" kubernetes_dir="$(dirname "$(readlink -f "$0")")" source "${kubernetes_dir}/../../gha-run-k8s-common.sh" +# shellcheck disable=1091 +source "${kubernetes_dir}/confidential_kbs.sh" # shellcheck disable=2154 tools_dir="${repo_root_dir}/tools" kata_tarball_dir="${2:-kata-artifacts}" @@ -105,8 +107,18 @@ function configure_snapshotter() { echo "::endgroup::" } +function delete_coco_kbs() { + kbs_k8s_delete +} + +# Deploy the CoCo KBS in Kubernetes +# +# Environment variables: +# KBS_INGRESS - (optional) specify the ingress implementation to expose the +# service externally +# function deploy_coco_kbs() { - echo "TODO: deploy https://github.com/confidential-containers/kbs" + kbs_k8s_deploy "$KBS_INGRESS" } function deploy_kata() { @@ -403,6 +415,7 @@ function main() { cleanup-garm) cleanup "garm" ;; cleanup-zvsi) cleanup "zvsi" ;; cleanup-snapshotter) cleanup_snapshotter ;; + delete-coco-kbs) delete_coco_kbs ;; delete-cluster) cleanup "aks" ;; delete-cluster-kcli) delete_cluster_kcli ;; *) >&2 echo "Invalid argument"; exit 2 ;; diff --git a/versions.yaml b/versions.yaml index 77a7afc1c..b6c32c298 100644 --- a/versions.yaml +++ b/versions.yaml @@ -199,6 +199,13 @@ externals: version: "42b7c9687ecd0907ef70da31cf290a60ee8432cd" toolchain: "1.72.0" + coco-kbs: + description: "Provides attestation and secret Management services" + url: "https://github.com/confidential-containers/kbs" + version: "18c8ee378c6d83446ee635a702d5dee389028d8f" + image: "ghcr.io/confidential-containers/staged-images/kbs" + image_tag: "18c8ee378c6d83446ee635a702d5dee389028d8f" + conmon: description: "An OCI container runtime monitor" url: "https://github.com/containers/conmon" @@ -260,6 +267,17 @@ externals: .*/v?([\d\.]+)\.tar\.gz version: "1.23.1-00" + kustomize: + description: "Kubernetes native configuration management" + url: "https://github.com/kubernetes-sigs/kustomize" + version: "v5.3.0" + checksum: + amd64: "3ab32f92360d752a2a53e56be073b649abc1e7351b912c0fb32b960d1def854c" + arm64: "a1ec622d4adeb483e3cdabd70f0d66058b1e4bcec013c4f74f370666e1e045d8" + # yamllint disable-line rule:line-length + ppc64le: "946b1aa9325e7234157881fe2098e59c05c6834e56205bf6ec0a9a5fc83c9cc4" + s390x: "0b1a00f0e33efa2ecaa6cda9eeb63141ddccf97a912425974d6b65e66cf96cd4" + libseccomp: description: "High level interface to Linux seccomp filter" url: "https://github.com/seccomp/libseccomp"