Merge pull request #9065 from wainersm/ci_install_kbs

CI: Install KBS on k8s for attestation tests

tests/gha-run-k8s-common.sh

@@ -27,11 +27,19 @@ function _print_instance_type() {
     esac
 }
 
+# Print the cluster name set by $AKS_NAME or generated out of runtime
+# metadata (e.g. pull request number, commit SHA, etc).
+#
 function _print_cluster_name() {
-    test_type="${1:-k8s}"
+    local test_type="${1:-k8s}"
+    local short_sha
 
-    short_sha="$(git rev-parse --short=12 HEAD)"
+    if [ -n "${AKS_NAME:-}" ]; then
-    echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}"
+        echo "$AKS_NAME"
+    else
+        short_sha="$(git rev-parse --short=12 HEAD)"
+        echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}"
+    fi
 }
 
 function _print_rg_name() {
@@ -40,6 +48,21 @@ function _print_rg_name() {
     echo "${AZ_RG:-"kataCI-$(_print_cluster_name ${test_type})"}"
 }
 
+# Enable the HTTP application routing add-on to AKS.
+# Use with ingress to expose a service API externally.
+#
+function enable_cluster_http_application_routing() {
+	local test_type="${1:-k8s}"
+	local cluster_name
+	local rg
+
+	rg="$(_print_rg_name "${test_type}")"
+	cluster_name="$(_print_cluster_name "${test_type}")"
+
+	az aks enable-addons -g "$rg" -n "$cluster_name" \
+		--addons http_application_routing
+}
+
 function install_azure_cli() {
     curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
     # The aks-preview extension is required while the Mariner Kata host is in preview.
@@ -94,6 +117,33 @@ function install_kubectl() {
     sudo az aks install-cli
 }
 
+# Install the kustomize tool in /usr/local/bin if it doesn't exist on
+# the system yet.
+#
+function install_kustomize() {
+	local arch
+	local checksum
+	local version
+
+	if command -v kustomize >/dev/null; then
+		return
+	fi
+
+	ensure_yq
+	version=$(get_from_kata_deps "externals.kustomize.version")
+	arch=$(arch_to_golang)
+	checksum=$(get_from_kata_deps "externals.kustomize.checksum.${arch}")
+
+	local tarball="kustomize_${version}_linux_${arch}.tar.gz"
+	curl -Lf -o "$tarball" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${version}/${tarball}"
+
+	local rc=0
+	echo "${checksum} $tarball" | sha256sum -c || rc=$?
+	[ $rc -eq 0 ] && sudo tar -xvzf "${tarball}" -C /usr/local/bin || rc=$?
+	rm -f "$tarball"
+	[ $rc -eq 0 ]
+}
+
 function get_cluster_credentials() {
     test_type="${1:-k8s}"
 
@@ -102,6 +152,24 @@ function get_cluster_credentials() {
         -n "$(_print_cluster_name ${test_type})"
 }
 
+
+# Get the AKS DNS zone name of HTTP application routing.
+#
+# Note: if the HTTP application routing add-on isn't installed in the cluster
+# then it will return an empty string.
+#
+function get_cluster_specific_dns_zone() {
+	local test_type="${1:-k8s}"
+	local cluster_name
+	local rg
+	local q="addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName"
+
+	rg="$(_print_rg_name "${test_type}")"
+	cluster_name="$(_print_cluster_name "${test_type}")"
+
+	az aks show -g "$rg" -n "$cluster_name" --query "$q" | tr -d \"
+}
+
 function delete_cluster() {
     test_type="${1:-k8s}"
     local rg

tests/integration/kubernetes/confidential_kbs.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Provides a library to deal with the CoCo KBS
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kubernetes_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck disable=1091
+source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
+
+# Where the kbs sources will be cloned
+readonly COCO_KBS_DIR="/tmp/kbs"
+# The k8s namespace where the kbs service is deployed
+readonly KBS_NS="coco-tenant"
+# The kbs service name
+readonly KBS_SVC_NAME="kbs"
+
+# Delete the kbs on Kubernetes
+#
+# Note: assume the kbs sources were cloned to $COCO_KBS_DIR
+#
+function kbs_k8s_delete() {
+	pushd "$COCO_KBS_DIR"
+	kubectl delete -k kbs/config/kubernetes/overlays
+	popd
+}
+
+# Deploy the kbs on Kubernetes
+#
+# Parameters:
+#	$1 - apply the specificed ingress handler to expose the service externally
+#
+function kbs_k8s_deploy() {
+	local image
+	local image_tag
+	local ingress=${1:-}
+	local repo
+	local svc_host
+	local timeout
+	local kbs_ip
+	local kbs_port
+	local version
+
+	# yq is needed by get_from_kata_deps
+	ensure_yq
+
+	# Read from versions.yaml
+	repo=$(get_from_kata_deps "externals.coco-kbs.url")
+	version=$(get_from_kata_deps "externals.coco-kbs.version")
+	image=$(get_from_kata_deps "externals.coco-kbs.image")
+	image_tag=$(get_from_kata_deps "externals.coco-kbs.image_tag")
+
+	# The ingress handler for AKS relies on the cluster's name which in turn
+	# contain the HEAD commit of the kata-containers repository (supposedly the
+	# current directory). It will be needed to save the cluster's name before
+	# it switches to the kbs repository and get a wrong HEAD commit.
+	if [ -z "${AKS_NAME:-}" ]; then
+		AKS_NAME=$(_print_cluster_name)
+		export AKS_NAME
+	fi
+
+	if [ -d "$COCO_KBS_DIR" ]; then
+		rm -rf "$COCO_KBS_DIR"
+	fi
+
+	echo "::group::Clone the kbs sources"
+	git clone --depth 1 "${repo}" "$COCO_KBS_DIR"
+	pushd "$COCO_KBS_DIR"
+	git fetch --depth=1 origin "${version}"
+	git checkout FETCH_HEAD -b kbs_$$
+	echo "::endgroup::"
+
+	pushd kbs/config/kubernetes/
+
+	# Tests should fill kbs resources later, however, the deployment
+	# expects at least one secret served at install time.
+	echo "somesecret" > overlays/key.bin
+
+	echo "::group::Update the kbs container image"
+	install_kustomize
+	pushd base
+	kustomize edit set image "kbs-container-image=${image}:${image_tag}"
+	popd
+	echo "::endgroup::"
+
+	[ -n "$ingress" ] && _handle_ingress "$ingress"
+
+	echo "::group::Deploy the KBS"
+	./deploy-kbs.sh
+	popd
+	popd
+
+	if ! waitForProcess "120" "10" "kubectl -n \"$KBS_NS\" get pods | \
+		grep -q '^kbs-.*Running.*'"; then
+		echo "ERROR: KBS service pod isn't running"
+		echo "::group::DEBUG - describe kbs deployments"
+		kubectl -n "$KBS_NS" get deployments || true
+		echo "::endgroup::"
+		echo "::group::DEBUG - describe kbs pod"
+		kubectl -n "$KBS_NS" describe pod -l app=kbs || true
+		echo "::endgroup::"
+		return 1
+	fi
+	echo "::endgroup::"
+
+	# By default, the KBS service is reachable within the cluster only,
+	# thus the following healthy checker should run from a pod. So start a
+	# debug pod where it will try to get a response from the service. The
+	# expected response is '404 Not Found' because it will request an endpoint
+	# that does not exist.
+	#
+	echo "::group::Check the service healthy"
+	kbs_ip=$(kubectl get -o jsonpath='{.spec.clusterIP}' svc "$KBS_SVC_NAME" -n "$KBS_NS" 2>/dev/null)
+	kbs_port=$(kubectl get -o jsonpath='{.spec.ports[0].port}' svc "$KBS_SVC_NAME" -n "$KBS_NS" 2>/dev/null)
+	local pod=kbs-checker-$$
+	kubectl run "$pod" --image=quay.io/prometheus/busybox --restart=Never -- \
+		sh -c "wget -O- --timeout=5 \"${kbs_ip}:${kbs_port}\" || true"
+	if ! waitForProcess "60" "10" "kubectl logs \"$pod\" 2>/dev/null | grep -q \"404 Not Found\""; then
+		echo "ERROR: KBS service is not responding to requests"
+		echo "::group::DEBUG - kbs logs"
+		kubectl -n "$KBS_NS" logs -l app=kbs || true
+		echo "::endgroup::"
+		kubectl delete pod "$pod"
+		return 1
+	fi
+	kubectl delete pod "$pod"
+	echo "KBS service respond to requests"
+	echo "::endgroup::"
+
+	if [ -n "$ingress" ]; then
+		echo "::group::Check the kbs service is exposed"
+		svc_host=$(kbs_k8s_svc_host)
+		if [ -z "$svc_host" ]; then
+			echo "ERROR: service host not found"
+			return 1
+		fi
+
+		# AZ DNS can take several minutes to update its records so that
+		# the host name will take a while to start resolving.
+		timeout=350
+		echo "Trying to connect at $svc_host. Timeout=$timeout"
+		if ! waitForProcess "$timeout" "30" "curl -s -I \"$svc_host\" | grep -q \"404 Not Found\""; then
+			echo "ERROR: service seems to not respond on $svc_host host"
+			curl -I "$svc_host"
+			return 1
+		fi
+		echo "KBS service respond to requests at $svc_host"
+		echo "::endgroup::"
+	fi
+}
+
+# Return the kbs service host name in case ingress is configured
+# otherwise the cluster IP.
+#
+kbs_k8s_svc_host() {
+	if kubectl get ingress -n "$KBS_NS" | grep -q kbs; then
+		kubectl get ingress kbs -n "$KBS_NS" \
+			-o jsonpath='{.spec.rules[0].host}' 2>/dev/null
+	else
+		kubectl get svc kbs -n "$KBS_NS" \
+			-o jsonpath='{.spec.clusterIP}' 2>/dev/null
+	fi
+}
+
+# Choose the appropriated ingress handler.
+#
+# To add a new handler, create a function named as _handle_ingress_NAME where
+# NAME is the handler name. This is enough for this method to pick up the right
+# implementation.
+#
+_handle_ingress() {
+	local ingress="$1"
+
+	type -a "_handle_ingress_$ingress" &>/dev/null || {
+		echo "ERROR: ingress '$ingress' handler not implemented";
+		return 1;
+	}
+
+	"_handle_ingress_$ingress"
+}
+
+# Implement the ingress handler for AKS.
+#
+_handle_ingress_aks() {
+	local dns_zone
+
+	dns_zone=$(get_cluster_specific_dns_zone "")
+
+	# In case the DNS zone name is empty, the cluster might not have the HTTP
+	# application routing add-on. Let's try to enable it.
+	if [ -z "$dns_zone" ]; then
+		echo "::group::Enable HTTP application routing add-on"
+		enable_cluster_http_application_routing ""
+		echo "::endgroup::"
+		dns_zone=$(get_cluster_specific_dns_zone "")
+	fi
+
+	if [ -z "$dns_zone" ]; then
+		echo "ERROR: the DNS zone name is nil, it cannot configure Ingress"
+		return 1
+	fi
+
+	pushd "$COCO_KBS_DIR/kbs/config/kubernetes/overlays"
+
+	echo "::group::$(pwd)/ingress.yaml"
+	KBS_INGRESS_CLASS="addon-http-application-routing" \
+		KBS_INGRESS_HOST="kbs.${dns_zone}" \
+		envsubst < ingress.yaml | tee ingress.yaml.tmp
+	echo "::endgroup::"
+	mv ingress.yaml.tmp ingress.yaml
+
+	kustomize edit add resource ingress.yaml
+	popd
+}

tests/integration/kubernetes/gha-run.sh

@@ -13,6 +13,8 @@ DEBUG="${DEBUG:-}"
 
 kubernetes_dir="$(dirname "$(readlink -f "$0")")"
 source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
+# shellcheck disable=1091
+source "${kubernetes_dir}/confidential_kbs.sh"
 # shellcheck disable=2154
 tools_dir="${repo_root_dir}/tools"
 kata_tarball_dir="${2:-kata-artifacts}"
@@ -105,8 +107,18 @@ function configure_snapshotter() {
 	echo "::endgroup::"
 }
 
+function delete_coco_kbs() {
+	kbs_k8s_delete
+}
+
+# Deploy the CoCo KBS in Kubernetes
+#
+# Environment variables:
+#	KBS_INGRESS - (optional) specify the ingress implementation to expose the
+#	              service externally
+#
 function deploy_coco_kbs() {
-	echo "TODO: deploy https://github.com/confidential-containers/kbs"
+	kbs_k8s_deploy "$KBS_INGRESS"
 }
 
 function deploy_kata() {
@@ -403,6 +415,7 @@ function main() {
 		cleanup-garm) cleanup "garm" ;;
 		cleanup-zvsi) cleanup "zvsi" ;;
 		cleanup-snapshotter) cleanup_snapshotter ;;
+		delete-coco-kbs) delete_coco_kbs ;;
 		delete-cluster) cleanup "aks" ;;
 		delete-cluster-kcli) delete_cluster_kcli ;;
 		*) >&2 echo "Invalid argument"; exit 2 ;;

versions.yaml

@@ -199,6 +199,13 @@ externals:
     version: "42b7c9687ecd0907ef70da31cf290a60ee8432cd"
     toolchain: "1.72.0"
 
+  coco-kbs:
+    description: "Provides attestation and secret Management services"
+    url: "https://github.com/confidential-containers/kbs"
+    version: "18c8ee378c6d83446ee635a702d5dee389028d8f"
+    image: "ghcr.io/confidential-containers/staged-images/kbs"
+    image_tag: "18c8ee378c6d83446ee635a702d5dee389028d8f"
+
   conmon:
     description: "An OCI container runtime monitor"
     url: "https://github.com/containers/conmon"
@@ -260,6 +267,17 @@ externals:
       .*/v?([\d\.]+)\.tar\.gz
     version: "1.23.1-00"
 
+  kustomize:
+    description: "Kubernetes native configuration management"
+    url: "https://github.com/kubernetes-sigs/kustomize"
+    version: "v5.3.0"
+    checksum:
+      amd64: "3ab32f92360d752a2a53e56be073b649abc1e7351b912c0fb32b960d1def854c"
+      arm64: "a1ec622d4adeb483e3cdabd70f0d66058b1e4bcec013c4f74f370666e1e045d8"
+      # yamllint disable-line rule:line-length
+      ppc64le: "946b1aa9325e7234157881fe2098e59c05c6834e56205bf6ec0a9a5fc83c9cc4"
+      s390x: "0b1a00f0e33efa2ecaa6cda9eeb63141ddccf97a912425974d6b65e66cf96cd4"
+
   libseccomp:
     description: "High level interface to Linux seccomp filter"
     url: "https://github.com/seccomp/libseccomp"