From 6a28c94d992b94655a52dcfff814c274987fcfba Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Thu, 8 Feb 2024 16:51:49 -0300
Subject: [PATCH 1/4] tests/k8s: add a kustomize installer

Kustomize has been used on some of our internal components (e.g.
kata-deploy) to manage k8s deployments. On CI it has been used
the `sed` tool to edit kustomization.yaml files, but `kustomize` is
more suitable for that purpose. So in order to use that tool on CI
scripts in the future, this commit introduces the `install_kustomize()`
function that is going to download and install the binary in
/usr/local/bin in case it's found on $PATH.

Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 tests/gha-run-k8s-common.sh | 27 +++++++++++++++++++++++++++
 versions.yaml               | 11 +++++++++++
 2 files changed, 38 insertions(+)

diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh
index 12128e902..2e3b23dda 100644
--- a/tests/gha-run-k8s-common.sh
+++ b/tests/gha-run-k8s-common.sh
@@ -94,6 +94,33 @@ function install_kubectl() {
     sudo az aks install-cli
 }
 
+# Install the kustomize tool in /usr/local/bin if it doesn't exist on
+# the system yet.
+#
+function install_kustomize() {
+	local arch
+	local checksum
+	local version
+
+	if command -v kustomize >/dev/null; then
+		return
+	fi
+
+	ensure_yq
+	version=$(get_from_kata_deps "externals.kustomize.version")
+	arch=$(arch_to_golang)
+	checksum=$(get_from_kata_deps "externals.kustomize.checksum.${arch}")
+
+	local tarball="kustomize_${version}_linux_${arch}.tar.gz"
+	curl -Lf -o "$tarball" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${version}/${tarball}"
+
+	local rc=0
+	echo "${checksum} $tarball" | sha256sum -c || rc=$?
+	[ $rc -eq 0 ] && sudo tar -xvzf "${tarball}" -C /usr/local/bin || rc=$?
+	rm -f "$tarball"
+	[ $rc -eq 0 ]
+}
+
 function get_cluster_credentials() {
     test_type="${1:-k8s}"
 
diff --git a/versions.yaml b/versions.yaml
index 77a7afc1c..82a229b10 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -260,6 +260,17 @@ externals:
       .*/v?([\d\.]+)\.tar\.gz
     version: "1.23.1-00"
 
+  kustomize:
+    description: "Kubernetes native configuration management"
+    url: "https://github.com/kubernetes-sigs/kustomize"
+    version: "v5.3.0"
+    checksum:
+      amd64: "3ab32f92360d752a2a53e56be073b649abc1e7351b912c0fb32b960d1def854c"
+      arm64: "a1ec622d4adeb483e3cdabd70f0d66058b1e4bcec013c4f74f370666e1e045d8"
+      # yamllint disable-line rule:line-length
+      ppc64le: "946b1aa9325e7234157881fe2098e59c05c6834e56205bf6ec0a9a5fc83c9cc4"
+      s390x: "0b1a00f0e33efa2ecaa6cda9eeb63141ddccf97a912425974d6b65e66cf96cd4"
+
   libseccomp:
     description: "High level interface to Linux seccomp filter"
     url: "https://github.com/seccomp/libseccomp"

From e1e0b949752e31fc20023035b1fd1eb5db877672 Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Wed, 7 Feb 2024 18:22:22 -0300
Subject: [PATCH 2/4] tests/k8s: introduce the CoCo kbs library

Introduce the tests/integration/kubernetes/confidential_kbs.sh library
that contains functions to manage the KBS on CI. Initially implemented
the kbs_k8s_deploy() and kbs_k8s_delete() functions to, respectively,
deploy and delete KBS on Kubernetes. Also hooked those functions in the
tests/integration/kubernetes/gha-run.sh script to follow the convention
of running commands from Github Workflows:

$ .tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
$ .tests/integration/kubernetes/gha-run.sh delete-coco-kbs

Fixes #9058
Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 .../kubernetes/confidential_kbs.sh            | 119 ++++++++++++++++++
 tests/integration/kubernetes/gha-run.sh       |   9 +-
 versions.yaml                                 |   7 ++
 3 files changed, 134 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/kubernetes/confidential_kbs.sh

diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh
new file mode 100644
index 000000000..5f024564b
--- /dev/null
+++ b/tests/integration/kubernetes/confidential_kbs.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Provides a library to deal with the CoCo KBS
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kubernetes_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck disable=1091
+source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
+
+# Where the kbs sources will be cloned
+readonly COCO_KBS_DIR="/tmp/kbs"
+# The k8s namespace where the kbs service is deployed
+readonly KBS_NS="coco-tenant"
+# The kbs service name
+readonly KBS_SVC_NAME="kbs"
+
+# Delete the kbs on Kubernetes
+#
+# Note: assume the kbs sources were cloned to $COCO_KBS_DIR
+#
+function kbs_k8s_delete() {
+	pushd "$COCO_KBS_DIR"
+	kubectl delete -k kbs/config/kubernetes/overlays
+	popd
+}
+
+# Deploy the kbs on Kubernetes
+#
+function kbs_k8s_deploy() {
+	local image
+	local image_tag
+	local repo
+	local kbs_ip
+	local kbs_port
+	local version
+
+	# yq is needed by get_from_kata_deps
+	ensure_yq
+
+	# Read from versions.yaml
+	repo=$(get_from_kata_deps "externals.coco-kbs.url")
+	version=$(get_from_kata_deps "externals.coco-kbs.version")
+	image=$(get_from_kata_deps "externals.coco-kbs.image")
+	image_tag=$(get_from_kata_deps "externals.coco-kbs.image_tag")
+
+	if [ -d "$COCO_KBS_DIR" ]; then
+		rm -rf "$COCO_KBS_DIR"
+	fi
+
+	echo "::group::Clone the kbs sources"
+	git clone --depth 1 "${repo}" "$COCO_KBS_DIR"
+	pushd "$COCO_KBS_DIR"
+	git fetch --depth=1 origin "${version}"
+	git checkout FETCH_HEAD -b kbs_$$
+	echo "::endgroup::"
+
+	pushd kbs/config/kubernetes/
+
+	# Tests should fill kbs resources later, however, the deployment
+	# expects at least one secret served at install time.
+	echo "somesecret" > overlays/key.bin
+
+	echo "::group::Update the kbs container image"
+	install_kustomize
+	pushd base
+	kustomize edit set image "kbs-container-image=${image}:${image_tag}"
+	popd
+	echo "::endgroup::"
+
+	echo "::group::Deploy the KBS"
+	./deploy-kbs.sh
+	popd
+	popd
+
+	if ! waitForProcess "120" "10" "kubectl -n \"$KBS_NS\" get pods | \
+		grep -q '^kbs-.*Running.*'"; then
+		echo "ERROR: KBS service pod isn't running"
+		echo "::group::DEBUG - describe kbs deployments"
+		kubectl -n "$KBS_NS" get deployments || true
+		echo "::endgroup::"
+		echo "::group::DEBUG - describe kbs pod"
+		kubectl -n "$KBS_NS" describe pod -l app=kbs || true
+		echo "::endgroup::"
+		return 1
+	fi
+	echo "::endgroup::"
+
+	# By default, the KBS service is reachable within the cluster only,
+	# thus the following healthy checker should run from a pod. So start a
+	# debug pod where it will try to get a response from the service. The
+	# expected response is '404 Not Found' because it will request an endpoint
+	# that does not exist.
+	#
+	echo "::group::Check the service healthy"
+	kbs_ip=$(kubectl get -o jsonpath='{.spec.clusterIP}' svc "$KBS_SVC_NAME" -n "$KBS_NS" 2>/dev/null)
+	kbs_port=$(kubectl get -o jsonpath='{.spec.ports[0].port}' svc "$KBS_SVC_NAME" -n "$KBS_NS" 2>/dev/null)
+	local pod=kbs-checker-$$
+	kubectl run "$pod" --image=quay.io/prometheus/busybox --restart=Never -- \
+		sh -c "wget -O- --timeout=5 \"${kbs_ip}:${kbs_port}\" || true"
+	if ! waitForProcess "60" "10" "kubectl logs \"$pod\" 2>/dev/null | grep -q \"404 Not Found\""; then
+		echo "ERROR: KBS service is not responding to requests"
+		echo "::group::DEBUG - kbs logs"
+		kubectl -n "$KBS_NS" logs -l app=kbs || true
+		echo "::endgroup::"
+		kubectl delete pod "$pod"
+		return 1
+	fi
+	kubectl delete pod "$pod"
+	echo "KBS service respond to requests"
+	echo "::endgroup::"
+}
\ No newline at end of file
diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index 8d5b69fc0..bf7f35e6e 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -13,6 +13,8 @@ DEBUG="${DEBUG:-}"
 
 kubernetes_dir="$(dirname "$(readlink -f "$0")")"
 source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
+# shellcheck disable=1091
+source "${kubernetes_dir}/confidential_kbs.sh"
 # shellcheck disable=2154
 tools_dir="${repo_root_dir}/tools"
 kata_tarball_dir="${2:-kata-artifacts}"
@@ -105,8 +107,12 @@ function configure_snapshotter() {
 	echo "::endgroup::"
 }
 
+function delete_coco_kbs() {
+	kbs_k8s_delete
+}
+
 function deploy_coco_kbs() {
-	echo "TODO: deploy https://github.com/confidential-containers/kbs"
+	kbs_k8s_deploy
 }
 
 function deploy_kata() {
@@ -389,6 +395,7 @@ function main() {
 		cleanup-garm) cleanup "garm" ;;
 		cleanup-zvsi) cleanup "zvsi" ;;
 		cleanup-snapshotter) cleanup_snapshotter ;;
+		delete-coco-kbs) delete_coco_kbs ;;
 		delete-cluster) cleanup "aks" ;;
 		delete-cluster-kcli) delete_cluster_kcli ;;
 		*) >&2 echo "Invalid argument"; exit 2 ;;
diff --git a/versions.yaml b/versions.yaml
index 82a229b10..b6c32c298 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -199,6 +199,13 @@ externals:
     version: "42b7c9687ecd0907ef70da31cf290a60ee8432cd"
     toolchain: "1.72.0"
 
+  coco-kbs:
+    description: "Provides attestation and secret Management services"
+    url: "https://github.com/confidential-containers/kbs"
+    version: "18c8ee378c6d83446ee635a702d5dee389028d8f"
+    image: "ghcr.io/confidential-containers/staged-images/kbs"
+    image_tag: "18c8ee378c6d83446ee635a702d5dee389028d8f"
+
   conmon:
     description: "An OCI container runtime monitor"
     url: "https://github.com/containers/conmon"

From 5e4b7bbd0463f2a68d193ee88d2d418a33791e7c Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Mon, 12 Feb 2024 18:30:21 -0300
Subject: [PATCH 3/4] tests/k8s: expose KBS service externally

Until this point the deployed KBS service is only reachable from within
the cluster. This introduces a generic mechanism to apply an Ingress
configuration to expose the service externally.

The first implemened ingress is for AKS. In case the HTTP application
routing isn't enabled in the cluster (this is required for ingress), an
add-on is applied.

It was added the get_cluster_specific_dns_zone() and
enable_cluster_http_application_routing() helper functions
to gha-run-k8s-common.sh.

Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 tests/gha-run-k8s-common.sh                   |  33 ++++++
 .../kubernetes/confidential_kbs.sh            | 102 ++++++++++++++++++
 tests/integration/kubernetes/gha-run.sh       |   8 +-
 3 files changed, 142 insertions(+), 1 deletion(-)

diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh
index 2e3b23dda..391970e90 100644
--- a/tests/gha-run-k8s-common.sh
+++ b/tests/gha-run-k8s-common.sh
@@ -40,6 +40,21 @@ function _print_rg_name() {
     echo "${AZ_RG:-"kataCI-$(_print_cluster_name ${test_type})"}"
 }
 
+# Enable the HTTP application routing add-on to AKS.
+# Use with ingress to expose a service API externally.
+#
+function enable_cluster_http_application_routing() {
+	local test_type="${1:-k8s}"
+	local cluster_name
+	local rg
+
+	rg="$(_print_rg_name "${test_type}")"
+	cluster_name="$(_print_cluster_name "${test_type}")"
+
+	az aks enable-addons -g "$rg" -n "$cluster_name" \
+		--addons http_application_routing
+}
+
 function install_azure_cli() {
     curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
     # The aks-preview extension is required while the Mariner Kata host is in preview.
@@ -129,6 +144,24 @@ function get_cluster_credentials() {
         -n "$(_print_cluster_name ${test_type})"
 }
 
+
+# Get the AKS DNS zone name of HTTP application routing.
+#
+# Note: if the HTTP application routing add-on isn't installed in the cluster
+# then it will return an empty string.
+#
+function get_cluster_specific_dns_zone() {
+	local test_type="${1:-k8s}"
+	local cluster_name
+	local rg
+	local q="addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName"
+
+	rg="$(_print_rg_name "${test_type}")"
+	cluster_name="$(_print_cluster_name "${test_type}")"
+
+	az aks show -g "$rg" -n "$cluster_name" --query "$q" | tr -d \"
+}
+
 function delete_cluster() {
     test_type="${1:-k8s}"
     local rg
diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh
index 5f024564b..259a55827 100644
--- a/tests/integration/kubernetes/confidential_kbs.sh
+++ b/tests/integration/kubernetes/confidential_kbs.sh
@@ -34,10 +34,16 @@ function kbs_k8s_delete() {
 
 # Deploy the kbs on Kubernetes
 #
+# Parameters:
+#	$1 - apply the specificed ingress handler to expose the service externally
+#
 function kbs_k8s_deploy() {
 	local image
 	local image_tag
+	local ingress=${1:-}
 	local repo
+	local svc_host
+	local timeout
 	local kbs_ip
 	local kbs_port
 	local version
@@ -51,6 +57,15 @@ function kbs_k8s_deploy() {
 	image=$(get_from_kata_deps "externals.coco-kbs.image")
 	image_tag=$(get_from_kata_deps "externals.coco-kbs.image_tag")
 
+	# The ingress handler for AKS relies on the cluster's name which in turn
+	# contain the HEAD commit of the kata-containers repository (supposedly the
+	# current directory). It will be needed to save the cluster's name before
+	# it switches to the kbs repository and get a wrong HEAD commit.
+	if [ -z "${AKS_NAME:-}" ]; then
+		AKS_NAME=$(_print_cluster_name)
+		export AKS_NAME
+	fi
+
 	if [ -d "$COCO_KBS_DIR" ]; then
 		rm -rf "$COCO_KBS_DIR"
 	fi
@@ -75,6 +90,8 @@ function kbs_k8s_deploy() {
 	popd
 	echo "::endgroup::"
 
+	[ -n "$ingress" ] && _handle_ingress "$ingress"
+
 	echo "::group::Deploy the KBS"
 	./deploy-kbs.sh
 	popd
@@ -116,4 +133,89 @@ function kbs_k8s_deploy() {
 	kubectl delete pod "$pod"
 	echo "KBS service respond to requests"
 	echo "::endgroup::"
+
+	if [ -n "$ingress" ]; then
+		echo "::group::Check the kbs service is exposed"
+		svc_host=$(kbs_k8s_svc_host)
+		if [ -z "$svc_host" ]; then
+			echo "ERROR: service host not found"
+			return 1
+		fi
+
+		# AZ DNS can take several minutes to update its records so that
+		# the host name will take a while to start resolving.
+		timeout=350
+		echo "Trying to connect at $svc_host. Timeout=$timeout"
+		if ! waitForProcess "$timeout" "30" "curl -s -I \"$svc_host\" | grep -q \"404 Not Found\""; then
+			echo "ERROR: service seems to not respond on $svc_host host"
+			curl -I "$svc_host"
+			return 1
+		fi
+		echo "KBS service respond to requests at $svc_host"
+		echo "::endgroup::"
+	fi
+}
+
+# Return the kbs service host name in case ingress is configured
+# otherwise the cluster IP.
+#
+kbs_k8s_svc_host() {
+	if kubectl get ingress -n "$KBS_NS" | grep -q kbs; then
+		kubectl get ingress kbs -n "$KBS_NS" \
+			-o jsonpath='{.spec.rules[0].host}' 2>/dev/null
+	else
+		kubectl get svc kbs -n "$KBS_NS" \
+			-o jsonpath='{.spec.clusterIP}' 2>/dev/null
+	fi
+}
+
+# Choose the appropriated ingress handler.
+#
+# To add a new handler, create a function named as _handle_ingress_NAME where
+# NAME is the handler name. This is enough for this method to pick up the right
+# implementation.
+#
+_handle_ingress() {
+	local ingress="$1"
+
+	type -a "_handle_ingress_$ingress" &>/dev/null || {
+		echo "ERROR: ingress '$ingress' handler not implemented";
+		return 1;
+	}
+
+	"_handle_ingress_$ingress"
+}
+
+# Implement the ingress handler for AKS.
+#
+_handle_ingress_aks() {
+	local dns_zone
+
+	dns_zone=$(get_cluster_specific_dns_zone "")
+
+	# In case the DNS zone name is empty, the cluster might not have the HTTP
+	# application routing add-on. Let's try to enable it.
+	if [ -z "$dns_zone" ]; then
+		echo "::group::Enable HTTP application routing add-on"
+		enable_cluster_http_application_routing ""
+		echo "::endgroup::"
+		dns_zone=$(get_cluster_specific_dns_zone "")
+	fi
+
+	if [ -z "$dns_zone" ]; then
+		echo "ERROR: the DNS zone name is nil, it cannot configure Ingress"
+		return 1
+	fi
+
+	pushd "$COCO_KBS_DIR/kbs/config/kubernetes/overlays"
+
+	echo "::group::$(pwd)/ingress.yaml"
+	KBS_INGRESS_CLASS="addon-http-application-routing" \
+		KBS_INGRESS_HOST="kbs.${dns_zone}" \
+		envsubst < ingress.yaml | tee ingress.yaml.tmp
+	echo "::endgroup::"
+	mv ingress.yaml.tmp ingress.yaml
+
+	kustomize edit add resource ingress.yaml
+	popd
 }
\ No newline at end of file
diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index bf7f35e6e..0f17a1acb 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -111,8 +111,14 @@ function delete_coco_kbs() {
 	kbs_k8s_delete
 }
 
+# Deploy the CoCo KBS in Kubernetes
+#
+# Environment variables:
+#	KBS_INGRESS - (optional) specify the ingress implementation to expose the
+#	              service externally
+#
 function deploy_coco_kbs() {
-	kbs_k8s_deploy
+	kbs_k8s_deploy "$KBS_INGRESS"
 }
 
 function deploy_kata() {

From 2c24977cb1a42198ed0a818fc7cd3b03af826c78 Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Thu, 15 Feb 2024 16:00:22 -0300
Subject: [PATCH 4/4] tests/k8s: allow to overwrite the cluster name

_print_cluster_name() create a string based information like the
pull request number and commit SHA. However, when you are developing the
scripts you might want to use an arbitrary name, so it was introduced
the $AKS_NAME variable that once exported it will overwrite the
generated name.

Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 tests/gha-run-k8s-common.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh
index 391970e90..d91899abf 100644
--- a/tests/gha-run-k8s-common.sh
+++ b/tests/gha-run-k8s-common.sh
@@ -27,11 +27,19 @@ function _print_instance_type() {
     esac
 }
 
+# Print the cluster name set by $AKS_NAME or generated out of runtime
+# metadata (e.g. pull request number, commit SHA, etc).
+#
 function _print_cluster_name() {
-    test_type="${1:-k8s}"
+    local test_type="${1:-k8s}"
+    local short_sha
 
-    short_sha="$(git rev-parse --short=12 HEAD)"
-    echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}"
+    if [ -n "${AKS_NAME:-}" ]; then
+        echo "$AKS_NAME"
+    else
+        short_sha="$(git rev-parse --short=12 HEAD)"
+        echo "${test_type}-${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64-${K8S_TEST_HOST_TYPE:0:1}"
+    fi
 }
 
 function _print_rg_name() {