From 14e74b8fc9d6dae7423d88c0d59ddd9aa8325af7 Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Tue, 15 Apr 2025 18:53:02 -0300
Subject: [PATCH] tests/k8s: fix kbs installation on Azure AKS

The Azure AKS addon-http-application-routing add-on is deprecated and
cannot be enabled on new clusters which has caused some CI jobs to fail.

Migrated our code to use approuting instead. Unlike
addon-http-application-routing, this add-on doesn't
configure a managed cluster DNS zone, but the created ingress has a
public IP. To avoid having to deal with DNS setup, we will be using that
address from now on. Thus, some functions no longer used are deleted.

Fixes #11156
Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 tests/gha-run-k8s-common.sh                   | 25 ++----------
 .../kubernetes/confidential_kbs.sh            | 39 ++++++++-----------
 2 files changed, 20 insertions(+), 44 deletions(-)

diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh
index 55878b4e71..d175e38c1d 100644
--- a/tests/gha-run-k8s-common.sh
+++ b/tests/gha-run-k8s-common.sh
@@ -75,10 +75,10 @@ function _print_rg_name() {
 	echo "${AZ_RG:-"kataCI-$(_print_cluster_name "${test_type}")"}"
 }
 
-# Enable the HTTP application routing add-on to AKS.
+# Enable the approuting routing add-on to AKS.
 # Use with ingress to expose a service API externally.
 #
-function enable_cluster_http_application_routing() {
+function enable_cluster_approuting() {
 	local test_type="${1:-k8s}"
 	local cluster_name
 	local rg
@@ -86,8 +86,7 @@ function enable_cluster_http_application_routing() {
 	rg="$(_print_rg_name "${test_type}")"
 	cluster_name="$(_print_cluster_name "${test_type}")"
 
-	az aks enable-addons -g "${rg}" -n "${cluster_name}" \
-		--addons http_application_routing
+	az aks approuting enable -g "${rg}" -n "${cluster_name}"
 }
 
 function install_azure_cli() {
@@ -194,24 +193,6 @@ function get_cluster_credentials() {
 		-n "$(_print_cluster_name "${test_type}")"
 }
 
-
-# Get the AKS DNS zone name of HTTP application routing.
-#
-# Note: if the HTTP application routing add-on isn't installed in the cluster
-# then it will return an empty string.
-#
-function get_cluster_specific_dns_zone() {
-	local test_type="${1:-k8s}"
-	local cluster_name
-	local rg
-	local q="addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName"
-
-	rg="$(_print_rg_name "${test_type}")"
-	cluster_name="$(_print_cluster_name "${test_type}")"
-
-	az aks show -g "${rg}" -n "${cluster_name}" --query "${q}" | tr -d \"
-}
-
 function delete_cluster() {
 	test_type="${1:-k8s}"
 	local rg
diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh
index d0d9c2f6b8..03a1a15f02 100644
--- a/tests/integration/kubernetes/confidential_kbs.sh
+++ b/tests/integration/kubernetes/confidential_kbs.sh
@@ -419,13 +419,20 @@ function kbs_k8s_deploy() {
 	fi
 }
 
-# Return the kbs service host name in case ingress is configured
+# Return the kbs service public IP in case ingress is configured
 # otherwise the cluster IP.
 #
 kbs_k8s_svc_host() {
 	if kubectl get ingress -n "$KBS_NS" 2>/dev/null | grep -q kbs; then
-		kubectl get ingress "$KBS_INGRESS_NAME" -n "$KBS_NS" \
-			-o jsonpath='{.spec.rules[0].host}' 2>/dev/null
+		local host
+		# The ingress IP address can take a while to show up.
+		SECONDS=0
+		while true; do
+			host=$(kubectl get ingress "${KBS_INGRESS_NAME}" -n "${KBS_NS}" -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+			[[ -z "${host}" && ${SECONDS} -lt 30 ]] || break
+			sleep 5
+		done
+		echo "${host}"
 	elif kubectl get svc "$KBS_SVC_NAME" -n "$KBS_NS" &>/dev/null; then
 			local host
 			host=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' -n "$KBS_NS")
@@ -514,29 +521,17 @@ _handle_ingress() {
 # Implement the ingress handler for AKS.
 #
 _handle_ingress_aks() {
-	local dns_zone
-
-	dns_zone=$(get_cluster_specific_dns_zone "")
-
-	# In case the DNS zone name is empty, the cluster might not have the HTTP
-	# application routing add-on. Let's try to enable it.
-	if [ -z "$dns_zone" ]; then
-		echo "::group::Enable HTTP application routing add-on"
-		enable_cluster_http_application_routing ""
-		echo "::endgroup::"
-		dns_zone=$(get_cluster_specific_dns_zone "")
-	fi
-
-	if [ -z "$dns_zone" ]; then
-		echo "ERROR: the DNS zone name is nil, it cannot configure Ingress"
-		return 1
-	fi
+	echo "::group::Enable approuting (application routing) add-on"
+	enable_cluster_approuting ""
+	echo "::endgroup::"
 
 	pushd "${COCO_KBS_DIR}/config/kubernetes/overlays/"
 
 	echo "::group::$(pwd)/ingress.yaml"
-	KBS_INGRESS_CLASS="addon-http-application-routing" \
-		KBS_INGRESS_HOST="kbs.${dns_zone}" \
+	# We don't use a cluster DNS zone, instead get the ingress public IP,
+	# thus KBS_INGRESS_HOST is set empty.
+	KBS_INGRESS_CLASS="webapprouting.kubernetes.azure.com" \
+		KBS_INGRESS_HOST="\"\"" \
 		envsubst < ingress.yaml | tee ingress.yaml.tmp
 	echo "::endgroup::"
 	mv ingress.yaml.tmp ingress.yaml