From 5e4b7bbd0463f2a68d193ee88d2d418a33791e7c Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Mon, 12 Feb 2024 18:30:21 -0300 Subject: [PATCH] tests/k8s: expose KBS service externally Until this point the deployed KBS service is only reachable from within the cluster. This introduces a generic mechanism to apply an Ingress configuration to expose the service externally. The first implemened ingress is for AKS. In case the HTTP application routing isn't enabled in the cluster (this is required for ingress), an add-on is applied. It was added the get_cluster_specific_dns_zone() and enable_cluster_http_application_routing() helper functions to gha-run-k8s-common.sh. Signed-off-by: Wainer dos Santos Moschetta --- tests/gha-run-k8s-common.sh | 33 ++++++ .../kubernetes/confidential_kbs.sh | 102 ++++++++++++++++++ tests/integration/kubernetes/gha-run.sh | 8 +- 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 2e3b23dda5..391970e90a 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -40,6 +40,21 @@ function _print_rg_name() { echo "${AZ_RG:-"kataCI-$(_print_cluster_name ${test_type})"}" } +# Enable the HTTP application routing add-on to AKS. +# Use with ingress to expose a service API externally. +# +function enable_cluster_http_application_routing() { + local test_type="${1:-k8s}" + local cluster_name + local rg + + rg="$(_print_rg_name "${test_type}")" + cluster_name="$(_print_cluster_name "${test_type}")" + + az aks enable-addons -g "$rg" -n "$cluster_name" \ + --addons http_application_routing +} + function install_azure_cli() { curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash # The aks-preview extension is required while the Mariner Kata host is in preview. @@ -129,6 +144,24 @@ function get_cluster_credentials() { -n "$(_print_cluster_name ${test_type})" } + +# Get the AKS DNS zone name of HTTP application routing. +# +# Note: if the HTTP application routing add-on isn't installed in the cluster +# then it will return an empty string. +# +function get_cluster_specific_dns_zone() { + local test_type="${1:-k8s}" + local cluster_name + local rg + local q="addonProfiles.httpApplicationRouting.config.HTTPApplicationRoutingZoneName" + + rg="$(_print_rg_name "${test_type}")" + cluster_name="$(_print_cluster_name "${test_type}")" + + az aks show -g "$rg" -n "$cluster_name" --query "$q" | tr -d \" +} + function delete_cluster() { test_type="${1:-k8s}" local rg diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh index 5f024564b2..259a55827c 100644 --- a/tests/integration/kubernetes/confidential_kbs.sh +++ b/tests/integration/kubernetes/confidential_kbs.sh @@ -34,10 +34,16 @@ function kbs_k8s_delete() { # Deploy the kbs on Kubernetes # +# Parameters: +# $1 - apply the specificed ingress handler to expose the service externally +# function kbs_k8s_deploy() { local image local image_tag + local ingress=${1:-} local repo + local svc_host + local timeout local kbs_ip local kbs_port local version @@ -51,6 +57,15 @@ function kbs_k8s_deploy() { image=$(get_from_kata_deps "externals.coco-kbs.image") image_tag=$(get_from_kata_deps "externals.coco-kbs.image_tag") + # The ingress handler for AKS relies on the cluster's name which in turn + # contain the HEAD commit of the kata-containers repository (supposedly the + # current directory). It will be needed to save the cluster's name before + # it switches to the kbs repository and get a wrong HEAD commit. + if [ -z "${AKS_NAME:-}" ]; then + AKS_NAME=$(_print_cluster_name) + export AKS_NAME + fi + if [ -d "$COCO_KBS_DIR" ]; then rm -rf "$COCO_KBS_DIR" fi @@ -75,6 +90,8 @@ function kbs_k8s_deploy() { popd echo "::endgroup::" + [ -n "$ingress" ] && _handle_ingress "$ingress" + echo "::group::Deploy the KBS" ./deploy-kbs.sh popd @@ -116,4 +133,89 @@ function kbs_k8s_deploy() { kubectl delete pod "$pod" echo "KBS service respond to requests" echo "::endgroup::" + + if [ -n "$ingress" ]; then + echo "::group::Check the kbs service is exposed" + svc_host=$(kbs_k8s_svc_host) + if [ -z "$svc_host" ]; then + echo "ERROR: service host not found" + return 1 + fi + + # AZ DNS can take several minutes to update its records so that + # the host name will take a while to start resolving. + timeout=350 + echo "Trying to connect at $svc_host. Timeout=$timeout" + if ! waitForProcess "$timeout" "30" "curl -s -I \"$svc_host\" | grep -q \"404 Not Found\""; then + echo "ERROR: service seems to not respond on $svc_host host" + curl -I "$svc_host" + return 1 + fi + echo "KBS service respond to requests at $svc_host" + echo "::endgroup::" + fi +} + +# Return the kbs service host name in case ingress is configured +# otherwise the cluster IP. +# +kbs_k8s_svc_host() { + if kubectl get ingress -n "$KBS_NS" | grep -q kbs; then + kubectl get ingress kbs -n "$KBS_NS" \ + -o jsonpath='{.spec.rules[0].host}' 2>/dev/null + else + kubectl get svc kbs -n "$KBS_NS" \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null + fi +} + +# Choose the appropriated ingress handler. +# +# To add a new handler, create a function named as _handle_ingress_NAME where +# NAME is the handler name. This is enough for this method to pick up the right +# implementation. +# +_handle_ingress() { + local ingress="$1" + + type -a "_handle_ingress_$ingress" &>/dev/null || { + echo "ERROR: ingress '$ingress' handler not implemented"; + return 1; + } + + "_handle_ingress_$ingress" +} + +# Implement the ingress handler for AKS. +# +_handle_ingress_aks() { + local dns_zone + + dns_zone=$(get_cluster_specific_dns_zone "") + + # In case the DNS zone name is empty, the cluster might not have the HTTP + # application routing add-on. Let's try to enable it. + if [ -z "$dns_zone" ]; then + echo "::group::Enable HTTP application routing add-on" + enable_cluster_http_application_routing "" + echo "::endgroup::" + dns_zone=$(get_cluster_specific_dns_zone "") + fi + + if [ -z "$dns_zone" ]; then + echo "ERROR: the DNS zone name is nil, it cannot configure Ingress" + return 1 + fi + + pushd "$COCO_KBS_DIR/kbs/config/kubernetes/overlays" + + echo "::group::$(pwd)/ingress.yaml" + KBS_INGRESS_CLASS="addon-http-application-routing" \ + KBS_INGRESS_HOST="kbs.${dns_zone}" \ + envsubst < ingress.yaml | tee ingress.yaml.tmp + echo "::endgroup::" + mv ingress.yaml.tmp ingress.yaml + + kustomize edit add resource ingress.yaml + popd } \ No newline at end of file diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index bf7f35e6e8..0f17a1acb8 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -111,8 +111,14 @@ function delete_coco_kbs() { kbs_k8s_delete } +# Deploy the CoCo KBS in Kubernetes +# +# Environment variables: +# KBS_INGRESS - (optional) specify the ingress implementation to expose the +# service externally +# function deploy_coco_kbs() { - kbs_k8s_deploy + kbs_k8s_deploy "$KBS_INGRESS" } function deploy_kata() {