From b535c7cbd8fb7def4a198c8e679bc8443b1567d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= <abombo@microsoft.com>
Date: Tue, 30 May 2023 11:01:28 -0700
Subject: [PATCH 1/9] tests: Enable running k8s tests on Mariner
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This removes the gate and lets CI run tests on Mariner.

Fixes: #6840

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
---
 tests/integration/kubernetes/run_kubernetes_tests.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
index 0975ec0d5..db1e16633 100644
--- a/tests/integration/kubernetes/run_kubernetes_tests.sh
+++ b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -54,10 +54,6 @@ else
 	)
 fi
 
-if [ ${KATA_HOST_OS} == "cbl-mariner" ]; then
-	exit 0
-fi
-
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then

From 532755ce31d069dafaa1427c55757f0973381647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= <abombo@microsoft.com>
Date: Mon, 17 Apr 2023 16:59:22 -0700
Subject: [PATCH 2/9] tests: Build Mariner rootfs initrd
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

 * Adds a new `rootfs-initrd-mariner` build target.
 * Sets the custom initrd path via annotation in `setup.sh` at test
   time.
 * Adapts versions.yaml to specify a `cbl-mariner` initrd variant.
 * Introduces env variable `HOST_OS` at deploy time to enable using a
   custom initrd.
 * Refactors the image builder so that its caller specifies the desired
   guest OS.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
---
 tests/integration/gha-run.sh                  |  7 ++-
 tests/integration/kubernetes/setup.sh         | 10 +++
 tools/packaging/guest-image/build_image.sh    | 63 +++++++++----------
 .../local-build/kata-deploy-binaries.sh       | 56 ++++++++++++++---
 .../kata-deploy/scripts/kata-deploy.sh        |  5 ++
 versions.yaml                                 | 26 +++++---
 6 files changed, 113 insertions(+), 54 deletions(-)

diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh
index 103ce2cda..b2493e3b9 100755
--- a/tests/integration/gha-run.sh
+++ b/tests/integration/gha-run.sh
@@ -9,7 +9,8 @@ set -o nounset
 set -o pipefail
 
 integration_dir="$(dirname "$(readlink -f "$0")")"
-tools_dir="${integration_dir}/../../tools"
+repo_root_dir="$(cd "${integration_dir}/../../" && pwd)"
+tools_dir="${repo_root_dir}/tools"
 
 function _print_cluster_name() {
     short_sha="$(git rev-parse --short=12 HEAD)"
@@ -56,9 +57,13 @@ function get_cluster_credentials() {
 }
 
 function run_tests() {
+    INSTALL_IN_GOPATH=false bash "${repo_root_dir}/ci/install_yq.sh"
+
     platform="${1}"
 
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
+    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
 
diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh
index 0c3baf2dc..63d9fb682 100755
--- a/tests/integration/kubernetes/setup.sh
+++ b/tests/integration/kubernetes/setup.sh
@@ -8,13 +8,23 @@ set -o nounset
 set -o pipefail
 
 kubernetes_dir=$(dirname "$(readlink -f "$0")")
+repo_root_dir="$(cd "${kubernetes_dir}/../../../" && pwd)"
 
 set_runtime_class() {
     sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml
 }
 
+set_initrd_path() {
+    if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
+        initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-cbl-mariner.img"
+        find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \;
+    fi
+}
+
 main() {
+    INSTALL_IN_GOPATH=false bash "${repo_root_dir}/ci/install_yq.sh"
     set_runtime_class
+    set_initrd_path
 }
 
 main "$@"
diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh
index 230538d1c..fad664651 100755
--- a/tools/packaging/guest-image/build_image.sh
+++ b/tools/packaging/guest-image/build_image.sh
@@ -22,45 +22,44 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
 export GOPATH=${GOPATH:-${HOME}/go}
 
 arch_target="$(uname -m)"
-final_image_name="kata-containers"
-final_initrd_name="kata-containers-initrd"
+final_artifact_name="kata-containers"
 image_initrd_extension=".img"
 
 build_initrd() {
 	info "Build initrd"
-	info "initrd os: $initrd_distro"
-	info "initrd os version: $initrd_os_version"
+	info "initrd os: $os_name"
+	info "initrd os version: $os_version"
 	sudo -E PATH="$PATH" make initrd \
-		DISTRO="$initrd_distro" \
+		DISTRO="$os_name" \
 		DEBUG="${DEBUG:-}" \
-		OS_VERSION="${initrd_os_version}" \
+		OS_VERSION="${os_version}" \
 		ROOTFS_BUILD_DEST="${builddir}/initrd-image" \
 		USE_DOCKER=1 \
 		AGENT_INIT="yes"
-	mv "kata-containers-initrd.img" "${install_dir}/${initrd_name}"
+	mv "kata-containers-initrd.img" "${install_dir}/${artifact_name}"
 	(
 		cd "${install_dir}"
-		ln -sf "${initrd_name}" "${final_initrd_name}${image_initrd_extension}"
+		ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}"
 	)
 }
 
 build_image() {
 	info "Build image"
-	info "image os: $img_distro"
-	info "image os version: $img_os_version"
+	info "image os: $os_name"
+	info "image os version: $os_version"
 	sudo -E PATH="${PATH}" make image \
-		DISTRO="${img_distro}" \
+		DISTRO="${os_name}" \
 		DEBUG="${DEBUG:-}" \
 		USE_DOCKER="1" \
-		IMG_OS_VERSION="${img_os_version}" \
+		IMG_OS_VERSION="${os_version}" \
 		ROOTFS_BUILD_DEST="${builddir}/rootfs-image"
-	mv -f "kata-containers.img" "${install_dir}/${image_name}"
+	mv -f "kata-containers.img" "${install_dir}/${artifact_name}"
 	if [ -e "root_hash.txt" ]; then
 	    cp root_hash.txt "${install_dir}/"
 	fi
 	(
 		cd "${install_dir}"
-		ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}"
+		ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}"
 	)
 }
 
@@ -74,6 +73,8 @@ Usage:
 ${script_name} [options]
 
 Options:
+ --osname=${os_name}
+ --osversion=${os_version}
  --imagetype=${image_type}
  --prefix=${prefix}
  --destdir=${destdir}
@@ -94,33 +95,20 @@ main() {
 		case "$opt" in
 		-)
 			case "${OPTARG}" in
+			osname=*)
+				os_name=${OPTARG#*=}
+				;;
+			osversion=*)
+				os_version=${OPTARG#*=}
+				;;
 			imagetype=image)
 				image_type=image
-				#image information
-				img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name")
-				img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version")
-				image_name="kata-${img_distro}-${img_os_version}.${image_type}"
 				;;
 			imagetype=initrd)
 				image_type=initrd
-				#initrd information
-				initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name")
-				initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version")
-				initrd_name="kata-${initrd_distro}-${initrd_os_version}.${image_type}"
 				;;
 			image_initrd_suffix=*)
 				image_initrd_suffix=${OPTARG#*=}
-				if [ "${image_initrd_suffix}" == "sev" ]; then
-					initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name")
-					initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version")
-					initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}"
-					final_initrd_name="${final_initrd_name}-${image_initrd_suffix}"
-				elif [ "${image_initrd_suffix}" == "tdx" ]; then
-					img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name")
-					img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version")
-					image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}"
-					final_image_name="${final_image_name}-${image_initrd_suffix}"
-				fi
 				;;
 			prefix=*)
 				prefix=${OPTARG#*=}
@@ -149,7 +137,16 @@ main() {
 
 	echo "build ${image_type}"
 
+	if [ "${image_type}" = "initrd" ]; then
+		final_artifact_name+="-initrd"
+	fi
 
+	if [ -n "${image_initrd_suffix}" ]; then
+		artifact_name="kata-${os_name}-${os_version}-${image_initrd_suffix}.${image_type}"
+		final_artifact_name+="-${image_initrd_suffix}"
+	else
+		artifact_name="kata-${os_name}-${os_version}.${image_type}"
+	fi
 
 	install_dir="${destdir}/${prefix}/share/kata-containers/"
 	readonly install_dir
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index a552aed12..58e62bb48 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -97,6 +97,7 @@ options:
 	rootfs-image
 	rootfs-image-tdx
 	rootfs-initrd
+	rootfs-initrd-mariner
 	rootfs-initrd-sev
 	shim-v2
 	tdvf
@@ -136,8 +137,13 @@ install_cached_tarball_component() {
 
 #Install guest image
 install_image() {
-	local image_type="${1:-"image"}"
-	local initrd_suffix="${2:-""}"
+	local variant="${1:-}"
+
+	image_type="image"
+	if [ -n "${variant}" ]; then
+		image_type+="-${variant}"
+	fi
+
 	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}"
 	local component="rootfs-${image_type}"
 
@@ -152,25 +158,39 @@ install_image() {
 	install_cached_tarball_component \
 		"${component}" \
 		"${jenkins}" \
-		"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \
+		"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${image_type}" \
 		"" \
 		"${final_tarball_name}" \
 		"${final_tarball_path}" \
 		&& return 0
 
 	info "Create image"
-	"${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
+
+	if [ -n "${variant}" ]; then
+		os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.name")"
+		os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.version")"
+	else
+		os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.name")"
+		os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.version")"
+	fi
+	
+	"${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}"
 }
 
 #Install guest image for tdx
 install_image_tdx() {
-	install_image "image-tdx" "tdx"
+	install_image "tdx"
 }
 
 #Install guest initrd
 install_initrd() {
-	local initrd_type="${1:-"initrd"}"
-	local initrd_suffix="${2:-""}"
+	local variant="${1:-}"
+
+	initrd_type="initrd"
+	if [ -n "${variant}" ]; then
+		initrd_type+="-${variant}"
+	fi
+
 	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}"
 	local component="rootfs-${initrd_type}"
 
@@ -192,12 +212,26 @@ install_initrd() {
 		&& return 0
 
 	info "Create initrd"
-	"${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
+
+	if [ -n "${variant}" ]; then
+		os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.name")"
+		os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.version")"
+	else
+		os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.name")"
+		os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.version")"
+	fi
+
+	"${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}"
+}
+
+#Install Mariner guest initrd
+install_initrd_mariner() {
+	install_initrd "cbl-mariner"
 }
 
 #Install guest initrd for sev
 install_initrd_sev() {
-	install_initrd "initrd-sev" "sev"
+	install_initrd "sev"
 }
 
 #Install kernel component helper
@@ -561,6 +595,7 @@ handle_build() {
 		install_firecracker
 		install_image
 		install_initrd
+		install_initrd_mariner
 		install_initrd_sev
 		install_kernel
 		install_kernel_dragonball_experimental
@@ -616,7 +651,7 @@ handle_build() {
 
 	rootfs-initrd) install_initrd ;;
 
-	rootfs-initrd-mariner) ;;
+	rootfs-initrd-mariner) install_initrd_mariner ;;
 
 	rootfs-initrd-sev) install_initrd_sev ;;
 	
@@ -662,6 +697,7 @@ main() {
 		qemu
 		rootfs-image
 		rootfs-initrd
+		rootfs-initrd-mariner
 		shim-v2
 		virtiofsd
 	)
diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 6bb660198..155bdf1ff 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -64,6 +64,11 @@ function install_artifacts() {
 	chmod +x /opt/kata/bin/*
 	[ -d /opt/kata/runtime-rs/bin ] && \
 		chmod +x /opt/kata/runtime-rs/bin/*
+
+	# Allow Mariner to specify a Mariner guest initrd.
+	if [ "${HOST_OS:-}" == "cbl-mariner" ]; then
+		sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd"]|' /opt/kata/share/defaults/kata-containers/configuration-clh.toml
+	fi
 }
 
 function wait_till_node_is_ready() {
diff --git a/versions.yaml b/versions.yaml
index ec9f8df7a..2a3623878 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -122,17 +122,20 @@ assets:
     url: "https://github.com/kata-containers/kata-containers/tools/osbuilder"
     architecture:
       aarch64:
-        name: "ubuntu"
-        version: "latest"
-      ppc64le:
-        name: "ubuntu"
-        version: "latest"
-      s390x:
-        name: "ubuntu"
-        version: "latest"
-      x86_64:
         name: &default-image-name "ubuntu"
-        version: "latest"
+        version: &default-image-version "latest"
+      ppc64le:
+        name: *default-image-name
+        version: *default-image-version
+      s390x:
+        name: *default-image-name
+        version: *default-image-version
+      x86_64:
+        name: *default-image-name
+        version: *default-image-version
+        tdx:
+          name: *default-image-name
+          version: *default-image-version
     meta:
       image-type: *default-image-name
 
@@ -156,6 +159,9 @@ assets:
       x86_64:
         name: *default-initrd-name
         version: *default-initrd-version
+        cbl-mariner:
+          name: "cbl-mariner"
+          version: "2.0"
         sev:
           name: *glibc-initrd-name
           version: *glibc-initrd-version

From 80c78eadcea44018687e63a84b975386258fdc59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= <abombo@microsoft.com>
Date: Tue, 18 Apr 2023 13:29:38 -0700
Subject: [PATCH 3/9] tests: Use baked-in kernel with Mariner
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mariner ships a bleeding-edge kernel that might be ahead of upstream, so
we use that to guarantee compatibility with the host.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
---
 tests/integration/kubernetes/setup.sh              | 8 ++++++++
 tools/packaging/kata-deploy/scripts/kata-deploy.sh | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh
index 63d9fb682..639826bc8 100755
--- a/tests/integration/kubernetes/setup.sh
+++ b/tests/integration/kubernetes/setup.sh
@@ -14,6 +14,13 @@ set_runtime_class() {
     sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml
 }
 
+set_kernel_path() {
+    if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
+        mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin"
+        find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \;
+    fi
+}
+
 set_initrd_path() {
     if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
         initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-cbl-mariner.img"
@@ -24,6 +31,7 @@ set_initrd_path() {
 main() {
     INSTALL_IN_GOPATH=false bash "${repo_root_dir}/ci/install_yq.sh"
     set_runtime_class
+    set_kernel_path
     set_initrd_path
 }
 
diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 155bdf1ff..368492bd7 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -65,9 +65,9 @@ function install_artifacts() {
 	[ -d /opt/kata/runtime-rs/bin ] && \
 		chmod +x /opt/kata/runtime-rs/bin/*
 
-	# Allow Mariner to specify a Mariner guest initrd.
+	# Allow Mariner to use custom configuration.
 	if [ "${HOST_OS:-}" == "cbl-mariner" ]; then
-		sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd"]|' /opt/kata/share/defaults/kata-containers/configuration-clh.toml
+		sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' /opt/kata/share/defaults/kata-containers/configuration-clh.toml
 	fi
 }
 

From 2b59756894e6e50ce85cf1ae4fd46d9566a049b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= <abombo@microsoft.com>
Date: Fri, 2 Jun 2023 13:17:34 -0700
Subject: [PATCH 4/9] tests: Build CLH with glibc for Mariner
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This enables building CLH with glibc and the mshv feature as required
for Mariner. At test time, it also configures Kata to use that CLH
flavor when running Mariner.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
---
 .../local-build/kata-deploy-binaries.sh       | 44 ++++++++++++++-----
 .../kata-deploy/scripts/kata-deploy.sh        |  6 ++-
 .../cloud-hypervisor/build-static-clh.sh      |  6 +--
 3 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 58e62bb48..14910f43f 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -79,6 +79,7 @@ options:
 --build=<asset>       :
 	all
 	cloud-hypervisor
+	cloud-hypervisor-glibc
 	firecracker
 	kernel
 	kernel-dragonball-experimental
@@ -447,26 +448,47 @@ install_firecracker() {
 	sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/jailer-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/jailer"
 }
 
-# Install static cloud-hypervisor asset
-install_clh() {
+install_clh_helper() {
+	libc="${1}"
+	features="${2}"
+	suffix="${3:-""}"
+
 	install_cached_tarball_component \
-		"cloud-hypervisor" \
-		"${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \
+		"cloud-hypervisor${suffix}" \
+		"${jenkins_url}/job/kata-containers-main-clh-$(uname -m)${suffix}/${cached_artifacts_path}" \
 		"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
 		"" \
 		"${final_tarball_name}" \
 		"${final_tarball_path}" \
 		&& return 0
 
-	if [[ "${ARCH}" == "x86_64" ]]; then
-		export features="tdx"
-	fi
-
 	info "build static cloud-hypervisor"
-	"${clh_builder}"
+	libc="${libc}" features="${features}" "${clh_builder}"
 	info "Install static cloud-hypervisor"
 	mkdir -p "${destdir}/opt/kata/bin/"
-	sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor"
+	sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor${suffix}"
+}
+
+# Install static cloud-hypervisor asset
+install_clh() {
+	if [[ "${ARCH}" == "x86_64" ]]; then
+		features="mshv,tdx"
+	else
+		features=""
+	fi
+
+	install_clh_helper "musl" "${features}"
+}
+
+# Install static cloud-hypervisor-glibc asset
+install_clh_glibc() {
+	if [[ "${ARCH}" == "x86_64" ]]; then
+		features="mshv"
+	else
+		features=""
+	fi
+
+	install_clh_helper "gnu" "${features}" "-glibc"
 }
 
 # Install static virtiofsd asset
@@ -613,7 +635,7 @@ handle_build() {
 
 	cloud-hypervisor) install_clh ;;
 
-	cloud-hypervisor-glibc) ;;
+	cloud-hypervisor-glibc) install_clh_glibc ;;
 
 	firecracker) install_firecracker ;;
 
diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 368492bd7..09d27cc65 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -67,7 +67,11 @@ function install_artifacts() {
 
 	# Allow Mariner to use custom configuration.
 	if [ "${HOST_OS:-}" == "cbl-mariner" ]; then
-		sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' /opt/kata/share/defaults/kata-containers/configuration-clh.toml
+		config_path="/opt/kata/share/defaults/kata-containers/configuration-clh.toml"
+		clh_path="/opt/kata/bin/cloud-hypervisor-glibc"
+		sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' "${config_path}"
+		sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}"
+		sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}"
 	fi
 }
 
diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
index 975a517a1..f381897bc 100755
--- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
+++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh
@@ -76,12 +76,12 @@ build_clh_from_source() {
 
 	if [ -n "${features}" ]; then
 		info "Build cloud-hypervisor enabling the following features: ${features}"
-		./scripts/dev_cli.sh build --release --libc musl --features "${features}"
+		./scripts/dev_cli.sh build --release --libc "${libc}" --features "${features}"
 	else
-		./scripts/dev_cli.sh build --release --libc musl
+		./scripts/dev_cli.sh build --release --libc "${libc}"
 	fi
 	rm -f cloud-hypervisor
-	cp build/cargo_target/$(uname -m)-unknown-linux-musl/release/cloud-hypervisor .
+	cp build/cargo_target/$(uname -m)-unknown-linux-${libc}/release/cloud-hypervisor .
 	popd
 }
 

From 0152c9aba5c8134111acb671d05d4722f986d3b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= <abombo@microsoft.com>
Date: Tue, 13 Jun 2023 11:37:18 -0700
Subject: [PATCH 5/9] tools: Introduce `USE_CACHE` environment variable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This allows setting `USE_CACHE=no` to test building e2e during
developmet without having to comment code blocks and so forth.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
---
 .../local-build/kata-deploy-binaries-in-docker.sh            | 1 +
 .../kata-deploy/local-build/kata-deploy-binaries.sh          | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
index 11589c88a..fc82082c4 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh
@@ -65,6 +65,7 @@ docker run \
 	--env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \
 	--env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \
 	--env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \
+	--env USE_CACHE="${USE_CACHE:-}" \
 	--rm \
 	-w ${script_dir} \
 	build-kata-deploy "${kata_deploy_create}" $@
diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
index 14910f43f..01a2d848f 100755
--- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
+++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -40,6 +40,7 @@ readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts"
 
 ARCH=$(uname -m)
 MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
+USE_CACHE="${USE_CACHE:-"yes"}"
 
 workdir="${WORKDIR:-$PWD}"
 
@@ -115,6 +116,10 @@ cleanup_and_fail() {
 }
 
 install_cached_tarball_component() {
+	if [ "${USE_CACHE}" != "yes" ]; then
+		return 1
+	fi
+
 	local component="${1}"
 	local jenkins_build_url="${2}"
 	local current_version="${3}"

From 1c211cd730e51008deb9cd1344e551f104cfb6d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= <abombo@microsoft.com>
Date: Tue, 13 Jun 2023 12:07:49 -0700
Subject: [PATCH 6/9] gha: Swap asset/release in build matrix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This simply displays the asset name first in GH's UI, so that the
release name (always "test") is truncated rather than the asset name.
Makes things slightly easier to read.

e.g.

    build-asset (cloud-hypervisor-glibc, te...

instead of

    build-asset (test, cloud-hypervisor-gli...

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
---
 .github/workflows/build-kata-static-tarball-amd64.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-kata-static-tarball-amd64.yaml b/.github/workflows/build-kata-static-tarball-amd64.yaml
index b5c7584fe..9a0cd755c 100644
--- a/.github/workflows/build-kata-static-tarball-amd64.yaml
+++ b/.github/workflows/build-kata-static-tarball-amd64.yaml
@@ -19,8 +19,6 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        stage:
-          - ${{ inputs.stage }}
         asset:
           - cloud-hypervisor
           - cloud-hypervisor-glibc
@@ -46,9 +44,11 @@ jobs:
           - shim-v2
           - tdvf
           - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
         exclude:
-          - stage: release
-            asset: cloud-hypervisor-glibc
+          - asset: cloud-hypervisor-glibc
+            stage: release
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}

From 40c46c75eda27e0db5fd1802f26e659c1242620b Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Mon, 3 Jul 2023 16:43:55 +0200
Subject: [PATCH 7/9] tests/integration: Perform yq install in run_tests()

We only need to install in run_tests() so that the yq install is picked up by
kubernets/setup.sh as well. We also need to either use (sudo &&
INSTALL_IN_GOPATH=false) || (INSTALL_IN_GOPATH=true).

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 tests/integration/gha-run.sh          | 10 ++++++++--
 tests/integration/kubernetes/setup.sh |  1 -
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh
index b2493e3b9..8fb6c14fa 100755
--- a/tests/integration/gha-run.sh
+++ b/tests/integration/gha-run.sh
@@ -56,10 +56,16 @@ function get_cluster_credentials() {
         -n "$(_print_cluster_name)"
 }
 
-function run_tests() {
-    INSTALL_IN_GOPATH=false bash "${repo_root_dir}/ci/install_yq.sh"
+function ensure_yq() {
+    : "${GOPATH:=${GITHUB_WORKSPACE}}"
+    export GOPATH
+    export PATH="${GOPATH}/bin:${PATH}"
+    INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh"
+}
 
+function run_tests() {
     platform="${1}"
+    ensure_yq
 
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
     yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh
index 639826bc8..6984ad286 100755
--- a/tests/integration/kubernetes/setup.sh
+++ b/tests/integration/kubernetes/setup.sh
@@ -29,7 +29,6 @@ set_initrd_path() {
 }
 
 main() {
-    INSTALL_IN_GOPATH=false bash "${repo_root_dir}/ci/install_yq.sh"
     set_runtime_class
     set_kernel_path
     set_initrd_path

From d6e96ea06dfa7746fd36cc666f555c71b9204a96 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Tue, 4 Jul 2023 11:27:47 +0200
Subject: [PATCH 8/9] tests/integration: Use AzureLinux instead of Mariner

as OSSKU value, to get rid of this warning when creating the AKS cluster:

WARNING: The osSKU "AzureLinux" should be used going forward instead of
"CBLMariner" or "Mariner". The osSKUs "CBLMariner" and "Mariner" will
eventually be deprecated.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 tests/integration/gha-run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh
index 8fb6c14fa..6abdafae2 100755
--- a/tests/integration/gha-run.sh
+++ b/tests/integration/gha-run.sh
@@ -38,7 +38,7 @@ function create_cluster() {
         -s "Standard_D4s_v5" \
         --node-count 1 \
         --generate-ssh-keys \
-        $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
+        $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation")
 }
 
 function install_bats() {

From b568c7f7d839509c40b94df38d9abc5ad41fd80d Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Tue, 4 Jul 2023 14:28:29 +0200
Subject: [PATCH 9/9] tests/integration: Provide default value for KATA_HOST_OS

Non AKS k8s tests (SEV/SNP/TDX) don't currently set KATA_HOST_OS, so provide a
default empty value for the variable so that those tests can run.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 tests/integration/gha-run.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh
index 6abdafae2..cdbce7211 100755
--- a/tests/integration/gha-run.sh
+++ b/tests/integration/gha-run.sh
@@ -68,8 +68,10 @@ function run_tests() {
     ensure_yq
 
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
-    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
+    if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
+        yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
+        yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
+    fi
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
 
@@ -131,6 +133,8 @@ function delete_cluster() {
 }
 
 function main() {
+    export KATA_HOST_OS="${KATA_HOST_OS:-}"
+
     action="${1:-}"
 
     case "${action}" in