Merge pull request #6839 from sprt/sprt/mariner-ci-tests

tests: Enable running k8s tests on Mariner

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -22,8 +22,6 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        stage:
-          - ${{ inputs.stage }}
         asset:
           - cloud-hypervisor
           - cloud-hypervisor-glibc
@@ -49,9 +47,11 @@ jobs:
           - shim-v2
           - tdvf
           - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
         exclude:
-          - stage: release
+          - asset: cloud-hypervisor-glibc
-            asset: cloud-hypervisor-glibc
+            stage: release
     steps:
       - name: Login to Kata Containers quay.io
         if: ${{ inputs.push-to-registry == 'yes' }}

tests/integration/gha-run.sh

@@ -9,7 +9,8 @@ set -o nounset
 set -o pipefail
 
 integration_dir="$(dirname "$(readlink -f "$0")")"
-tools_dir="${integration_dir}/../../tools"
+repo_root_dir="$(cd "${integration_dir}/../../" && pwd)"
+tools_dir="${repo_root_dir}/tools"
 
 function _print_cluster_name() {
     short_sha="$(git rev-parse --short=12 HEAD)"
@@ -37,7 +38,7 @@ function create_cluster() {
         -s "Standard_D4s_v5" \
         --node-count 1 \
         --generate-ssh-keys \
-        $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
+        $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku AzureLinux --workload-runtime KataMshvVmIsolation")
 }
 
 function install_bats() {
@@ -55,8 +56,16 @@ function get_cluster_credentials() {
         -n "$(_print_cluster_name)"
 }
 
+function ensure_yq() {
+    : "${GOPATH:=${GITHUB_WORKSPACE}}"
+    export GOPATH
+    export PATH="${GOPATH}/bin:${PATH}"
+    INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh"
+}
+
 function run_tests() {
     platform="${1}"
+    ensure_yq
 
     # Emsure we're in the default namespace
     kubectl config set-context --current --namespace=default
@@ -65,6 +74,10 @@ function run_tests() {
     kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true
 
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+    if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
+        yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
+        yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
+    fi
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
 
@@ -134,6 +147,8 @@ function delete_cluster() {
 }
 
 function main() {
+    export KATA_HOST_OS="${KATA_HOST_OS:-}"
+
     action="${1:-}"
 
     case "${action}" in

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -54,10 +54,6 @@ else
 	)
 fi
 
-if [ ${KATA_HOST_OS} == "cbl-mariner" ]; then
-	exit 0
-fi
-
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then

tests/integration/kubernetes/setup.sh

@@ -8,13 +8,30 @@ set -o nounset
 set -o pipefail
 
 kubernetes_dir=$(dirname "$(readlink -f "$0")")
+repo_root_dir="$(cd "${kubernetes_dir}/../../../" && pwd)"
 
 set_runtime_class() {
     sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml
 }
 
+set_kernel_path() {
+    if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
+        mariner_kernel_path="/usr/share/cloud-hypervisor/vmlinux.bin"
+        find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.kernel]' "${mariner_kernel_path}" \;
+    fi
+}
+
+set_initrd_path() {
+    if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
+        initrd_path="/opt/kata/share/kata-containers/kata-containers-initrd-cbl-mariner.img"
+        find ${kubernetes_dir}/runtimeclass_workloads/*.yaml -exec yq write -i {} 'metadata.annotations[io.katacontainers.config.hypervisor.initrd]' "${initrd_path}" \;
+    fi
+}
+
 main() {
     set_runtime_class
+    set_kernel_path
+    set_initrd_path
 }
 
 main "$@"

tools/packaging/guest-image/build_image.sh

@@ -22,45 +22,44 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
 export GOPATH=${GOPATH:-${HOME}/go}
 
 arch_target="$(uname -m)"
-final_image_name="kata-containers"
+final_artifact_name="kata-containers"
-final_initrd_name="kata-containers-initrd"
 image_initrd_extension=".img"
 
 build_initrd() {
 	info "Build initrd"
-	info "initrd os: $initrd_distro"
+	info "initrd os: $os_name"
-	info "initrd os version: $initrd_os_version"
+	info "initrd os version: $os_version"
 	sudo -E PATH="$PATH" make initrd \
-		DISTRO="$initrd_distro" \
+		DISTRO="$os_name" \
 		DEBUG="${DEBUG:-}" \
-		OS_VERSION="${initrd_os_version}" \
+		OS_VERSION="${os_version}" \
 		ROOTFS_BUILD_DEST="${builddir}/initrd-image" \
 		USE_DOCKER=1 \
 		AGENT_INIT="yes"
-	mv "kata-containers-initrd.img" "${install_dir}/${initrd_name}"
+	mv "kata-containers-initrd.img" "${install_dir}/${artifact_name}"
 	(
 		cd "${install_dir}"
-		ln -sf "${initrd_name}" "${final_initrd_name}${image_initrd_extension}"
+		ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}"
 	)
 }
 
 build_image() {
 	info "Build image"
-	info "image os: $img_distro"
+	info "image os: $os_name"
-	info "image os version: $img_os_version"
+	info "image os version: $os_version"
 	sudo -E PATH="${PATH}" make image \
-		DISTRO="${img_distro}" \
+		DISTRO="${os_name}" \
 		DEBUG="${DEBUG:-}" \
 		USE_DOCKER="1" \
-		IMG_OS_VERSION="${img_os_version}" \
+		IMG_OS_VERSION="${os_version}" \
 		ROOTFS_BUILD_DEST="${builddir}/rootfs-image"
-	mv -f "kata-containers.img" "${install_dir}/${image_name}"
+	mv -f "kata-containers.img" "${install_dir}/${artifact_name}"
 	if [ -e "root_hash.txt" ]; then
 	    cp root_hash.txt "${install_dir}/"
 	fi
 	(
 		cd "${install_dir}"
-		ln -sf "${image_name}" "${final_image_name}${image_initrd_extension}"
+		ln -sf "${artifact_name}" "${final_artifact_name}${image_initrd_extension}"
 	)
 }
 
@@ -74,6 +73,8 @@ Usage:
 ${script_name} [options]
 
 Options:
+ --osname=${os_name}
+ --osversion=${os_version}
  --imagetype=${image_type}
  --prefix=${prefix}
  --destdir=${destdir}
@@ -94,33 +95,20 @@ main() {
 		case "$opt" in
 		-)
 			case "${OPTARG}" in
+			osname=*)
+				os_name=${OPTARG#*=}
+				;;
+			osversion=*)
+				os_version=${OPTARG#*=}
+				;;
 			imagetype=image)
 				image_type=image
-				#image information
-				img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name")
-				img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version")
-				image_name="kata-${img_distro}-${img_os_version}.${image_type}"
 				;;
 			imagetype=initrd)
 				image_type=initrd
-				#initrd information
-				initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.name")
-				initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.version")
-				initrd_name="kata-${initrd_distro}-${initrd_os_version}.${image_type}"
 				;;
 			image_initrd_suffix=*)
 				image_initrd_suffix=${OPTARG#*=}
-				if [ "${image_initrd_suffix}" == "sev" ]; then
-					initrd_distro=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.name")
-					initrd_os_version=$(get_from_kata_deps "assets.initrd.architecture.${arch_target}.sev.version")
-					initrd_name="kata-${initrd_distro}-${initrd_os_version}-${image_initrd_suffix}.${image_type}"
-					final_initrd_name="${final_initrd_name}-${image_initrd_suffix}"
-				elif [ "${image_initrd_suffix}" == "tdx" ]; then
-					img_distro=$(get_from_kata_deps "assets.image.architecture.${arch_target}.name")
-					img_os_version=$(get_from_kata_deps "assets.image.architecture.${arch_target}.version")
-					image_name="kata-${img_distro}-${img_os_version}-${image_initrd_suffix}.${image_type}"
-					final_image_name="${final_image_name}-${image_initrd_suffix}"
-				fi
 				;;
 			prefix=*)
 				prefix=${OPTARG#*=}
@@ -149,7 +137,16 @@ main() {
 
 	echo "build ${image_type}"
 
+	if [ "${image_type}" = "initrd" ]; then
+		final_artifact_name+="-initrd"
+	fi
 
+	if [ -n "${image_initrd_suffix}" ]; then
+		artifact_name="kata-${os_name}-${os_version}-${image_initrd_suffix}.${image_type}"
+		final_artifact_name+="-${image_initrd_suffix}"
+	else
+		artifact_name="kata-${os_name}-${os_version}.${image_type}"
+	fi
 
 	install_dir="${destdir}/${prefix}/share/kata-containers/"
 	readonly install_dir

tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh

@@ -65,6 +65,7 @@ docker run \
 	--env TDSHIM_CONTAINER_BUILDER="${TDSHIM_CONTAINER_BUILDER:-}" \
 	--env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \
 	--env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \
+	--env USE_CACHE="${USE_CACHE:-}" \
 	--rm \
 	-w ${script_dir} \
 	build-kata-deploy "${kata_deploy_create}" $@

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -40,6 +40,7 @@ readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts"
 
 ARCH=$(uname -m)
 MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
+USE_CACHE="${USE_CACHE:-"yes"}"
 
 workdir="${WORKDIR:-$PWD}"
 
@@ -79,6 +80,7 @@ options:
 --build=<asset>       :
 	all
 	cloud-hypervisor
+	cloud-hypervisor-glibc
 	firecracker
 	kernel
 	kernel-dragonball-experimental
@@ -97,6 +99,7 @@ options:
 	rootfs-image
 	rootfs-image-tdx
 	rootfs-initrd
+	rootfs-initrd-mariner
 	rootfs-initrd-sev
 	shim-v2
 	tdvf
@@ -113,6 +116,10 @@ cleanup_and_fail() {
 }
 
 install_cached_tarball_component() {
+	if [ "${USE_CACHE}" != "yes" ]; then
+		return 1
+	fi
+
 	local component="${1}"
 	local jenkins_build_url="${2}"
 	local current_version="${3}"
@@ -136,8 +143,13 @@ install_cached_tarball_component() {
 
 #Install guest image
 install_image() {
-	local image_type="${1:-"image"}"
+	local variant="${1:-}"
-	local initrd_suffix="${2:-""}"
+
+	image_type="image"
+	if [ -n "${variant}" ]; then
+		image_type+="-${variant}"
+	fi
+
 	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}"
 	local component="rootfs-${image_type}"
 
@@ -152,25 +164,39 @@ install_image() {
 	install_cached_tarball_component \
 		"${component}" \
 		"${jenkins}" \
-		"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-image" \
+		"${osbuilder_last_commit}-${guest_image_last_commit}-${agent_last_commit}-${libs_last_commit}-${gperf_version}-${libseccomp_version}-${rust_version}-${image_type}" \
 		"" \
 		"${final_tarball_name}" \
 		"${final_tarball_path}" \
 		&& return 0
 
 	info "Create image"
-	"${rootfs_builder}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
+
+	if [ -n "${variant}" ]; then
+		os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.name")"
+		os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.${variant}.version")"
+	else
+		os_name="$(get_from_kata_deps "assets.image.architecture.${ARCH}.name")"
+		os_version="$(get_from_kata_deps "assets.image.architecture.${ARCH}.version")"
+	fi
+	
+	"${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=image --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}"
 }
 
 #Install guest image for tdx
 install_image_tdx() {
-	install_image "image-tdx" "tdx"
+	install_image "tdx"
 }
 
 #Install guest initrd
 install_initrd() {
-	local initrd_type="${1:-"initrd"}"
+	local variant="${1:-}"
-	local initrd_suffix="${2:-""}"
+
+	initrd_type="initrd"
+	if [ -n "${variant}" ]; then
+		initrd_type+="-${variant}"
+	fi
+
 	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}"
 	local component="rootfs-${initrd_type}"
 
@@ -192,12 +218,26 @@ install_initrd() {
 		&& return 0
 
 	info "Create initrd"
-	"${rootfs_builder}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${initrd_suffix}"
+
+	if [ -n "${variant}" ]; then
+		os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.name")"
+		os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.${variant}.version")"
+	else
+		os_name="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.name")"
+		os_version="$(get_from_kata_deps "assets.initrd.architecture.${ARCH}.version")"
+	fi
+
+	"${rootfs_builder}" --osname="${os_name}" --osversion="${os_version}" --imagetype=initrd --prefix="${prefix}" --destdir="${destdir}" --image_initrd_suffix="${variant}"
+}
+
+#Install Mariner guest initrd
+install_initrd_mariner() {
+	install_initrd "cbl-mariner"
 }
 
 #Install guest initrd for sev
 install_initrd_sev() {
-	install_initrd "initrd-sev" "sev"
+	install_initrd "sev"
 }
 
 #Install kernel component helper
@@ -413,26 +453,47 @@ install_firecracker() {
 	sudo install -D --owner root --group root --mode 0744 release-${firecracker_version}-${ARCH}/jailer-${firecracker_version}-${ARCH} "${destdir}/opt/kata/bin/jailer"
 }
 
-# Install static cloud-hypervisor asset
+install_clh_helper() {
-install_clh() {
+	libc="${1}"
+	features="${2}"
+	suffix="${3:-""}"
+
 	install_cached_tarball_component \
-		"cloud-hypervisor" \
+		"cloud-hypervisor${suffix}" \
-		"${jenkins_url}/job/kata-containers-main-clh-$(uname -m)/${cached_artifacts_path}" \
+		"${jenkins_url}/job/kata-containers-main-clh-$(uname -m)${suffix}/${cached_artifacts_path}" \
 		"$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version")" \
 		"" \
 		"${final_tarball_name}" \
 		"${final_tarball_path}" \
 		&& return 0
 
-	if [[ "${ARCH}" == "x86_64" ]]; then
-		export features="tdx"
-	fi
-
 	info "build static cloud-hypervisor"
-	"${clh_builder}"
+	libc="${libc}" features="${features}" "${clh_builder}"
 	info "Install static cloud-hypervisor"
 	mkdir -p "${destdir}/opt/kata/bin/"
-	sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor"
+	sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor${suffix}"
+}
+
+# Install static cloud-hypervisor asset
+install_clh() {
+	if [[ "${ARCH}" == "x86_64" ]]; then
+		features="mshv,tdx"
+	else
+		features=""
+	fi
+
+	install_clh_helper "musl" "${features}"
+}
+
+# Install static cloud-hypervisor-glibc asset
+install_clh_glibc() {
+	if [[ "${ARCH}" == "x86_64" ]]; then
+		features="mshv"
+	else
+		features=""
+	fi
+
+	install_clh_helper "gnu" "${features}" "-glibc"
 }
 
 # Install static virtiofsd asset
@@ -561,6 +622,7 @@ handle_build() {
 		install_firecracker
 		install_image
 		install_initrd
+		install_initrd_mariner
 		install_initrd_sev
 		install_kernel
 		install_kernel_dragonball_experimental
@@ -578,7 +640,7 @@ handle_build() {
 
 	cloud-hypervisor) install_clh ;;
 
-	cloud-hypervisor-glibc) ;;
+	cloud-hypervisor-glibc) install_clh_glibc ;;
 
 	firecracker) install_firecracker ;;
 
@@ -616,7 +678,7 @@ handle_build() {
 
 	rootfs-initrd) install_initrd ;;
 
-	rootfs-initrd-mariner) ;;
+	rootfs-initrd-mariner) install_initrd_mariner ;;
 
 	rootfs-initrd-sev) install_initrd_sev ;;
 	
@@ -662,6 +724,7 @@ main() {
 		qemu
 		rootfs-image
 		rootfs-initrd
+		rootfs-initrd-mariner
 		shim-v2
 		virtiofsd
 	)

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -64,6 +64,15 @@ function install_artifacts() {
 	chmod +x /opt/kata/bin/*
 	[ -d /opt/kata/runtime-rs/bin ] && \
 		chmod +x /opt/kata/runtime-rs/bin/*
+
+	# Allow Mariner to use custom configuration.
+	if [ "${HOST_OS:-}" == "cbl-mariner" ]; then
+		config_path="/opt/kata/share/defaults/kata-containers/configuration-clh.toml"
+		clh_path="/opt/kata/bin/cloud-hypervisor-glibc"
+		sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' "${config_path}"
+		sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}"
+		sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}"
+	fi
 }
 
 function wait_till_node_is_ready() {

tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh

@@ -76,12 +76,12 @@ build_clh_from_source() {
 
 	if [ -n "${features}" ]; then
 		info "Build cloud-hypervisor enabling the following features: ${features}"
-		./scripts/dev_cli.sh build --release --libc musl --features "${features}"
+		./scripts/dev_cli.sh build --release --libc "${libc}" --features "${features}"
 	else
-		./scripts/dev_cli.sh build --release --libc musl
+		./scripts/dev_cli.sh build --release --libc "${libc}"
 	fi
 	rm -f cloud-hypervisor
-	cp build/cargo_target/$(uname -m)-unknown-linux-musl/release/cloud-hypervisor .
+	cp build/cargo_target/$(uname -m)-unknown-linux-${libc}/release/cloud-hypervisor .
 	popd
 }
 

versions.yaml

@@ -122,17 +122,20 @@ assets:
     url: "https://github.com/kata-containers/kata-containers/tools/osbuilder"
     architecture:
       aarch64:
-        name: "ubuntu"
-        version: "latest"
-      ppc64le:
-        name: "ubuntu"
-        version: "latest"
-      s390x:
-        name: "ubuntu"
-        version: "latest"
-      x86_64:
         name: &default-image-name "ubuntu"
-        version: "latest"
+        version: &default-image-version "latest"
+      ppc64le:
+        name: *default-image-name
+        version: *default-image-version
+      s390x:
+        name: *default-image-name
+        version: *default-image-version
+      x86_64:
+        name: *default-image-name
+        version: *default-image-version
+        tdx:
+          name: *default-image-name
+          version: *default-image-version
     meta:
       image-type: *default-image-name
 
@@ -156,6 +159,9 @@ assets:
       x86_64:
         name: *default-initrd-name
         version: *default-initrd-version
+        cbl-mariner:
+          name: "cbl-mariner"
+          version: "2.0"
         sev:
           name: *glibc-initrd-name
           version: *glibc-initrd-version