ci: Use helm to deploy kata-deploy

Rather then modifying the kata-depoy scripts let's use Helm and
create a values.yaml that can be used to render the final templates

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

tests/common.bash

@@ -529,6 +529,29 @@ function ensure_yq() {
 	hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
 }
 
+function ensure_helm() {
+	ensure_yq
+	# The get-helm-3 script will take care of downloaading and installing Helm
+	# properly on the system respecting ARCH, OS and other configurations.
+	DESIRED_VERSION=$(get_from_kata_deps ".externals.helm.version")
+	export DESIRED_VERSION
+
+	# Check if helm is available in the system's PATH
+	if ! command -v helm &> /dev/null; then
+		echo "Helm is not installed. Installing Helm..."
+		curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+		# Verify the installation
+		if command -v helm &> /dev/null; then
+			echo "Helm installed successfully."
+		else
+			echo "Failed to install Helm."
+			exit 1
+		fi
+	else
+		echo "Helm is already installed."
+	fi
+}
+
 # dependency: What we want to get the version from the versions.yaml file
 function get_from_kata_deps() {
         versions_file="${repo_root_dir}/versions.yaml"

tests/integration/kubernetes/gha-run.sh

@@ -17,6 +17,7 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
 source "${kubernetes_dir}/confidential_kbs.sh"
 # shellcheck disable=2154
 tools_dir="${repo_root_dir}/tools"
+helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy"
 kata_tarball_dir="${2:-kata-artifacts}"
 
 DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io}
@@ -146,6 +147,7 @@ function deploy_coco_kbs() {
 
 function deploy_kata() {
 	platform="${1}"
+	ensure_helm
 	ensure_yq
 
 	[ "$platform" = "kcli" ] && \
@@ -157,82 +159,59 @@ function deploy_kata() {
 
 	set_default_cluster_namespace
 
-	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	local values_yaml
+	values_yaml=$(mktemp /tmp/values_yaml.XXXXXX)
 
-	# Enable debug for Kata Containers
+	yq -i ".k8sDistribution = \"${KUBERNETES}\""                     "${values_yaml}"
-	yq -i \
+	yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}"
-	  '.spec.template.spec.containers[0].env[1].value = "true"' \
+	yq -i ".image.tag = \"${DOCKER_TAG}\""                           "${values_yaml}"
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	yq -i ".env.debug = \"true\""                                    "${values_yaml}"
-	# Create the runtime class only for the shim that's being tested
+	yq -i ".env.shims = \"${KATA_HYPERVISOR}\""                      "${values_yaml}"
-	yq -i \
+	yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\""                "${values_yaml}"
-	  ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
+	yq -i ".env.createRuntimeClasses = \"true\""                     "${values_yaml}"
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	yq -i ".env.createDefaultRuntimeClass = \"true\""                "${values_yaml}"
-	# Set the tested hypervisor as the default `kata` shim
+	yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\""    "${values_yaml}"
-	yq -i \
+	yq -i ".env.snapshotterHandlerMapping = \"\""                    "${values_yaml}"
-	  ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
+	yq -i ".env.agentHttpsProxy = \"\""                              "${values_yaml}"
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	yq -i ".env.agentNoProxy = \"\""                                 "${values_yaml}"
-	# Let the `kata-deploy` script take care of the runtime class creation / removal
+	yq -i ".env.pullTypeMapping = \"\""                              "${values_yaml}"
-	yq -i \
+	yq -i ".env.hostOS = \"\""                                       "${values_yaml}"
-	  '.spec.template.spec.containers[0].env[4].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Let the `kata-deploy` create the default `kata` runtime class
-	yq -i \
-	  '.spec.template.spec.containers[0].env[5].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Enable 'default_vcpus' hypervisor annotation
-	yq -i \
-	  '.spec.template.spec.containers[0].env[6].value = "default_vcpus"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
 	if [ -n "${SNAPSHOTTER}" ]; then
-		yq -i \
+		yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
-		  ".spec.template.spec.containers[0].env[7].value = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	fi
 
 	if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
-		ALLOWED_HYPERVISOR_ANNOTATIONS="initrd kernel default_vcpus"
+		yq -i ".env.allowedHypervisorAnnotations = \"initrd kernel default_vcpus\"" "${values_yaml}"
-		HOST_OS=${KATA_HOST_OS}
+		yq -i ".env.hostOS = \"${KATA_HOST_OS}\""                                   "${values_yaml}"
 	fi
 
 	if [ "${KATA_HYPERVISOR}" = "qemu" ]; then
-		yq -i \
+		yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
-		  '.spec.template.spec.containers[0].env[6].value = "image initrd kernel default_vcpus"' \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	fi
 
 	if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
-		yq -i \
+		yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
-		  ".spec.template.spec.containers[0].env[8].value = \"${HTTPS_PROXY}\"" \
+		yq -i ".env.agentNoProxy = \"${NO_PROXY}\""       "${values_yaml}"
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-
-		yq -i \
-		  ".spec.template.spec.containers[0].env[9].value = \"${NO_PROXY}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	fi
 
 	# Set the PULL_TYPE_MAPPING
 	if [ "${PULL_TYPE}" != "default" ]; then
-		yq -i \
+		yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
-		  ".spec.template.spec.containers[0].env[10].value = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	fi
 
-	echo "::group::Final kata-deploy.yaml that is used in the test"
+	echo "::group::Final kata-deploy manifests used in the test"
-	cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	cat "${values_yaml}"
-	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
+	helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
+	[ "$(yq .image.reference ${values_yaml})" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ] || die "Failed to set image reference"
+	[ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
 	echo "::endgroup::"
 
-	kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+	# will wait until all Pods, PVCs, Services, and minimum number of Pods
-	case "${KUBERNETES}" in
+	# of a Deployment, StatefulSet, or ReplicaSet are in a ready state
-		k0s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;;
+	# before marking the release as successful. It will wait for as long
-		k3s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;;
+	# as --timeout -- Ready >> Running
-		rke2) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;;
+	helm install --wait --timeout 10m kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
-		*) kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	esac
-
-	local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
-	waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
 
 	# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
 	# which may cause issues like not having the node properly labeled or the artefacts

tools/packaging/kata-deploy/helm-chart/Makefile

@@ -16,18 +16,3 @@ package: helm release
 
 clean:
 	rm kata-deploy-*.tgz
-	
-# go-get-tool will 'go get' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
-@[ -f $(PROJECT_DIR)/bin/$(1) ] || command -v $(1) >/dev/null 2>&1 || { \
-set -e ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
-
-HELM = $(shell pwd)/bin/helm
-helm: ## Download helm locally if necessary.
-	$(call go-get-tool,$(HELM),helm.sh/helm/v3/cmd/helm@latest)

tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl

@@ -7,14 +7,14 @@
 Set the correct containerd conf path depending on the k8s distribution
 */}}
 {{- define "containerdConfPath" -}}
-{{- if eq .k8sDistribution "k8s" -}}
+{{- if eq .k8sDistribution "rke2" -}}
-/etc/containerd/
-{{- else if eq .k8sDistribution "rke2" -}}
 /var/lib/rancher/rke2/agent/etc/containerd/
 {{- else if eq .k8sDistribution "k3s" -}}
  /var/lib/rancher/k3s/agent/etc/containerd/
 {{- else if eq .k8sDistribution "k0s" -}}
 /etc/k0s/containerd.d/
+{{- else -}}
+/etc/containerd/
 {{- end -}}
 {{- end -}}
 

versions.yaml

@@ -218,6 +218,11 @@ externals:
     version: "1.36.1"
     url: "https://busybox.net/downloads"
 
+  helm:
+    description: "Kubernetes package manager"
+    url: "https://get.helm.sh/"
+    version: "v3.15.2"
+
   cni-plugins:
     description: "CNI network plugins"
     url: "https://github.com/containernetworking/plugins"