From 424347bf0e6810b8a06445e5f3193aaa98d41879 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano@fidencio.org>
Date: Wed, 21 Aug 2024 11:44:27 +0200
Subject: [PATCH 1/7] Reapply "kata-deploy: Add Helm Chart" (partially)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit b18c3dfce385a9fd33c028f6ee7f9d36108aab5f, as the
pacthes for helm are coming as part of this series.

Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
---
 tests/integration/kubernetes/gha-run.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index c8d165b0dc..0e3c976c92 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -191,12 +191,8 @@ function deploy_kata() {
 	fi
 
 	if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
-		yq -i \
-		  '.spec.template.spec.containers[0].env[6].value = "initrd kernel default_vcpus"' \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-		yq -i \
-		  ".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+		ALLOWED_HYPERVISOR_ANNOTATIONS="initrd kernel default_vcpus"
+		HOST_OS=${KATA_HOST_OS}
 	fi
 
 	if [ "${KATA_HYPERVISOR}" = "qemu" ]; then

From cfe6e4ae7152802b4ee015e696db63bf892c8b7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano@fidencio.org>
Date: Wed, 21 Aug 2024 11:45:17 +0200
Subject: [PATCH 2/7] Reapply "ci: Use helm to deploy kata-deploy" (partially)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 36f4038a89a7157f609375de4c0ff81d5fbdfe26, as the
pacthes for helm are coming as part of this series.

Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
---
 tests/common.bash                       | 23 +++++++
 tests/integration/kubernetes/gha-run.sh | 91 ++++++++++---------------
 versions.yaml                           |  5 ++
 3 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/tests/common.bash b/tests/common.bash
index ba44fd89cd..8084f210c1 100644
--- a/tests/common.bash
+++ b/tests/common.bash
@@ -529,6 +529,29 @@ function ensure_yq() {
 	hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
 }
 
+function ensure_helm() {
+	ensure_yq
+	# The get-helm-3 script will take care of downloaading and installing Helm
+	# properly on the system respecting ARCH, OS and other configurations.
+	DESIRED_VERSION=$(get_from_kata_deps ".externals.helm.version")
+	export DESIRED_VERSION
+
+	# Check if helm is available in the system's PATH
+	if ! command -v helm &> /dev/null; then
+		echo "Helm is not installed. Installing Helm..."
+		curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+		# Verify the installation
+		if command -v helm &> /dev/null; then
+			echo "Helm installed successfully."
+		else
+			echo "Failed to install Helm."
+			exit 1
+		fi
+	else
+		echo "Helm is already installed."
+	fi
+}
+
 # dependency: What we want to get the version from the versions.yaml file
 function get_from_kata_deps() {
         versions_file="${repo_root_dir}/versions.yaml"
diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index 0e3c976c92..096b4a92b6 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -17,6 +17,7 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
 source "${kubernetes_dir}/confidential_kbs.sh"
 # shellcheck disable=2154
 tools_dir="${repo_root_dir}/tools"
+helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy"
 kata_tarball_dir="${2:-kata-artifacts}"
 
 DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io}
@@ -146,6 +147,7 @@ function deploy_coco_kbs() {
 
 function deploy_kata() {
 	platform="${1:-}"
+	ensure_helm
 	ensure_yq
 
 	[ "$platform" = "kcli" ] && \
@@ -157,82 +159,59 @@ function deploy_kata() {
 
 	set_default_cluster_namespace
 
-	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	local values_yaml
+	values_yaml=$(mktemp /tmp/values_yaml.XXXXXX)
 
-	# Enable debug for Kata Containers
-	yq -i \
-	  '.spec.template.spec.containers[0].env[1].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Create the runtime class only for the shim that's being tested
-	yq -i \
-	  ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Set the tested hypervisor as the default `kata` shim
-	yq -i \
-	  ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Let the `kata-deploy` script take care of the runtime class creation / removal
-	yq -i \
-	  '.spec.template.spec.containers[0].env[4].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Let the `kata-deploy` create the default `kata` runtime class
-	yq -i \
-	  '.spec.template.spec.containers[0].env[5].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	# Enable 'default_vcpus' hypervisor annotation
-	yq -i \
-	  '.spec.template.spec.containers[0].env[6].value = "default_vcpus"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	yq -i ".k8sDistribution = \"${KUBERNETES}\""                     "${values_yaml}"
+	yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}"
+	yq -i ".image.tag = \"${DOCKER_TAG}\""                           "${values_yaml}"
+	yq -i ".env.debug = \"true\""                                    "${values_yaml}"
+	yq -i ".env.shims = \"${KATA_HYPERVISOR}\""                      "${values_yaml}"
+	yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\""                "${values_yaml}"
+	yq -i ".env.createRuntimeClasses = \"true\""                     "${values_yaml}"
+	yq -i ".env.createDefaultRuntimeClass = \"true\""                "${values_yaml}"
+	yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\""    "${values_yaml}"
+	yq -i ".env.snapshotterHandlerMapping = \"\""                    "${values_yaml}"
+	yq -i ".env.agentHttpsProxy = \"\""                              "${values_yaml}"
+	yq -i ".env.agentNoProxy = \"\""                                 "${values_yaml}"
+	yq -i ".env.pullTypeMapping = \"\""                              "${values_yaml}"
+	yq -i ".env.hostOS = \"\""                                       "${values_yaml}"
 
 	if [ -n "${SNAPSHOTTER}" ]; then
-		yq -i \
-		  ".spec.template.spec.containers[0].env[7].value = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+		yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
 	fi
 
 	if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
-		ALLOWED_HYPERVISOR_ANNOTATIONS="initrd kernel default_vcpus"
-		HOST_OS=${KATA_HOST_OS}
+		yq -i ".env.allowedHypervisorAnnotations = \"initrd kernel default_vcpus\"" "${values_yaml}"
+		yq -i ".env.hostOS = \"${KATA_HOST_OS}\""                                   "${values_yaml}"
 	fi
 
 	if [ "${KATA_HYPERVISOR}" = "qemu" ]; then
-		yq -i \
-		  '.spec.template.spec.containers[0].env[6].value = "image initrd kernel default_vcpus"' \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+		yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
 	fi
 
 	if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
-		yq -i \
-		  ".spec.template.spec.containers[0].env[8].value = \"${HTTPS_PROXY}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-
-		yq -i \
-		  ".spec.template.spec.containers[0].env[9].value = \"${NO_PROXY}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+		yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
+		yq -i ".env.agentNoProxy = \"${NO_PROXY}\""       "${values_yaml}"
 	fi
 
 	# Set the PULL_TYPE_MAPPING
 	if [ "${PULL_TYPE}" != "default" ]; then
-		yq -i \
-		  ".spec.template.spec.containers[0].env[10].value = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" \
-		  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+		yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
 	fi
 
-	echo "::group::Final kata-deploy.yaml that is used in the test"
-	cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
+	echo "::group::Final kata-deploy manifests used in the test"
+	cat "${values_yaml}"
+	helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
+	[ "$(yq .image.reference ${values_yaml})" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ] || die "Failed to set image reference"
+	[ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
 	echo "::endgroup::"
 
-	kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
-	case "${KUBERNETES}" in
-		k0s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;;
-		k3s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;;
-		rke2) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;;
-		*) kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-	esac
-
-	local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
-	waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
+	# will wait until all Pods, PVCs, Services, and minimum number of Pods
+	# of a Deployment, StatefulSet, or ReplicaSet are in a ready state
+	# before marking the release as successful. It will wait for as long
+	# as --timeout -- Ready >> Running
+	helm install --wait --timeout 10m kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
 
 	# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
 	# which may cause issues like not having the node properly labeled or the artefacts
diff --git a/versions.yaml b/versions.yaml
index c2265d830e..94b606a0dd 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -218,6 +218,11 @@ externals:
     version: "1.36.1"
     url: "https://busybox.net/downloads"
 
+  helm:
+    description: "Kubernetes package manager"
+    url: "https://get.helm.sh/"
+    version: "v3.15.2"
+
   cni-plugins:
     description: "CNI network plugins"
     url: "https://github.com/containernetworking/plugins"

From 40f8aae6dbe80a480b0535a29428f40088f496b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano@fidencio.org>
Date: Wed, 21 Aug 2024 11:45:40 +0200
Subject: [PATCH 3/7] Reapply "ci: make cleanup_kata_deploy really simple"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 21f9f01e1df325b2ecf074016aa37da46778716c, as the
pacthes for helm are coming as part of this series.

Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
---
 tests/integration/kubernetes/gha-run.sh | 55 ++-----------------------
 1 file changed, 4 insertions(+), 51 deletions(-)

diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index 096b4a92b6..cfbdebb585 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -381,58 +381,11 @@ function collect_artifacts() {
 }
 
 function cleanup_kata_deploy() {
-	ensure_yq
+	ensure_helm
 
-	case "${KUBERNETES}" in
-		k0s)
-			deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s""
-			cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k0s""
-			;;
-		k3s)
-			deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
-			cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s""
-			;;
-		rke2)
-			deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2""
-			cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/rke2""
-			;;
-		*)
-			deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
-			cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
-			;;
-	esac
-
-	# shellcheck disable=2086
-	kubectl_retry delete --ignore-not-found ${deploy_spec}
-	kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-	# Let the `kata-deploy` script take care of the runtime class creation / removal
-	yq -i \
-	  '.spec.template.spec.containers[0].env[4].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
-	# Create the runtime class only for the shim that's being tested
-	yq -i \
-	  ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
-	  "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
-	# Set the tested hypervisor as the default `kata` shim
-	yq -i \
-	  ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
-	  "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
-	# Let the `kata-deploy` create the default `kata` runtime class
-	yq -i \
-	  '.spec.template.spec.containers[0].env[5].value = "true"' \
-	  "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-
-	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
-	cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
-	grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
-	# shellcheck disable=2086
-	kubectl_retry apply ${cleanup_spec}
-	sleep 180s
-
-	# shellcheck disable=2086
-	kubectl_retry delete --ignore-not-found ${cleanup_spec}
-	kubectl_retry delete --ignore-not-found -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+	# Do not return after deleting only the parent object cascade=foreground
+	# means also wait for child/dependent object deletion
+	helm uninstall kata-deploy --ignore-not-found --wait --cascade foreground --timeout 10m --namespace kube-system
 }
 
 function cleanup() {

From 3b23d626353965162326597d4422bcb3360aaf2f Mon Sep 17 00:00:00 2001
From: Wainer dos Santos Moschetta <wainersm@redhat.com>
Date: Tue, 20 Aug 2024 12:13:42 -0300
Subject: [PATCH 4/7] tests/k8s: fix wait for pods on deploy-kata action

On commit 51690bc157b we switched the installation from kubectl to helm
and used its `--wait` expecting the execution would continue when all
kata-deploy Pods were Ready. It turns out that there is a limitation on
helm install that won't wait properly when the daemonset is made of a
single replica and maxUnavailable=1. In order to fix that issue, let's
revert the changes partially to keep using kubectl and waitForProcess
to the exection while Pods aren't Running.

Fixes #10168
Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>
---
 tests/integration/kubernetes/gha-run.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index cfbdebb585..e8a124af14 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -207,11 +207,12 @@ function deploy_kata() {
 	[ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
 	echo "::endgroup::"
 
-	# will wait until all Pods, PVCs, Services, and minimum number of Pods
-	# of a Deployment, StatefulSet, or ReplicaSet are in a ready state
-	# before marking the release as successful. It will wait for as long
-	# as --timeout -- Ready >> Running
-	helm install --wait --timeout 10m kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
+	helm install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
+
+	# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
+	# like kata-deploy on CI. So wait for pods being Running in the "tradicional" way.
+	local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
+	waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
 
 	# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
 	# which may cause issues like not having the node properly labeled or the artefacts

From 64ccb1645d9f0b48d89d8af7aea62cb75491aff9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano.fidencio@intel.com>
Date: Tue, 20 Aug 2024 10:03:10 +0200
Subject: [PATCH 5/7] helm: Add a post-delete hook
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Instead of using a lifecycle.preStop hook, as done when we're using
using the helm chat, let's add a post-delete hook to take care of
properly cleaning up the node during when uninstalling kata-deploy.

The reason why the lifecyle.preStop hook would never work on our case is
simply because each helm chart operation follows the Kuberentes
"declarative" approach, meaning that an operation won't wait for its
previous operation to successfully finish before being called, leading
to us trying to access content that's defined by our RBAC, in an
operation that was started before our RBAC was deleted, but having the
RBAC being deleted before the operation actually started.

Unfortunately this hook brings in some code duplicatioon, mainly related
to the RBAC parts, but that's not new as the same happens with our
deamonset.

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
---
 .../kata-deploy/templates/kata-deploy.yaml    |   4 -
 .../templates/post-delete-job.yaml            | 116 ++++++++++++++++++
 .../kata-deploy/scripts/kata-deploy.sh        |  30 ++++-
 3 files changed, 145 insertions(+), 5 deletions(-)
 create mode 100644 tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml

diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml
index 5d339e47b5..0416501444 100644
--- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml
+++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml
@@ -22,10 +22,6 @@ spec:
       - name: kube-kata
         image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
-        lifecycle:
-          preStop:
-            exec:
-              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
         command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install"]
         env:
         - name: NODE_NAME
diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml
new file mode 100644
index 0000000000..c49b349279
--- /dev/null
+++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/post-delete-job.yaml
@@ -0,0 +1,116 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Chart.Name }}-sa-cleanup
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Chart.Name }}-role-cleanup
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-2"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+- apiGroups: ["node.k8s.io"]
+  resources: ["runtimeclasses"]
+  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Chart.Name }}-rb-cleanup
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Chart.Name }}-role-cleanup
+subjects:
+- kind: ServiceAccount
+  name: {{ .Chart.Name }}-sa-cleanup
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Chart.Name }}-cleanup
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        role: cleanup
+    spec:
+      serviceAccountName: {{ .Chart.Name }}-sa-cleanup
+      hostPID: true
+      containers:
+      - name: kube-kata-cleanup
+        image: {{ .Values.image.reference }}:{{ default .Chart.AppVersion .Values.image.tag }}
+        imagePullPolicy: IfNotPresent
+        command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: DEBUG
+          value: {{ .Values.env.debug | quote }}
+        - name: SHIMS
+          value: {{ .Values.env.shims | quote }}
+        - name: DEFAULT_SHIM
+          value: {{ .Values.env.defaultShim | quote }}
+        - name: CREATE_RUNTIMECLASSES
+          value: {{ .Values.env.createRuntimeClasses | quote }}
+        - name: CREATE_DEFAULT_RUNTIMECLASS
+          value: {{ .Values.env.createDefaultRuntimeClass | quote }}
+        - name: ALLOWED_HYPERVISOR_ANNOTATIONS
+          value: {{ .Values.env.allowedHypervisorAnnotations | quote }}
+        - name: SNAPSHOTTER_HANDLER_MAPPING
+          value: {{ .Values.env.snapshotterHandlerMapping | quote }}
+        - name: AGENT_HTTPS_PROXY
+          value: {{ .Values.env.agentHttpsProxy | quote }}
+        - name: AGENT_NO_PROXY
+          value: {{ .Values.env.agentNoProxy | quote }}
+        - name: PULL_TYPE_MAPPING
+          value: {{ .Values.env.pullTypeMapping | quote }}
+        - name: HELM_POST_DELETE_HOOK
+          value: "true"
+{{- with .Values.env.hostOS }}
+        - name: HOST_OS
+          value: {{ . | quote }}
+{{- end }}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: crio-conf
+          mountPath: /etc/crio/
+        - name: containerd-conf
+          mountPath: /etc/containerd/
+        - name: host
+          mountPath: /host/
+      volumes:
+      - name: crio-conf
+        hostPath:
+          path: /etc/crio/
+      - name: containerd-conf
+        hostPath:
+          path: '{{- template "containerdConfPath" .Values }}'
+      - name: host
+        hostPath:
+          path: /
+      restartPolicy: Never 
diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 03e602b1a0..0780261d85 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -48,6 +48,8 @@ fi
 # doubled here as well, as: `/host//opt/kata`
 host_install_dir="/host${dest_dir}"
 
+HELM_POST_DELETE_HOOK="${HELM_POST_DELETE_HOOK:-"false"}"
+
 # If we fail for any reason a message will be displayed
 die() {
         msg="$*"
@@ -560,6 +562,16 @@ function cleanup_cri_runtime() {
 		;;
 	esac
 
+	[ "${HELM_POST_DELETE_HOOK}" == "false" ] && return
+
+	# Only run this code in the HELM_POST_DELETE_HOOK
+	if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
+		# do nothing, k0s will automatically unload the config on the fly
+		:
+	else
+		host_systemctl daemon-reload
+		host_systemctl restart "$1"
+	fi
 }
 
 function cleanup_crio() {
@@ -659,6 +671,7 @@ function main() {
 	echo "* AGENT_NO_PROXY: ${AGENT_NO_PROXY}"
 	echo "* PULL_TYPE_MAPPING: ${PULL_TYPE_MAPPING}"
 	echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}"
+	echo "* HELM_POST_DELETE_HOOK: ${HELM_POST_DELETE_HOOK}"
 
 	# script requires that user is root
 	euid=$(id -u)
@@ -716,9 +729,24 @@ function main() {
 			       containerd_conf_file="${containerd_conf_tmpl_file}"
 			fi
 
+			if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then
+				# Remove the label as the first thing, so we ensure no more kata-containers
+				# pods would be scheduled here.
+				kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime-
+			fi
+
 			cleanup_cri_runtime "$runtime"
-			kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup
+			if [ "${HELM_POST_DELETE_HOOK}" == "false" ]; then
+				# The Confidential Containers operator relies on this label
+				kubectl label node "$NODE_NAME" --overwrite katacontainers.io/kata-runtime=cleanup
+			fi
 			remove_artifacts
+
+			if [ "${HELM_POST_DELETE_HOOK}" == "true" ]; then
+				# After everything was cleaned up, there's no reason to continue
+				# and sleep forever.  Let's just return success..
+				exit 0
+			fi
 			;;
 		reset)
 			reset_runtime $runtime

From a7737975945fc1d1e5d619dd65ae1260b9329ef1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano@fidencio.org>
Date: Tue, 20 Aug 2024 20:31:31 +0200
Subject: [PATCH 6/7] ci: Pass --debug to helm
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Just to make ourlives a little bit easier.

Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
---
 tests/integration/kubernetes/gha-run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index e8a124af14..9f359db874 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -207,7 +207,7 @@ function deploy_kata() {
 	[ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
 	echo "::endgroup::"
 
-	helm install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
+	helm install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug
 
 	# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
 	# like kata-deploy on CI. So wait for pods being Running in the "tradicional" way.
@@ -386,7 +386,7 @@ function cleanup_kata_deploy() {
 
 	# Do not return after deleting only the parent object cascade=foreground
 	# means also wait for child/dependent object deletion
-	helm uninstall kata-deploy --ignore-not-found --wait --cascade foreground --timeout 10m --namespace kube-system
+	helm uninstall kata-deploy --ignore-not-found --wait --cascade foreground --timeout 10m --namespace kube-system --debug
 }
 
 function cleanup() {

From bb9bcd886acd9677affb401cd0a0252475c0531e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <fabiano@fidencio.org>
Date: Tue, 3 Sep 2024 09:52:25 +0200
Subject: [PATCH 7/7] kata-deploy: Add reset_cri_runtime()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This will help to avoid code duplication on what's needed on the helm
and non-helm cases.

The reason it's not been added as part of the commit which adds the
post-delete hook is simply for helping the reviewer (as the diff would
be less readable with this change).

Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
---
 .../kata-deploy/scripts/kata-deploy.sh        | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
index 0780261d85..79bfab26fd 100755
--- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh
+++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh
@@ -552,6 +552,18 @@ function remove_artifacts() {
 	fi
 }
 
+function restart_cri_runtime() {
+	local runtime="${1}"
+
+	if [ "${runtime}" == "k0s-worker" ] || [ "${runtime}" == "k0s-controller" ]; then
+		# do nothing, k0s will automatically unload the config on the fly
+		:
+	else
+		host_systemctl daemon-reload
+		host_systemctl restart "${runtime}"
+	fi
+}
+
 function cleanup_cri_runtime() {
 	case $1 in
 	crio)
@@ -565,13 +577,7 @@ function cleanup_cri_runtime() {
 	[ "${HELM_POST_DELETE_HOOK}" == "false" ] && return
 
 	# Only run this code in the HELM_POST_DELETE_HOOK
-	if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
-		# do nothing, k0s will automatically unload the config on the fly
-		:
-	else
-		host_systemctl daemon-reload
-		host_systemctl restart "$1"
-	fi
+	restart_cri_runtime "$1"
 }
 
 function cleanup_crio() {
@@ -590,13 +596,7 @@ function cleanup_containerd() {
 
 function reset_runtime() {
 	kubectl label node "$NODE_NAME" katacontainers.io/kata-runtime-
-	if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then
-		# do nothing, k0s will auto restart
-		:
-	else
-		host_systemctl daemon-reload
-		host_systemctl restart "$1"
-	fi
+	restart_cri_runtime "$1"
 
 	if [ "$1" == "crio" ] || [ "$1" == "containerd" ]; then
 		host_systemctl restart kubelet