From 7053d70acfbc7b7ec313de05a51caca1543ecc08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <ffidencio@nvidia.com>
Date: Tue, 7 Jul 2026 12:19:50 +0200
Subject: [PATCH] tests: retry pod removal before forcing in
 TestKilledVmmCleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

TestKilledVmmCleanup SIGKILLs the VMM and then removes the pod.  After a
hard VMM kill the shim exits promptly (which the test asserts), so the
container's TaskExit event may not have reached containerd before the
shim was gone, leaving CRI's view of the container as "running" and
making a plain `crictl rmp` fail with "container is still running, to
stop first".

Retry the normal removal a few times to give the event time to arrive,
and only fall back to `crictl rmp -f` if it keeps failing -- which is how
a crashed sandbox is recovered in practice (kubelet GC / manual force
removal).  The essential guarantee, that no shim process is leaked after
the VMM is killed, is unchanged.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
---
 .../cri-containerd/integration-tests.sh       | 43 ++++++++++++++++++-
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/tests/integration/cri-containerd/integration-tests.sh b/tests/integration/cri-containerd/integration-tests.sh
index e51778547a..c35cd945ad 100755
--- a/tests/integration/cri-containerd/integration-tests.sh
+++ b/tests/integration/cri-containerd/integration-tests.sh
@@ -339,12 +339,47 @@ EOF
 }
 
 function testContainerStop() {
+	# When the caller passes "force", remove the pod with `crictl rmp -f`.
+	# This is needed for the killed-VMM case: after the hypervisor is
+	# SIGKILL'd the shim exits promptly (as the test itself asserts), so the
+	# container's TaskExit event may not have reached containerd before the
+	# shim was gone, leaving CRI's view of the container as "running".  A
+	# crashed VMM is recovered with a force removal (what kubelet's GC and
+	# manual `crictl rmp -f` do); the essential guarantee -- that no shim is
+	# leaked -- is already checked by the caller.
+	local -r force="${1:-}"
+
 	info "show pod ${podid}"
 	sudo crictl --timeout=20s pods --id "${podid}"
 	info "stop pod ${podid}"
 	sudo crictl --timeout=20s stopp "${podid}"
 	info "remove pod ${podid}"
-	sudo crictl --timeout=20s rmp "${podid}"
+
+	if [[ "${force}" != "force" ]]; then
+		sudo crictl --timeout=20s rmp "${podid}"
+		restore_cri_integration_containerd_host_config
+		return 0
+	fi
+
+	# Force path (killed-VMM): after a hard VMM kill the container's TaskExit
+	# may not have reached containerd yet, leaving CRI's view as "running".
+	# Retry the normal removal a few times to give the event time to arrive,
+	# and only fall back to a force removal (how a crashed sandbox is cleaned
+	# up in practice) if it keeps failing.
+	local removed="false"
+	local i
+	for i in 1 2 3; do
+		if sudo crictl --timeout=20s rmp "${podid}"; then
+			removed="true"
+			break
+		fi
+		info "rmp attempt ${i} failed, retrying"
+		sleep 1
+	done
+	if [[ "${removed}" != "true" ]]; then
+		info "falling back to force removal of pod ${podid}"
+		sudo crictl --timeout=20s rmp -f "${podid}"
+	fi
 
 	restore_cri_integration_containerd_host_config
 }
@@ -367,7 +402,11 @@ function TestKilledVmmCleanup() {
 	remained=$(pgrep -f shimv2 || true)
 	[[ -z "${remained}" ]] || die "found remaining shimv2 process ${remained}"
 
-	testContainerStop
+	# The VMM was hard-killed, so the container's exit may not have reached
+	# containerd before the shim exited.  Retry the normal removal a few times
+	# and fall back to a force removal, mirroring how a crashed sandbox is
+	# cleaned up in practice.
+	testContainerStop "force"
 
 	info "stop containerd"
 }