From 80e3b071947849ea22eec30ffb4e9fa981331379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= <ffidencio@nvidia.com>
Date: Tue, 7 Jul 2026 07:43:37 +0200
Subject: [PATCH] runtime: don't block shim cleanup on a dead agent
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For `docker run --rm`, containerd invokes the shim `delete` binary
(cleanupAfterDeadShim) once the container task exits.  Kata's Cleanup
path re-loads the sandbox and calls StopContainer/DeleteContainer/Stop,
each of which lazily connects to the guest agent over vsock.

When the sandbox was already torn down by the main shim (the common
case for a short-lived `docker run --rm`), the VM -- and its agent --
are gone, so that vsock connect blocks until containerd's delete
timeout SIGKILLs the binary.  The removal then fails and `docker run
--rm` returns non-zero even though the container itself exited 0.

Detect the already-dead hypervisor (its pidfile is gone / the pid no
longer maps to a live process) at the start of CleanupContainer and
mark the agent dead.  Subsequent agent RPCs then fail fast with "Dead
agent" and the force path performs only host-side cleanup, so the
delete binary returns promptly instead of hanging.

The legitimate "shim crashed but VM still alive" cleanup is unaffected:
the hypervisor is still running, so the agent is not marked dead and
the normal agent-based teardown proceeds.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
---
 src/runtime/virtcontainers/api.go        | 12 ++++++++++++
 src/runtime/virtcontainers/hypervisor.go | 15 +++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/runtime/virtcontainers/api.go b/src/runtime/virtcontainers/api.go
index 6cda1db3db..0bd1d650b9 100644
--- a/src/runtime/virtcontainers/api.go
+++ b/src/runtime/virtcontainers/api.go
@@ -141,6 +141,18 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force
 	}
 	defer s.Release(ctx)
 
+	// If the hypervisor process is already gone -- e.g. the sandbox was torn
+	// down by the main shim and this is containerd invoking the `delete`
+	// binary (cleanupAfterDeadShim) for a `docker run --rm` -- any agent RPC
+	// below would block on a dead vsock connection until the caller times out
+	// and SIGKILLs us, which surfaces as a failed container removal.  Mark the
+	// agent dead so those calls fail fast; the force path then only performs
+	// host-side cleanup.
+	if !IsHypervisorRunning(s.hypervisor) {
+		s.Logger().Info("hypervisor is not running, marking agent dead before cleanup")
+		s.agent.markDead(ctx)
+	}
+
 	_, err = s.StopContainer(ctx, containerID, force)
 	if err != nil && !force {
 		return err
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
index 4aa2812df3..71b932deef 100644
--- a/src/runtime/virtcontainers/hypervisor.go
+++ b/src/runtime/virtcontainers/hypervisor.go
@@ -18,6 +18,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/pkg/errors"
 
@@ -1252,6 +1253,20 @@ func GetHypervisorPid(h Hypervisor) int {
 	return pids[0]
 }
 
+// IsHypervisorRunning reports whether the hypervisor process backing the
+// sandbox is still alive.  It is best-effort: a missing pidfile or a pid that
+// no longer maps to a live process is treated as "not running".
+func IsHypervisorRunning(h Hypervisor) bool {
+	pid := GetHypervisorPid(h)
+	if pid <= 0 {
+		return false
+	}
+	// Signal 0 performs error checking without sending a signal: nil means
+	// the process exists, EPERM means it exists but we may not signal it.
+	err := syscall.Kill(pid, 0)
+	return err == nil || err == syscall.EPERM
+}
+
 // Kind of guest protection
 type guestProtection uint8