diff --git a/test/e2e/dra/deploy.go b/test/e2e/dra/deploy.go
index 6c4394edca8..ff44051f9d7 100644
--- a/test/e2e/dra/deploy.go
+++ b/test/e2e/dra/deploy.go
@@ -432,6 +432,21 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr
 				klog.Background().Info("deleting CDI file", "node", nodename, "filename", name)
 				return d.removeFile(&pod, name)
 			},
+			ErrorHandler: func(ctx context.Context, err error, msg string) {
+				// Record a failure, but don't kill the background goroutine.
+				defer ginkgo.GinkgoRecover()
+				// During tests when canceling the context it is possible to get all kinds of
+				// follow-up errors for that, like:
+				//   processing ResourceSlice objects: retrieve node "127.0.0.1": client rate limiter Wait returned an error: context canceled
+				//
+				// The "context canceled" error was not wrapped, so `errors.Is` doesn't work.
+				// Instead of trying to detect errors which can be ignored, let's only
+				// treat errors as failures which definitely shouldn't occur:
+				var droppedFields *resourceslice.DroppedFieldsError
+				if errors.As(err, &droppedFields) {
+					framework.Failf("driver %s: %v", d.Name, err)
+				}
+			},
 		}
 
 		if dr, ok := driverResources[nodename]; ok {