diff --git a/test/e2e/apimachinery/aggregator.go b/test/e2e/apimachinery/aggregator.go
index e753efcb0da..4b95a0c035d 100644
--- a/test/e2e/apimachinery/aggregator.go
+++ b/test/e2e/apimachinery/aggregator.go
@@ -57,18 +57,8 @@ const (
 )
 
 var _ = SIGDescribe("Aggregator", func() {
-	var ns string
-	var c clientset.Interface
 	var aggrclient *aggregatorclient.Clientset
 
-	// BeforeEachs run in LIFO order, AfterEachs run in FIFO order.
-	// We want cleanTest to happen before the namespace cleanup AfterEach
-	// inserted by NewDefaultFramework, so we put this AfterEach in front
-	// of NewDefaultFramework.
-	ginkgo.AfterEach(func() {
-		cleanTest(c, aggrclient, ns)
-	})
-
 	f := framework.NewDefaultFramework("aggregator")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 
@@ -76,19 +66,15 @@ var _ = SIGDescribe("Aggregator", func() {
 	// NewDefaultFramework to happen before this, so we put this BeforeEach
 	// after NewDefaultFramework.
 	ginkgo.BeforeEach(func() {
-		c = f.ClientSet
-		ns = f.Namespace.Name
-
-		if aggrclient == nil {
-			config, err := framework.LoadConfig()
-			if err != nil {
-				framework.Failf("could not load config: %v", err)
-			}
-			aggrclient, err = aggregatorclient.NewForConfig(config)
-			if err != nil {
-				framework.Failf("could not create aggregator client: %v", err)
-			}
+		config, err := framework.LoadConfig()
+		if err != nil {
+			framework.Failf("could not load config: %v", err)
 		}
+		aggrclient, err = aggregatorclient.NewForConfig(config)
+		if err != nil {
+			framework.Failf("could not create aggregator client: %v", err)
+		}
+		ginkgo.DeferCleanup(cleanTest, f.ClientSet, aggrclient, f.Namespace.Name)
 	})
 
 	/*