diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go
index 405ffb9e0a9..1fea953a983 100644
--- a/test/integration/framework/test_server.go
+++ b/test/integration/framework/test_server.go
@@ -65,12 +65,12 @@ func StartTestServer(t testing.TB, setup TestServerSetup) (client.Interface, *re
 		t.Fatalf("Couldn't create temp dir: %v", err)
 	}
 
-	stopCh := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
 	var errCh chan error
 	tearDownFn := func() {
-		// Closing stopCh is stopping apiserver and cleaning up
+		// Calling cancel function is stopping apiserver and cleaning up
 		// after itself, including shutting down its storage layer.
-		close(stopCh)
+		cancel()
 
 		// If the apiserver was started, let's wait for it to
 		// shutdown clearly.
@@ -173,7 +173,7 @@ func StartTestServer(t testing.TB, setup TestServerSetup) (client.Interface, *re
 	errCh = make(chan error)
 	go func() {
 		defer close(errCh)
-		if err := kubeAPIServer.GenericAPIServer.PrepareRun().Run(stopCh); err != nil {
+		if err := kubeAPIServer.GenericAPIServer.PrepareRun().Run(ctx.Done()); err != nil {
 			errCh <- err
 		}
 	}()
@@ -203,15 +203,15 @@ func StartTestServer(t testing.TB, setup TestServerSetup) (client.Interface, *re
 		}
 
 		healthStatus := 0
-		kubeClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).StatusCode(&healthStatus)
+		kubeClient.Discovery().RESTClient().Get().AbsPath("/healthz").Do(ctx).StatusCode(&healthStatus)
 		if healthStatus != http.StatusOK {
 			return false, nil
 		}
 
-		if _, err := kubeClient.CoreV1().Namespaces().Get(context.TODO(), "default", metav1.GetOptions{}); err != nil {
+		if _, err := kubeClient.CoreV1().Namespaces().Get(ctx, "default", metav1.GetOptions{}); err != nil {
 			return false, nil
 		}
-		if _, err := kubeClient.CoreV1().Namespaces().Get(context.TODO(), "kube-system", metav1.GetOptions{}); err != nil {
+		if _, err := kubeClient.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}); err != nil {
 			return false, nil
 		}
 
diff --git a/test/integration/scheduler/plugins/plugins_test.go b/test/integration/scheduler/plugins/plugins_test.go
index 20c3bec480e..df7e4b5ae1c 100644
--- a/test/integration/scheduler/plugins/plugins_test.go
+++ b/test/integration/scheduler/plugins/plugins_test.go
@@ -2556,6 +2556,8 @@ func initTestSchedulerForFrameworkTest(t *testing.T, testCtx *testutils.TestCont
 
 	if nodeCount > 0 {
 		if _, err := createAndWaitForNodesInCache(testCtx, "test-node", st.MakeNode(), nodeCount); err != nil {
+			// Make sure to cleanup the resources when initializing error.
+			testutils.CleanupTest(t, testCtx)
 			t.Fatal(err)
 		}
 	}