diff --git a/cmd/kubeadm/app/util/runtime/runtime.go b/cmd/kubeadm/app/util/runtime/runtime.go
index 4dc08805a13..56e7d4abad2 100644
--- a/cmd/kubeadm/app/util/runtime/runtime.go
+++ b/cmd/kubeadm/app/util/runtime/runtime.go
@@ -140,10 +140,15 @@ func (runtime *CRIRuntime) RemoveContainers(containers []string) error {
 func (runtime *DockerRuntime) RemoveContainers(containers []string) error {
 	errs := []error{}
 	for _, container := range containers {
-		out, err := runtime.exec.Command("docker", "rm", "--force", "--volumes", container).CombinedOutput()
+		out, err := runtime.exec.Command("docker", "stop", container).CombinedOutput()
 		if err != nil {
 			// don't stop on errors, try to remove as many containers as possible
-			errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out)))
+			errs = append(errs, errors.Wrapf(err, "failed to stop running container %s: output: %s, error", container, string(out)))
+		} else {
+			out, err = runtime.exec.Command("docker", "rm", "--volumes", container).CombinedOutput()
+			if err != nil {
+				errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out)))
+			}
 		}
 	}
 	return errorsutil.NewAggregate(errs)
diff --git a/cmd/kubeadm/app/util/runtime/runtime_test.go b/cmd/kubeadm/app/util/runtime/runtime_test.go
index ea116450563..df92e691705 100644
--- a/cmd/kubeadm/app/util/runtime/runtime_test.go
+++ b/cmd/kubeadm/app/util/runtime/runtime_test.go
@@ -185,8 +185,9 @@ func TestRemoveContainers(t *testing.T) {
 			fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1
 			fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK,
 			fakeErr, fakeOK, fakeOK, fakeErr, fakeOK,
-			fakeOK, fakeOK, fakeOK,
-			fakeOK, fakeErr, fakeOK,
+			fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK,
+			fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK,
+			fakeErr, fakeOK, fakeOK, fakeErr, fakeOK,
 		},
 	}
 	execer := fakeexec.FakeExec{
@@ -204,7 +205,8 @@ func TestRemoveContainers(t *testing.T) {
 		{"invalid: CRI rmp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true},
 		{"invalid: CRI stopp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true},
 		{"valid: remove containers using docker", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, false},
-		{"invalid: remove containers using docker", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true},
+		{"invalid: docker rm failure", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true},
+		{"invalid: docker stop failure", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true},
 	}
 
 	for _, tc := range cases {