diff --git a/cmd/kubeadm/app/componentconfigs/configset.go b/cmd/kubeadm/app/componentconfigs/configset.go
index 0be33d0a292..a1ab2bfa972 100644
--- a/cmd/kubeadm/app/componentconfigs/configset.go
+++ b/cmd/kubeadm/app/componentconfigs/configset.go
@@ -72,7 +72,7 @@ func (h *handler) FromDocumentMap(docmap kubeadmapi.DocumentMap) (kubeadmapi.Com
 // fromConfigMap is an utility function, which will load the value of a key of a config map and use h.FromDocumentMap() to perform the parsing
 // This is an utility func. Used by the component config support implementations. Don't use it outside of that context.
 func (h *handler) fromConfigMap(client clientset.Interface, cmName, cmKey string, mustExist bool) (kubeadmapi.ComponentConfig, error) {
-	configMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, cmName)
+	configMap, err := apiclient.GetConfigMapWithShortRetry(client, metav1.NamespaceSystem, cmName)
 	if err != nil {
 		if !mustExist && (apierrors.IsNotFound(err) || apierrors.IsForbidden(err)) {
 			klog.Warningf("Warning: No %s config is loaded. Continuing without it: %v", h.GroupVersion, err)
diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go
index 6677e1f7266..828dab9c7d7 100644
--- a/cmd/kubeadm/app/util/apiclient/idempotency.go
+++ b/cmd/kubeadm/app/util/apiclient/idempotency.go
@@ -19,6 +19,7 @@ package apiclient
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -331,13 +332,14 @@ func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Nod
 	return lastError
 }
 
-// GetConfigMapWithRetry tries to retrieve a ConfigMap using the given client,
-// retrying if we get an unexpected error.
-func GetConfigMapWithRetry(client clientset.Interface, namespace, name string) (*v1.ConfigMap, error) {
+// GetConfigMapWithShortRetry tries to retrieve a ConfigMap using the given client, retrying for a short
+// time if it gets an unexpected error. The main usage of this function is in areas of the code that
+// fallback to a default ConfigMap value in case the one from the API cannot be quickly obtained.
+func GetConfigMapWithShortRetry(client clientset.Interface, namespace, name string) (*v1.ConfigMap, error) {
 	var cm *v1.ConfigMap
 	var lastError error
 	err := wait.PollUntilContextTimeout(context.Background(),
-		constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration,
+		time.Millisecond*50, time.Millisecond*350,
 		true, func(ctx context.Context) (bool, error) {
 			var err error
 			cm, err = client.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{})
diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go
index bd956eedc36..afd58b3345d 100644
--- a/cmd/kubeadm/app/util/config/cluster.go
+++ b/cmd/kubeadm/app/util/config/cluster.go
@@ -71,7 +71,7 @@ func FetchInitConfigurationFromCluster(client clientset.Interface, printer outpu
 // getInitConfigurationFromCluster is separate only for testing purposes, don't call it directly, use FetchInitConfigurationFromCluster instead
 func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Interface, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) {
 	// Also, the config map really should be KubeadmConfigConfigMap...
-	configMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap)
+	configMap, err := apiclient.GetConfigMapWithShortRetry(client, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get config map")
 	}