diff --git a/cmd/kube-controller-manager/app/config/config.go b/cmd/kube-controller-manager/app/config/config.go
index c565f23ab6d..eafc6a38583 100644
--- a/cmd/kube-controller-manager/app/config/config.go
+++ b/cmd/kube-controller-manager/app/config/config.go
@@ -29,6 +29,9 @@ type Config struct {
 	ComponentConfig componentconfig.KubeControllerManagerConfiguration
 
 	SecureServing *apiserver.SecureServingInfo
+	// LoopbackClientConfig is a config for a privileged loopback connection
+	LoopbackClientConfig *restclient.Config
+
 	// TODO: remove deprecated insecure serving
 	InsecureServing *apiserver.DeprecatedInsecureServingInfo
 	Authentication  apiserver.AuthenticationInfo
@@ -60,5 +63,8 @@ type CompletedConfig struct {
 // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
 func (c *Config) Complete() *CompletedConfig {
 	cc := completedConfig{c}
+
+	apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization)
+
 	return &CompletedConfig{&cc}
 }
diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go
index d33466cfcfd..0e19fefc0a0 100644
--- a/cmd/kube-controller-manager/app/options/options.go
+++ b/cmd/kube-controller-manager/app/options/options.go
@@ -83,9 +83,9 @@ type KubeControllerManagerOptions struct {
 	Controllers               []string
 	ExternalCloudVolumePlugin string
 
-	SecureServing *apiserveroptions.SecureServingOptions
+	SecureServing *apiserveroptions.SecureServingOptionsWithLoopback
 	// TODO: remove insecure serving mode
-	InsecureServing *apiserveroptions.DeprecatedInsecureServingOptions
+	InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback
 	Authentication  *apiserveroptions.DelegatingAuthenticationOptions
 	Authorization   *apiserveroptions.DelegatingAuthorizationOptions
 
@@ -179,12 +179,12 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
 			ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs,
 		},
 		Controllers:   componentConfig.Controllers,
-		SecureServing: apiserveroptions.NewSecureServingOptions(),
-		InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{
+		SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
+		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
 			BindAddress: net.ParseIP(componentConfig.KubeCloudShared.Address),
 			BindPort:    int(componentConfig.KubeCloudShared.Port),
 			BindNetwork: "tcp",
-		},
+		}).WithLoopback(),
 		Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(),
 		Authorization:  apiserveroptions.NewDelegatingAuthorizationOptions(),
 	}
@@ -347,10 +347,10 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e
 	if err := s.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil {
 		return err
 	}
-	if err := s.InsecureServing.ApplyTo(&c.InsecureServing); err != nil {
+	if err := s.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil {
 		return err
 	}
-	if err := s.SecureServing.ApplyTo(&c.SecureServing); err != nil {
+	if err := s.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
 		return err
 	}
 	if s.SecureServing.BindPort != 0 || s.SecureServing.Listener != nil {
diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go
index 0670c7ccd3d..73ceb19dc89 100644
--- a/cmd/kube-controller-manager/app/options/options_test.go
+++ b/cmd/kube-controller-manager/app/options/options_test.go
@@ -250,7 +250,7 @@ func TestAddFlags(t *testing.T) {
 			ConcurrentServiceSyncs: 2,
 		},
 		Controllers: []string{"foo", "bar"},
-		SecureServing: &apiserveroptions.SecureServingOptions{
+		SecureServing: (&apiserveroptions.SecureServingOptions{
 			BindPort:    10001,
 			BindAddress: net.ParseIP("192.168.4.21"),
 			ServerCert: apiserveroptions.GeneratableKeyCert{
@@ -258,12 +258,12 @@ func TestAddFlags(t *testing.T) {
 				PairName:      "kube-controller-manager",
 			},
 			HTTP2MaxStreamsPerConnection: 47,
-		},
-		InsecureServing: &apiserveroptions.DeprecatedInsecureServingOptions{
+		}).WithLoopback(),
+		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
 			BindAddress: net.ParseIP("192.168.4.10"),
 			BindPort:    int(10000),
 			BindNetwork: "tcp",
-		},
+		}).WithLoopback(),
 		Authentication: &apiserveroptions.DelegatingAuthenticationOptions{
 			CacheTTL:   10 * time.Second,
 			ClientCert: apiserveroptions.ClientCertAuthenticationOptions{},