Add DisableCloudProviders FG

FeatureGate acts as a secondary switch to disable cloud-controller loops
in KCM, Kubelet and KAPI.

Provide comprehensive logging information to users, so they will be
guided in adoption of out-of-tree cloud provider implementation.

cmd/kube-apiserver/app/server.go

@@ -72,6 +72,7 @@ import (
 	"k8s.io/kubernetes/pkg/controlplane"
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	"k8s.io/kubernetes/pkg/controlplane/tunneler"
+	"k8s.io/kubernetes/pkg/features"
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
@@ -244,6 +245,14 @@ func CreateNodeDialer(s completedServerRunOptions) (tunneler.Tunneler, *http.Tra
 		// Get ssh key distribution func, if supported
 		var installSSHKey tunneler.InstallSSHKey
 
+		if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) && cloudprovider.IsDeprecatedInternal(s.CloudProvider.CloudProvider) {
+			cloudprovider.DisableWarningForProvider(s.CloudProvider.CloudProvider)
+			return nil, nil, fmt.Errorf("cloud provider %q and ssh-user %q was specified, but built-in cloud providers are disabled. "+
+				"Please set --cloud-provider=external and use an external network proxy, see https://github.com/kubernetes-sigs/apiserver-network-proxy",
+				s.CloudProvider.CloudProvider, s.SSHUser)
+
+		}
+
 		cloudprovider.DeprecationWarningForProvider(s.CloudProvider.CloudProvider)
 		cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile)
 		if err != nil {

cmd/kube-controller-manager/app/cloudproviders.go

@@ -19,7 +19,9 @@ package app
 import (
 	"fmt"
 
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 
 	"k8s.io/client-go/informers"
 	cloudprovider "k8s.io/cloud-provider"
@@ -32,6 +34,14 @@ func createCloudProvider(cloudProvider string, externalCloudVolumePlugin string,
 	var cloud cloudprovider.Interface
 	var loopMode ControllerLoopMode
 	var err error
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) && cloudprovider.IsDeprecatedInternal(cloudProvider) {
+		cloudprovider.DisableWarningForProvider(cloudProvider)
+		return nil, ExternalLoops, fmt.Errorf(
+			"cloud provider %q was specified, but built-in cloud providers are disabled. Please set --cloud-provider=external and migrate to an external cloud provider",
+			cloudProvider)
+	}
+
 	if cloudprovider.IsExternal(cloudProvider) {
 		loopMode = ExternalLoops
 		if externalCloudVolumePlugin == "" {

pkg/features/kube_features.go

@@ -617,6 +617,12 @@ const (
 	// Enable kubelet exec plugins for image pull credentials.
 	KubeletCredentialProviders featuregate.Feature = "KubeletCredentialProviders"
 
+	// owner: @andrewsykim
+	// alpha: v1.22
+	//
+	// Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag.
+	DisableCloudProviders featuregate.Feature = "DisableCloudProviders"
+
 	// owner: @zshihang
 	// alpha: v1.20
 	// beta: v1.21
@@ -837,6 +843,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	NamespaceDefaultLabelName:                      {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.24
 	CSIVolumeHealth:                                {Default: false, PreRelease: featuregate.Alpha},
 	WindowsHostProcessContainers:                   {Default: false, PreRelease: featuregate.Alpha},
+	DisableCloudProviders:                          {Default: false, PreRelease: featuregate.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/kubelet/kubelet.go

@@ -387,6 +387,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		}
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) && cloudprovider.IsDeprecatedInternal(cloudProvider) {
+		cloudprovider.DisableWarningForProvider(cloudProvider)
+		return nil, fmt.Errorf("cloud provider %q was specified, but built-in cloud providers are disabled. Please set --cloud-provider=external and migrate to an external cloud provider", cloudProvider)
+	}
+
 	var nodeHasSynced cache.InformerSynced
 	var nodeLister corelisters.NodeLister
 

staging/src/k8s.io/cloud-provider/plugins.go

@@ -40,11 +40,11 @@ var (
 		external bool
 		detail   string
 	}{
-		{"aws", false, "The AWS provider is deprecated and will be removed in a future release"},
+		{"aws", false, "The AWS provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-aws"},
-		{"azure", false, "The Azure provider is deprecated and will be removed in a future release"},
+		{"azure", false, "The Azure provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes-sigs/cloud-provider-azure"},
-		{"gce", false, "The GCE provider is deprecated and will be removed in a future release"},
+		{"gce", false, "The GCE provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-gcp"},
 		{"openstack", true, "https://github.com/kubernetes/cloud-provider-openstack"},
-		{"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release"},
+		{"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-vsphere"},
 	}
 )
 
@@ -91,6 +91,33 @@ func IsExternal(name string) bool {
 	return name == externalCloudProvider
 }
 
+// IsDeprecatedInternal is responsible for preventing cloud.Interface
+// from being initialized in kubelet, kube-controller-manager or kube-api-server
+func IsDeprecatedInternal(name string) bool {
+	for _, provider := range deprecatedCloudProviders {
+		if provider.name == name {
+			return true
+		}
+	}
+
+	return false
+}
+
+// DisableWarningForProvider logs information about disabled cloud provider state
+func DisableWarningForProvider(providerName string) {
+	for _, provider := range deprecatedCloudProviders {
+		if provider.name == providerName {
+			klog.Infof("INFO: Please make sure you are running external cloud controller manager binary for provider %q."+
+				"In-tree cloud providers are currently disabled. Refer to https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/cloud-provider/sample"+
+				"for example implementation.", providerName)
+			detail := fmt.Sprintf("Please reach to sig-cloud-provider and use 'external' cloud provider for %q: %s", providerName, provider.detail)
+			klog.Warningf("WARNING: %q built-in cloud provider is now disabled. %s", providerName, detail)
+			break
+		}
+	}
+}
+
+// DeprecationWarningForProvider logs information about deprecated cloud provider state
 func DeprecationWarningForProvider(providerName string) {
 	for _, provider := range deprecatedCloudProviders {
 		if provider.name != providerName {