From 5d57b3794c1a08b707d8ac6857bcb93cdd4975f0 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 11 Mar 2021 13:39:25 +0100 Subject: [PATCH] Add DisableCloudProviders FG FeatureGate acts as a secondary switch to disable cloud-controller loops in KCM, Kubelet and KAPI. Provide comprehensive logging information to users, so they will be guided in adoption of out-of-tree cloud provider implementation. --- cmd/kube-apiserver/app/server.go | 9 +++++ .../app/cloudproviders.go | 10 ++++++ pkg/features/kube_features.go | 7 ++++ pkg/kubelet/kubelet.go | 5 +++ staging/src/k8s.io/cloud-provider/plugins.go | 35 ++++++++++++++++--- 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 03205f07626..15edadd614d 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -72,6 +72,7 @@ import ( "k8s.io/kubernetes/pkg/controlplane" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/pkg/controlplane/tunneler" + "k8s.io/kubernetes/pkg/features" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" "k8s.io/kubernetes/pkg/kubeapiserver" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" @@ -244,6 +245,14 @@ func CreateNodeDialer(s completedServerRunOptions) (tunneler.Tunneler, *http.Tra // Get ssh key distribution func, if supported var installSSHKey tunneler.InstallSSHKey + if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) && cloudprovider.IsDeprecatedInternal(s.CloudProvider.CloudProvider) { + cloudprovider.DisableWarningForProvider(s.CloudProvider.CloudProvider) + return nil, nil, fmt.Errorf("cloud provider %q and ssh-user %q was specified, but built-in cloud providers are disabled. "+ + "Please set --cloud-provider=external and use an external network proxy, see https://github.com/kubernetes-sigs/apiserver-network-proxy", + s.CloudProvider.CloudProvider, s.SSHUser) + + } + cloudprovider.DeprecationWarningForProvider(s.CloudProvider.CloudProvider) cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile) if err != nil { diff --git a/cmd/kube-controller-manager/app/cloudproviders.go b/cmd/kube-controller-manager/app/cloudproviders.go index ee5b0babf15..c9775a9c1a4 100644 --- a/cmd/kube-controller-manager/app/cloudproviders.go +++ b/cmd/kube-controller-manager/app/cloudproviders.go @@ -19,7 +19,9 @@ package app import ( "fmt" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/client-go/informers" cloudprovider "k8s.io/cloud-provider" @@ -32,6 +34,14 @@ func createCloudProvider(cloudProvider string, externalCloudVolumePlugin string, var cloud cloudprovider.Interface var loopMode ControllerLoopMode var err error + + if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) && cloudprovider.IsDeprecatedInternal(cloudProvider) { + cloudprovider.DisableWarningForProvider(cloudProvider) + return nil, ExternalLoops, fmt.Errorf( + "cloud provider %q was specified, but built-in cloud providers are disabled. Please set --cloud-provider=external and migrate to an external cloud provider", + cloudProvider) + } + if cloudprovider.IsExternal(cloudProvider) { loopMode = ExternalLoops if externalCloudVolumePlugin == "" { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 1f7a7e27c44..b0d0bd23ca2 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -617,6 +617,12 @@ const ( // Enable kubelet exec plugins for image pull credentials. KubeletCredentialProviders featuregate.Feature = "KubeletCredentialProviders" + // owner: @andrewsykim + // alpha: v1.22 + // + // Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag. + DisableCloudProviders featuregate.Feature = "DisableCloudProviders" + // owner: @zshihang // alpha: v1.20 // beta: v1.21 @@ -837,6 +843,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NamespaceDefaultLabelName: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.24 CSIVolumeHealth: {Default: false, PreRelease: featuregate.Alpha}, WindowsHostProcessContainers: {Default: false, PreRelease: featuregate.Alpha}, + DisableCloudProviders: {Default: false, PreRelease: featuregate.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index ef171407d5f..7f390290496 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -387,6 +387,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } } + if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) && cloudprovider.IsDeprecatedInternal(cloudProvider) { + cloudprovider.DisableWarningForProvider(cloudProvider) + return nil, fmt.Errorf("cloud provider %q was specified, but built-in cloud providers are disabled. Please set --cloud-provider=external and migrate to an external cloud provider", cloudProvider) + } + var nodeHasSynced cache.InformerSynced var nodeLister corelisters.NodeLister diff --git a/staging/src/k8s.io/cloud-provider/plugins.go b/staging/src/k8s.io/cloud-provider/plugins.go index e408e9cbc48..bfd73163149 100644 --- a/staging/src/k8s.io/cloud-provider/plugins.go +++ b/staging/src/k8s.io/cloud-provider/plugins.go @@ -40,11 +40,11 @@ var ( external bool detail string }{ - {"aws", false, "The AWS provider is deprecated and will be removed in a future release"}, - {"azure", false, "The Azure provider is deprecated and will be removed in a future release"}, - {"gce", false, "The GCE provider is deprecated and will be removed in a future release"}, + {"aws", false, "The AWS provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-aws"}, + {"azure", false, "The Azure provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes-sigs/cloud-provider-azure"}, + {"gce", false, "The GCE provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-gcp"}, {"openstack", true, "https://github.com/kubernetes/cloud-provider-openstack"}, - {"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release"}, + {"vsphere", false, "The vSphere provider is deprecated and will be removed in a future release. Please use https://github.com/kubernetes/cloud-provider-vsphere"}, } ) @@ -91,6 +91,33 @@ func IsExternal(name string) bool { return name == externalCloudProvider } +// IsDeprecatedInternal is responsible for preventing cloud.Interface +// from being initialized in kubelet, kube-controller-manager or kube-api-server +func IsDeprecatedInternal(name string) bool { + for _, provider := range deprecatedCloudProviders { + if provider.name == name { + return true + } + } + + return false +} + +// DisableWarningForProvider logs information about disabled cloud provider state +func DisableWarningForProvider(providerName string) { + for _, provider := range deprecatedCloudProviders { + if provider.name == providerName { + klog.Infof("INFO: Please make sure you are running external cloud controller manager binary for provider %q."+ + "In-tree cloud providers are currently disabled. Refer to https://github.com/kubernetes/kubernetes/tree/master/staging/src/k8s.io/cloud-provider/sample"+ + "for example implementation.", providerName) + detail := fmt.Sprintf("Please reach to sig-cloud-provider and use 'external' cloud provider for %q: %s", providerName, provider.detail) + klog.Warningf("WARNING: %q built-in cloud provider is now disabled. %s", providerName, detail) + break + } + } +} + +// DeprecationWarningForProvider logs information about deprecated cloud provider state func DeprecationWarningForProvider(providerName string) { for _, provider := range deprecatedCloudProviders { if provider.name != providerName {