Merge pull request #117503 from dims/phase-4-kep-2395-removing-in-tree-cloud-providers

[KEP-2395] Phase 4 - Disabling In-Tree Providers

cluster/gce/config-default.sh

@@ -559,13 +559,4 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}"
 # --image-credential-provider-bin-dir=${path-to-auth-provider-binary}
 # Also, it is required that DisableKubeletCloudCredentialProviders
 # feature gates are set to true for kubelet to use external credential provider.
-export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
+export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
-
-# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags
-# DisableKubeletCloudCredentialProviders and DisableCloudProviders
-if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then
-  export ENABLE_AUTH_PROVIDER_GCP=true
-  if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then
-    export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True"
-  fi
-fi

cluster/gce/config-test.sh

@@ -608,13 +608,4 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}"
 # --image-credential-provider-bin-dir=${path-to-auth-provider-binary}
 # Also, it is required that DisableKubeletCloudCredentialProviders and KubeletCredentialProviders
 # feature gates are set to true for kubelet to use external credential provider.
-export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
+export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
-
-# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags
-# DisableKubeletCloudCredentialProviders and DisableCloudProviders
-if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then
-  export ENABLE_AUTH_PROVIDER_GCP=true
-  if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then
-    export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True"
-  fi
-fi

cmd/kube-apiserver/app/server.go

@@ -46,6 +46,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/keyutil"
+	cloudprovider "k8s.io/cloud-provider"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/logs"
@@ -67,6 +68,7 @@ import (
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
@@ -292,6 +294,11 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 		config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
 	}
 
+	err = validateCloudProviderOptions(opts.CloudProvider)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to validate cloud provider: %w", err)
+	}
+
 	// setup admission
 	admissionConfig := &kubeapiserveradmission.Config{
 		ExternalInformers:    versionedInformers,
@@ -356,6 +363,34 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 	return config, serviceResolver, pluginInitializers, nil
 }
 
+func validateCloudProviderOptions(opts *kubeoptions.CloudProviderOptions) error {
+	if opts.CloudProvider == "" {
+		return nil
+	}
+	if opts.CloudProvider == "external" {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableCloudProviders feature to true", opts.CloudProvider)
+		}
+		if !utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableKubeletCloudCredentialProviders feature to true", opts.CloudProvider)
+		}
+		return nil
+	} else if cloudprovider.IsDeprecatedInternal(opts.CloudProvider) {
+		if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableCloudProviders feature to false", opts.CloudProvider)
+		}
+		if utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableKubeletCloudCredentialProviders feature to false", opts.CloudProvider)
+		}
+		return nil
+	}
+	return fmt.Errorf("unknown --cloud-provider : %s", opts.CloudProvider)
+}
+
 var testServiceResolver webhook.ServiceResolver
 
 // SetServiceResolverForTests allows the service resolver to be overridden during tests.

pkg/credentialprovider/gcp/metadata_test.go

@@ -30,7 +30,10 @@ import (
 	"testing"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	"k8s.io/kubernetes/pkg/credentialprovider"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
 	"k8s.io/legacy-cloud-providers/gce/gcpcredential"
 )
 
@@ -53,6 +56,9 @@ func TestMetadata(t *testing.T) {
 	if runtime.GOOS == "windows" && !onGCEVM() {
 		t.Skip("Skipping test on Windows, not on GCE.")
 	}
+
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.DisableKubeletCloudCredentialProviders, false)()
+
 	var err error
 	gceProductNameFile, err = createProductNameFile()
 	if err != nil {

pkg/features/kube_features.go

@@ -229,12 +229,14 @@ const (
 
 	// owner: @andrewsykim
 	// alpha: v1.22
+	// beta: v1.29
 	//
 	// Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag.
 	DisableCloudProviders featuregate.Feature = "DisableCloudProviders"
 
 	// owner: @andrewsykim
 	// alpha: v1.23
+	// beta: v1.29
 	//
 	// Disable in-tree functionality in kubelet to authenticate to cloud provider container registries for image pull credentials.
 	DisableKubeletCloudCredentialProviders featuregate.Feature = "DisableKubeletCloudCredentialProviders"
@@ -1012,9 +1014,9 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	DefaultHostNetworkHostPortsInPodTemplates: {Default: false, PreRelease: featuregate.Deprecated},
 
-	DisableCloudProviders: {Default: false, PreRelease: featuregate.Alpha},
+	DisableCloudProviders: {Default: true, PreRelease: featuregate.Beta},
 
-	DisableKubeletCloudCredentialProviders: {Default: false, PreRelease: featuregate.Alpha},
+	DisableKubeletCloudCredentialProviders: {Default: true, PreRelease: featuregate.Beta},
 
 	DevicePluginCDIDevices: {Default: false, PreRelease: featuregate.Alpha},