Validate the cloud-provider passed in and the corresponding feature flags

Signed-off-by: Davanum Srinivas <davanum@gmail.com>

cluster/gce/config-default.sh

@@ -559,13 +559,4 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}"
 # --image-credential-provider-bin-dir=${path-to-auth-provider-binary}
 # Also, it is required that DisableKubeletCloudCredentialProviders
 # feature gates are set to true for kubelet to use external credential provider.
-export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
+export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
-
-# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags
-# DisableKubeletCloudCredentialProviders and DisableCloudProviders
-if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then
-  export ENABLE_AUTH_PROVIDER_GCP=true
-  if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then
-    export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True"
-  fi
-fi

cluster/gce/config-test.sh

@@ -608,13 +608,4 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}"
 # --image-credential-provider-bin-dir=${path-to-auth-provider-binary}
 # Also, it is required that DisableKubeletCloudCredentialProviders and KubeletCredentialProviders
 # feature gates are set to true for kubelet to use external credential provider.
-export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
+export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}"
-
-# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags
-# DisableKubeletCloudCredentialProviders and DisableCloudProviders
-if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then
-  export ENABLE_AUTH_PROVIDER_GCP=true
-  if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then
-    export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True"
-  fi
-fi

cmd/kube-apiserver/app/server.go

@@ -46,6 +46,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/keyutil"
+	cloudprovider "k8s.io/cloud-provider"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/logs"
@@ -67,6 +68,7 @@ import (
 	"k8s.io/kubernetes/pkg/controlplane/reconcilers"
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 )
 
@@ -292,6 +294,11 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 		config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders
 	}
 
+	err = validateCloudProviderOptions(opts.CloudProvider)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to validate cloud provider: %w", err)
+	}
+
 	// setup admission
 	admissionConfig := &kubeapiserveradmission.Config{
 		ExternalInformers:    versionedInformers,
@@ -356,6 +363,34 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 	return config, serviceResolver, pluginInitializers, nil
 }
 
+func validateCloudProviderOptions(opts *kubeoptions.CloudProviderOptions) error {
+	if opts.CloudProvider == "" {
+		return nil
+	}
+	if opts.CloudProvider == "external" {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableCloudProviders feature to true", opts.CloudProvider)
+		}
+		if !utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableKubeletCloudCredentialProviders feature to true", opts.CloudProvider)
+		}
+		return nil
+	} else if cloudprovider.IsDeprecatedInternal(opts.CloudProvider) {
+		if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableCloudProviders feature to false", opts.CloudProvider)
+		}
+		if utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) {
+			return fmt.Errorf("when using --cloud-provider set to '%s', "+
+				"please set DisableKubeletCloudCredentialProviders feature to false", opts.CloudProvider)
+		}
+		return nil
+	}
+	return fmt.Errorf("unknown --cloud-provider : %s", opts.CloudProvider)
+}
+
 var testServiceResolver webhook.ServiceResolver
 
 // SetServiceResolverForTests allows the service resolver to be overridden during tests.

pkg/features/kube_features.go

@@ -229,13 +229,14 @@ const (
 
 	// owner: @andrewsykim
 	// alpha: v1.22
-	// beta: v1.28
+	// beta: v1.29
 	//
 	// Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag.
 	DisableCloudProviders featuregate.Feature = "DisableCloudProviders"
 
 	// owner: @andrewsykim
 	// alpha: v1.23
+	// beta: v1.29
 	//
 	// Disable in-tree functionality in kubelet to authenticate to cloud provider container registries for image pull credentials.
 	DisableKubeletCloudCredentialProviders featuregate.Feature = "DisableKubeletCloudCredentialProviders"