From cdcbfcc0a6a02a83fdc3b98d478abaec6c6fa32e Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 20 Apr 2023 09:22:40 -0400 Subject: [PATCH 1/3] [KEP-2395] Phase 4 - Disabling In-Tree Providers https://github.com/kubernetes/enhancements/tree/master/keps/sig-cloud-provider/2395-removing-in-tree-cloud-providers#phase-4---disabling-in-tree-providers DisableCloudProviders - this feature gate will disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the --cloud-provider component flag. DisableKubeletCloudCredentialProvider - this feature gate will disable in-tree functionality in the kubelet to authenticate to the Azure and GCP container registries for image pull credentials. Signed-off-by: Davanum Srinivas --- pkg/features/kube_features.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 7d07dba6e36..7ea31692dcc 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -1012,9 +1012,9 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS DefaultHostNetworkHostPortsInPodTemplates: {Default: false, PreRelease: featuregate.Deprecated}, - DisableCloudProviders: {Default: false, PreRelease: featuregate.Alpha}, + DisableCloudProviders: {Default: true, PreRelease: featuregate.Beta}, - DisableKubeletCloudCredentialProviders: {Default: false, PreRelease: featuregate.Alpha}, + DisableKubeletCloudCredentialProviders: {Default: true, PreRelease: featuregate.Beta}, DevicePluginCDIDevices: {Default: false, PreRelease: featuregate.Alpha}, From 42e8cfa28acddf6b8820a293111b25409c3d82c7 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 20 Apr 2023 10:41:32 -0400 Subject: [PATCH 2/3] fix failing metadata test Signed-off-by: Davanum Srinivas --- pkg/credentialprovider/gcp/metadata_test.go | 6 ++++++ pkg/features/kube_features.go | 1 + 2 files changed, 7 insertions(+) diff --git a/pkg/credentialprovider/gcp/metadata_test.go b/pkg/credentialprovider/gcp/metadata_test.go index 130da5bb1c6..6de2dc7b838 100644 --- a/pkg/credentialprovider/gcp/metadata_test.go +++ b/pkg/credentialprovider/gcp/metadata_test.go @@ -30,7 +30,10 @@ import ( "testing" utilnet "k8s.io/apimachinery/pkg/util/net" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/credentialprovider" + kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/legacy-cloud-providers/gce/gcpcredential" ) @@ -53,6 +56,9 @@ func TestMetadata(t *testing.T) { if runtime.GOOS == "windows" && !onGCEVM() { t.Skip("Skipping test on Windows, not on GCE.") } + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.DisableKubeletCloudCredentialProviders, false)() + var err error gceProductNameFile, err = createProductNameFile() if err != nil { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 7ea31692dcc..6782d2b909a 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -229,6 +229,7 @@ const ( // owner: @andrewsykim // alpha: v1.22 + // beta: v1.28 // // Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag. DisableCloudProviders featuregate.Feature = "DisableCloudProviders" From ceaed508ce71513ab7a2315e7420f773a9ca027e Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Fri, 1 Sep 2023 07:01:05 -0400 Subject: [PATCH 3/3] Validate the cloud-provider passed in and the corresponding feature flags Signed-off-by: Davanum Srinivas --- cluster/gce/config-default.sh | 11 +--------- cluster/gce/config-test.sh | 11 +--------- cmd/kube-apiserver/app/server.go | 35 ++++++++++++++++++++++++++++++++ pkg/features/kube_features.go | 3 ++- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 124c36cd4ed..91c6dec3162 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -559,13 +559,4 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}" # --image-credential-provider-bin-dir=${path-to-auth-provider-binary} # Also, it is required that DisableKubeletCloudCredentialProviders # feature gates are set to true for kubelet to use external credential provider. -export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" - -# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags -# DisableKubeletCloudCredentialProviders and DisableCloudProviders -if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then - export ENABLE_AUTH_PROVIDER_GCP=true - if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then - export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True" - fi -fi +export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" \ No newline at end of file diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 9ed43a34049..fe684359f4c 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -608,13 +608,4 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}" # --image-credential-provider-bin-dir=${path-to-auth-provider-binary} # Also, it is required that DisableKubeletCloudCredentialProviders and KubeletCredentialProviders # feature gates are set to true for kubelet to use external credential provider. -export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" - -# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags -# DisableKubeletCloudCredentialProviders and DisableCloudProviders -if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then - export ENABLE_AUTH_PROVIDER_GCP=true - if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then - export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True" - fi -fi \ No newline at end of file +export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" \ No newline at end of file diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 807c169631f..04deb53bee0 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -46,6 +46,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/keyutil" + cloudprovider "k8s.io/cloud-provider" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/cli/globalflag" "k8s.io/component-base/logs" @@ -67,6 +68,7 @@ import ( "k8s.io/kubernetes/pkg/controlplane/reconcilers" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" "k8s.io/kubernetes/pkg/serviceaccount" ) @@ -292,6 +294,11 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders } + err = validateCloudProviderOptions(opts.CloudProvider) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to validate cloud provider: %w", err) + } + // setup admission admissionConfig := &kubeapiserveradmission.Config{ ExternalInformers: versionedInformers, @@ -356,6 +363,34 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( return config, serviceResolver, pluginInitializers, nil } +func validateCloudProviderOptions(opts *kubeoptions.CloudProviderOptions) error { + if opts.CloudProvider == "" { + return nil + } + if opts.CloudProvider == "external" { + if !utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) { + return fmt.Errorf("when using --cloud-provider set to '%s', "+ + "please set DisableCloudProviders feature to true", opts.CloudProvider) + } + if !utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) { + return fmt.Errorf("when using --cloud-provider set to '%s', "+ + "please set DisableKubeletCloudCredentialProviders feature to true", opts.CloudProvider) + } + return nil + } else if cloudprovider.IsDeprecatedInternal(opts.CloudProvider) { + if utilfeature.DefaultFeatureGate.Enabled(features.DisableCloudProviders) { + return fmt.Errorf("when using --cloud-provider set to '%s', "+ + "please set DisableCloudProviders feature to false", opts.CloudProvider) + } + if utilfeature.DefaultFeatureGate.Enabled(features.DisableKubeletCloudCredentialProviders) { + return fmt.Errorf("when using --cloud-provider set to '%s', "+ + "please set DisableKubeletCloudCredentialProviders feature to false", opts.CloudProvider) + } + return nil + } + return fmt.Errorf("unknown --cloud-provider : %s", opts.CloudProvider) +} + var testServiceResolver webhook.ServiceResolver // SetServiceResolverForTests allows the service resolver to be overridden during tests. diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 6782d2b909a..2548fe6f2b2 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -229,13 +229,14 @@ const ( // owner: @andrewsykim // alpha: v1.22 - // beta: v1.28 + // beta: v1.29 // // Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag. DisableCloudProviders featuregate.Feature = "DisableCloudProviders" // owner: @andrewsykim // alpha: v1.23 + // beta: v1.29 // // Disable in-tree functionality in kubelet to authenticate to cloud provider container registries for image pull credentials. DisableKubeletCloudCredentialProviders featuregate.Feature = "DisableKubeletCloudCredentialProviders"