From 8f6a79b29fa58b3ea1bc7b0002b2776666c7b238 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sat, 17 Dec 2022 18:05:57 +0000 Subject: [PATCH] use external cloud provider by default with gce This change enables the external cloud provider by default for GCE infrastructure. It is the result of several squashed commits, these are their commit messages: * no longer need to enable endpoinslices controller since GA * use external by default * DisableKubeletCloudCredentialProviders * temp test feature gates --- cluster/gce/config-default.sh | 18 +++++++++++------- cluster/gce/config-test.sh | 18 +++++++++++------- cluster/gce/gci/configure-helper.sh | 4 ++-- cluster/gce/gci/configure-kubeapiserver.sh | 2 +- cluster/gce/util.sh | 2 +- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index b797683f040..124c36cd4ed 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -256,11 +256,6 @@ if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then export RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}" fi -# If feature gates includes AllAlpha or EndpointSlice, and EndpointSlice has not been disabled, add EndpointSlice controller to list of controllers to run. -if [[ (( "${KUBE_FEATURE_GATES:-}" == *"AllAlpha=true"* ) || ( "${KUBE_FEATURE_GATES:-}" == *"EndpointSlice=true"* )) && "${KUBE_FEATURE_GATES:-}" != *"EndpointSlice=false"* ]]; then - RUN_CONTROLLERS="${RUN_CONTROLLERS:-*,endpointslice}" -fi - # By default disable gkenetworkparamset controller in CCM RUN_CCM_CONTROLLERS="${RUN_CCM_CONTROLLERS:-*,-gkenetworkparamset}" @@ -556,7 +551,7 @@ export TLS_CIPHER_SUITES="" # CLOUD_PROVIDER_FLAG defines the cloud-provider value presented to KCM, apiserver, # and kubelet -export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-gce}" +export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}" # When ENABLE_AUTH_PROVIDER_GCP is set, following flags for out-of-tree credential provider for GCP # are presented to kubelet: @@ -564,4 +559,13 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-gce}" # --image-credential-provider-bin-dir=${path-to-auth-provider-binary} # Also, it is required that DisableKubeletCloudCredentialProviders # feature gates are set to true for kubelet to use external credential provider. -ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" +export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" + +# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags +# DisableKubeletCloudCredentialProviders and DisableCloudProviders +if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then + export ENABLE_AUTH_PROVIDER_GCP=true + if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then + export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True" + fi +fi diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index e93c9a8e682..9ed43a34049 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -157,11 +157,6 @@ if [[ "${KUBE_FEATURE_GATES:-}" = 'AllAlpha=true' ]]; then RUNTIME_CONFIG=${KUBE_RUNTIME_CONFIG:-api/all=true} fi -# If feature gates includes AllAlpha or EndpointSlice, and EndpointSlice has not been disabled, add EndpointSlice controller to list of controllers to run. -if [[ (( "${KUBE_FEATURE_GATES:-}" = *"AllAlpha=true"* ) || ( "${KUBE_FEATURE_GATES:-}" = *"EndpointSlice=true"* )) && "${KUBE_FEATURE_GATES:-}" != *"EndpointSlice=false"* ]]; then - RUN_CONTROLLERS=${RUN_CONTROLLERS:-*,endpointslice} -fi - # By default disable gkenetworkparamset controller in CCM RUN_CCM_CONTROLLERS="${RUN_CCM_CONTROLLERS:-*,-gkenetworkparamset}" @@ -605,7 +600,7 @@ export TLS_CIPHER_SUITES="" # CLOUD_PROVIDER_FLAG defines the cloud-provider value presented to KCM, apiserver, # and kubelet -export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-gce}" +export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-external}" # When ENABLE_AUTH_PROVIDER_GCP is set, following flags for out-of-tree credential provider for GCP # are presented to kubelet: @@ -613,4 +608,13 @@ export CLOUD_PROVIDER_FLAG="${CLOUD_PROVIDER_FLAG:-gce}" # --image-credential-provider-bin-dir=${path-to-auth-provider-binary} # Also, it is required that DisableKubeletCloudCredentialProviders and KubeletCredentialProviders # feature gates are set to true for kubelet to use external credential provider. -ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-true}" +export ENABLE_AUTH_PROVIDER_GCP="${ENABLE_AUTH_PROVIDER_GCP:-false}" + +# External cloud provider requires ENABLE_AUTH_PROVIDER_GCP and feature flags +# DisableKubeletCloudCredentialProviders and DisableCloudProviders +if [[ "${CLOUD_PROVIDER_FLAG:-}" == "external" ]]; then + export ENABLE_AUTH_PROVIDER_GCP=true + if [[ -n "${FEATURE_GATES:-DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True}" ]]; then + export FEATURE_GATES="${FEATURE_GATES},DisableKubeletCloudCredentialProviders=True,DisableCloudProviders=True" + fi +fi \ No newline at end of file diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 51f21dd7642..603ab6b171b 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2157,7 +2157,7 @@ function start-kube-controller-manager { local params=("${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"}" "${CONTROLLER_MANAGER_TEST_ARGS:-}" "${CLOUD_CONFIG_OPT}") local config_path='/etc/srv/kubernetes/kube-controller-manager/kubeconfig' params+=("--use-service-account-credentials") - params+=("--cloud-provider=${CLOUD_PROVIDER_FLAG:-gce}") + params+=("--cloud-provider=${CLOUD_PROVIDER_FLAG:-external}") params+=("--kubeconfig=${config_path}" "--authentication-kubeconfig=${config_path}" "--authorization-kubeconfig=${config_path}") params+=("--root-ca-file=${CA_CERT_BUNDLE_PATH}") params+=("--service-account-private-key-file=${SERVICEACCOUNT_KEY_PATH}") @@ -3572,7 +3572,7 @@ function main() { log-wrap 'StartKonnectivityServer' start-konnectivity-server fi log-wrap 'StartKubeControllerManager' start-kube-controller-manager - if [[ "${CLOUD_PROVIDER_FLAG:-gce}" == "external" ]]; then + if [[ "${CLOUD_PROVIDER_FLAG:-external}" == "external" ]]; then log-wrap 'StartCloudControllerManager' start-cloud-controller-manager fi log-wrap 'StartKubeScheduler' start-kube-scheduler diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index aaf68112e56..db39e47eda8 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -68,7 +68,7 @@ function start-kube-apiserver { # Calculate variables and assemble the command line. local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" params+=" --allow-privileged=true" - params+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-gce}" + params+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}" params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" # params is passed by reference, so no "$" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 2a0554cc7e7..1f6aa522227 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -745,7 +745,7 @@ function yaml-map-string-string { # Returns kubelet flags used on both Linux and Windows nodes. function construct-common-kubelet-flags { local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}" - flags+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-gce}" + flags+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}" # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it? if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then flags+=" --rotate-certificates=true"