From 8820dc452230fc0e364ddf9b6b0920b8a4945f27 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Tue, 8 Dec 2020 11:37:13 -0500 Subject: [PATCH] Revert "iAdd host IP to etcd listen client URLs." This reverts commit 8b4e164a781822bbc5e8249b64346cb74280abf2. --- cluster/gce/config-default.sh | 5 ---- cluster/gce/config-test.sh | 5 ---- cluster/gce/gci/apiserver_etcd_test.go | 15 +--------- cluster/gce/gci/configure-helper.sh | 29 +++---------------- cluster/gce/gci/configure-kubeapiserver.sh | 17 +++-------- .../gci/testdata/kube-apiserver/etcd.template | 2 -- cluster/gce/manifests/etcd.manifest | 7 ++--- 7 files changed, 11 insertions(+), 69 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 44ab472732a..c38d924529b 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -551,11 +551,6 @@ export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}" # kube-apiserver is healthchecked on host IP instead of 127.0.0.1. export KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP="${KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP:-false}" -# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP, -# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers -# through host IP. -export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}" - # ETCD_PROGRESS_NOTIFY_INTERVAL defines the interval for etcd watch progress notify events. export ETCD_PROGRESS_NOTIFY_INTERVAL="${ETCD_PROGRESS_NOTIFY_INTERVAL:-10m}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 9b78547b842..b70775b5be5 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -587,11 +587,6 @@ export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}" # kube-apiserver is healthchecked on host IP instead of 127.0.0.1. export KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP="${KUBE_APISERVER_HEALTHCHECK_ON_HOST_IP:-false}" -# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP, -# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers -# through host IP. -export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}" - # ETCD_PROGRESS_NOTIFY_INTERVAL defines the interval for etcd watch progress notify events. export ETCD_PROGRESS_NOTIFY_INTERVAL="${ETCD_PROGRESS_NOTIFY_INTERVAL:-10m}" diff --git a/cluster/gce/gci/apiserver_etcd_test.go b/cluster/gce/gci/apiserver_etcd_test.go index c01fa612b73..c90936720e4 100644 --- a/cluster/gce/gci/apiserver_etcd_test.go +++ b/cluster/gce/gci/apiserver_etcd_test.go @@ -37,8 +37,6 @@ type kubeAPIServeETCDEnv struct { StorageBackend string StorageMediaType string CompactionInterval string - HostPrimaryIP string - ETCDListenOnHostIP string } func TestServerOverride(t *testing.T) { @@ -54,7 +52,7 @@ func TestServerOverride(t *testing.T) { }, }, { - desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES are set", + desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES iare set", env: kubeAPIServeETCDEnv{ ETCDServers: "ETCDServers", ETCDServersOverride: "ETCDServersOverrides", @@ -63,17 +61,6 @@ func TestServerOverride(t *testing.T) { "--etcd-servers-overrides=ETCDServersOverrides", }, }, - { - desc: "HOST_PRIMARY_IP is set and etcd is set to listen to host IP", - env: kubeAPIServeETCDEnv{ - HostPrimaryIP: "HostPrimaryIP", - ETCDListenOnHostIP: "true", - }, - want: []string{ - "--etcd-servers-overrides=/events#http://HostPrimaryIP:4002", - "--etcd-servers=http://HostPrimaryIP:2379", - }, - }, } for _, tc := range testCases { diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 098d849b32b..89ee145a55d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1702,8 +1702,8 @@ function start-kube-proxy { # Replaces the variables in the etcd manifest file with the real values, and then # copy the file to the manifest dir # $1: value for variable 'suffix' -# $2: value for variable 'port', for listening to clients -# $3: value for variable 'server_port', for etcd peering +# $2: value for variable 'port' +# $3: value for variable 'server_port' # $4: value for variable 'cpulimit' # $5: pod name, which should be either etcd or etcd-events function prepare-etcd-manifest { @@ -1722,43 +1722,23 @@ function prepare-etcd-manifest { if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then cluster_state="${INITIAL_ETCD_CLUSTER_STATE}" fi - - # Configure mTLS for etcd peers. if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth " etcd_protocol="https" fi - # host_primary_ip is the primary internal IP of the host. - # Override host primary IP if specifically provided. - local host_primary_ip - host_primary_ip="${HOST_PRIMARY_IP:-$(hostname -i)}" - - # Configure mTLS for clients (e.g. kube-apiserver). - # mTLS should only be enabled for etcd server but not etcd-events. If $1 suffix is empty, it's etcd server. - local etcd_listen_metrics_urls="" + # mTLS should only be enabled for etcd server but not etcd-events. if $1 suffix is empty, it's etcd server. if [[ -z "${suffix}" && -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} " etcd_apiserver_protocol="https" etcd_livenessprobe_port="2382" - etcd_listen_metrics_urls="http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port}" - if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]]; then - etcd_listen_metrics_urls+=",http://${host_primary_ip}:${etcd_livenessprobe_port}" - fi - etcd_extra_args+=" --listen-metrics-urls=${etcd_listen_metrics_urls} " + etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port} " fi if [[ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]]; then etcd_extra_args+=" --experimental-watch-progress-notify-interval=${ETCD_PROGRESS_NOTIFY_INTERVAL}" fi - # If etcd is configured to listen on host IP, an additional client listening URL is added. - local etcd_listen_client_urls="${etcd_apiserver_protocol}://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:$2" - if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then - etcd_listen_client_urls+=",${etcd_apiserver_protocol}://${host_primary_ip}:$2" - fi - - # Generate etcd member URLs. for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3" if [[ -n "${etcd_cluster}" ]]; then @@ -1778,7 +1758,6 @@ function prepare-etcd-manifest { sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}" sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}" sed -i -e "s@{{ *listen_client_ip *}}@${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}@g" "${temp_file}" - sed -i -e "s@{{ *etcd_listen_client_urls *}}@${etcd_listen_client_urls:-}@g" "${temp_file}" # Get default storage backend from manifest file. local -r default_storage_backend=$( \ grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" "${temp_file}" | \ diff --git a/cluster/gce/gci/configure-kubeapiserver.sh b/cluster/gce/gci/configure-kubeapiserver.sh index 5010e6cf3f4..045ea62b587 100644 --- a/cluster/gce/gci/configure-kubeapiserver.sh +++ b/cluster/gce/gci/configure-kubeapiserver.sh @@ -14,34 +14,25 @@ # limitations under the License. -# Configures etcd related parameters of kube-apiserver. +# Configures etcd related flags of kube-apiserver. function configure-etcd-params { local -n params_ref=$1 - local host_ip="127.0.0.1" - # If etcd is configured to listen on host IP, - # host_ip is set to the primary internal IP of host VM. - if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then - host_ip="${HOST_PRIMARY_IP:-$(hostname -i)}" - fi - - # Configure the main etcd. if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then - params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://${host_ip}:2379}" + params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}" params_ref+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" params_ref+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" params_ref+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then - params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://${host_ip}:2379}" + params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled." else echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential." exit 1 fi - # Configure the event log etcd. if [[ -z "${ETCD_SERVERS:-}" ]]; then - params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://${host_ip}:4002}" + params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}" elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" fi diff --git a/cluster/gce/gci/testdata/kube-apiserver/etcd.template b/cluster/gce/gci/testdata/kube-apiserver/etcd.template index b210688d38d..1624451c56b 100644 --- a/cluster/gce/gci/testdata/kube-apiserver/etcd.template +++ b/cluster/gce/gci/testdata/kube-apiserver/etcd.template @@ -13,5 +13,3 @@ readonly ETCD_SERVERS_OVERRIDES={{.ETCDServersOverride}} readonly STORAGE_BACKEND={{.StorageBackend}} readonly STORAGE_MEDIA_TYPE={{.StorageMediaType}} readonly ETCD_COMPACTION_INTERVAL_SEC={{.CompactionInterval}} -readonly HOST_PRIMARY_IP={{.HostPrimaryIP}} -readonly ETCD_LISTEN_ON_HOST_IP={{.ETCDListenOnHostIP}} diff --git a/cluster/gce/manifests/etcd.manifest b/cluster/gce/manifests/etcd.manifest index adac6ad7e61..664883ddf92 100644 --- a/cluster/gce/manifests/etcd.manifest +++ b/cluster/gce/manifests/etcd.manifest @@ -26,7 +26,7 @@ "command": [ "/bin/sh", "-c", - "set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_listen_client_urls }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + "set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_apiserver_protocol }}://{{ listen_client_ip }}:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" ], "env": [ { "name": "TARGET_STORAGE", @@ -58,11 +58,8 @@ }, { "name": "ETCD_HOSTNAME", "value": "{{ hostname }}" - }, - { "name": "LISTEN_CLIENT_URLS", - "value": "{{ etcd_listen_client_urls }}" } - ], + ], "livenessProbe": { "httpGet": { "host": "127.0.0.1",