github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 11:21:47 +00:00
Code Issues Projects Releases Wiki Activity

iAdd host IP to etcd listen client URLs.

Browse Source 
Allow kube-apiserver to use host IP to connect to etcd.
Update etcd/migrate to allow additional client listening URLs.
This commit is contained in:
Ben Hu 2020-10-05 20:32:27 +00:00
parent d5ea0edc09
commit 8b4e164a78
7 changed files with 69 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cluster/gce/config-default.sh
View File

@ -524,3 +524,8 @@ export KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE="${KUBE_KONNECTIVITY_SERVICE_PRO
# Optional: Enable Windows CSI-Proxy
export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"
# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP,
# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers
# through host IP.
export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}"

5
cluster/gce/config-test.sh
View File

@ -571,3 +571,8 @@ export GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA=true
# Optoinal: Enable Windows CSI-Proxy
export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"
# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP,
# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers
# through host IP.
export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}"

15
cluster/gce/gci/apiserver_etcd_test.go
View File

@ -37,6 +37,8 @@ type kubeAPIServeETCDEnv struct {
StorageBackend string
StorageMediaType string
CompactionInterval string
HostPrimaryIP string
ETCDListenOnHostIP string
}
func TestServerOverride(t *testing.T) {
@ -52,7 +54,7 @@ func TestServerOverride(t *testing.T) {
},
},
{
desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES iare set",
desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES are set",
env: kubeAPIServeETCDEnv{
ETCDServers: "ETCDServers",
ETCDServersOverride: "ETCDServersOverrides",
@ -61,6 +63,17 @@ func TestServerOverride(t *testing.T) {
"--etcd-servers-overrides=ETCDServersOverrides",
},
},
{
desc: "HOST_PRIMARY_IP is set and etcd is set to listen to host IP",
env: kubeAPIServeETCDEnv{
HostPrimaryIP: "HostPrimaryIP",
ETCDListenOnHostIP: "true",
},
want: []string{
"--etcd-servers-overrides=/events#http://HostPrimaryIP:4002",
"--etcd-servers=http://HostPrimaryIP:2379",
},
},
}
for _, tc := range testCases {

29
cluster/gce/gci/configure-helper.sh
View File

@ -1686,8 +1686,8 @@ function start-kube-proxy {
# Replaces the variables in the etcd manifest file with the real values, and then
# copy the file to the manifest dir
# $1: value for variable 'suffix'
# $2: value for variable 'port'
# $3: value for variable 'server_port'
# $2: value for variable 'port', for listening to clients
# $3: value for variable 'server_port', for etcd peering
# $4: value for variable 'cpulimit'
# $5: pod name, which should be either etcd or etcd-events
function prepare-etcd-manifest {
@ -1706,19 +1706,39 @@ function prepare-etcd-manifest {
if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
fi
# Configure mTLS for etcd peers.
if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
etcd_protocol="https"
fi
# mTLS should only be enabled for etcd server but not etcd-events. if $1 suffix is empty, it's etcd server.
# host_primary_ip is the primary internal IP of the host.
# Override host primary IP if specifically provided.
local host_primary_ip
host_primary_ip="${HOST_PRIMARY_IP:-$(hostname -i)}"
# Configure mTLS for clients (e.g. kube-apiserver).
# mTLS should only be enabled for etcd server but not etcd-events. If $1 suffix is empty, it's etcd server.
local etcd_listen_metrics_urls=""
if [[ -z "${suffix}" && -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} "
etcd_apiserver_protocol="https"
etcd_livenessprobe_port="2382"
etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port} "
etcd_listen_metrics_urls="http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port}"
if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]]; then
etcd_listen_metrics_urls+=",http://${host_primary_ip}:${etcd_livenessprobe_port}"
fi
etcd_extra_args+=" --listen-metrics-urls=${etcd_listen_metrics_urls} "
fi
# If etcd is configured to listen on host IP, an additional client listening URL is added.
local etcd_listen_client_urls="${etcd_apiserver_protocol}://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:$2"
if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then
etcd_listen_client_urls+=",${etcd_apiserver_protocol}://${host_primary_ip}:$2"
fi
# Generate etcd member URLs.
for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
if [[ -n "${etcd_cluster}" ]]; then
@ -1738,6 +1758,7 @@ function prepare-etcd-manifest {
sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}"
sed -i -e "s@{{ *listen_client_ip *}}@${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}@g" "${temp_file}"
sed -i -e "s@{{ *etcd_listen_client_urls *}}@${etcd_listen_client_urls:-}@g" "${temp_file}"
# Get default storage backend from manifest file.
local -r default_storage_backend=$( \
grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" "${temp_file}" | \

17
cluster/gce/gci/configure-kubeapiserver.sh
View File

@ -14,25 +14,34 @@
# limitations under the License.
# Configures etcd related flags of kube-apiserver.
# Configures etcd related parameters of kube-apiserver.
function configure-etcd-params {
local -n params_ref=$1
local host_ip="127.0.0.1"
# If etcd is configured to listen on host IP,
# host_ip is set to the primary internal IP of host VM.
if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then
host_ip="${HOST_PRIMARY_IP:-$(hostname -i)}"
fi
# Configure the main etcd.
if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}"
params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://${host_ip}:2379}"
params_ref+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}"
params_ref+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}"
params_ref+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}"
elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}"
params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://${host_ip}:2379}"
echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled."
else
echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential."
exit 1
fi
# Configure the event log etcd.
if [[ -z "${ETCD_SERVERS:-}" ]]; then
params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}"
params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://${host_ip}:4002}"
elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then
params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}"
fi

2
cluster/gce/gci/testdata/kube-apiserver/etcd.template vendored
View File

@ -13,3 +13,5 @@ readonly ETCD_SERVERS_OVERRIDES={{.ETCDServersOverride}}
readonly STORAGE_BACKEND={{.StorageBackend}}
readonly STORAGE_MEDIA_TYPE={{.StorageMediaType}}
readonly ETCD_COMPACTION_INTERVAL_SEC={{.CompactionInterval}}
readonly HOST_PRIMARY_IP={{.HostPrimaryIP}}
readonly ETCD_LISTEN_ON_HOST_IP={{.ETCDListenOnHostIP}}

7
cluster/gce/manifests/etcd.manifest
View File

@ -26,7 +26,7 @@
"command": [
"/bin/sh",
"-c",
"set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_apiserver_protocol }}://{{ listen_client_ip }}:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
"set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_listen_client_urls }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
],
"env": [
{ "name": "TARGET_STORAGE",
@ -58,8 +58,11 @@
},
{ "name": "ETCD_HOSTNAME",
"value": "{{ hostname }}"
},
{ "name": "LISTEN_CLIENT_URLS",
"value": "{{ etcd_listen_client_urls }}"
}
],
],
"livenessProbe": {
"httpGet": {
"host": "127.0.0.1",