Merge pull request #104921 from cheftako/anp-cp

Add mTLS as default HTTPConnect egress configuration for GCP.
This commit is contained in:
Kubernetes Prow Robot 2021-11-09 22:15:25 -08:00 committed by GitHub
commit e9a8bd94f7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 220 additions and 11 deletions

View File

@ -33,12 +33,14 @@ spec:
args: [
"--logtostderr=true",
"--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
__EXTRA_PARAMS__
"--proxy-server-host=__APISERVER_IP__",
"--proxy-server-port=8132",
"--sync-interval=5s",
"--sync-interval-cap=30s",
"--probe-interval=5s",
"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token"
"--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token",
"--agent-identifiers=ipv4=$(HOST_IP)"
]
env:
- name: POD_NAME
@ -49,12 +51,17 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
resources:
requests:
cpu: 50m
limits:
memory: 30Mi
volumeMounts:
__EXTRA_VOL_MNTS__
- mountPath: /var/run/secrets/tokens
name: konnectivity-agent-token
livenessProbe:
@ -65,6 +72,7 @@ spec:
timeoutSeconds: 15
serviceAccountName: konnectivity-agent
volumes:
__EXTRA_VOLS__
- name: konnectivity-agent-token
projected:
sources:

View File

@ -657,6 +657,16 @@ function create-node-pki {
KUBELET_KEY_PATH="${pki_dir}/kubelet.key"
write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}"
fi
if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then
mkdir -p "${pki_dir}/konnectivity-agent"
KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt"
KONNECTIVITY_AGENT_CLIENT_KEY_PATH="${pki_dir}/konnectivity-agent/client.key"
KONNECTIVITY_AGENT_CLIENT_CERT_PATH="${pki_dir}/konnectivity-agent/client.crt"
write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}"
write-pki-data "${KONNECTIVITY_AGENT_CLIENT_KEY}" "${KONNECTIVITY_AGENT_CLIENT_KEY_PATH}"
write-pki-data "${KONNECTIVITY_AGENT_CLIENT_CERT}" "${KONNECTIVITY_AGENT_CLIENT_CERT_PATH}"
fi
}
function create-master-pki {
@ -724,6 +734,39 @@ function create-master-pki {
PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt"
write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}"
fi
if [[ -n "${KONNECTIVITY_SERVER_CA_CERT:-}" ]]; then
mkdir -p "${pki_dir}"/konnectivity-server
KONNECTIVITY_SERVER_CA_CERT_PATH="${pki_dir}/konnectivity-server/ca.crt"
write-pki-data "${KONNECTIVITY_SERVER_CA_CERT}" "${KONNECTIVITY_SERVER_CA_CERT_PATH}"
KONNECTIVITY_SERVER_KEY_PATH="${pki_dir}/konnectivity-server/server.key"
write-pki-data "${KONNECTIVITY_SERVER_KEY}" "${KONNECTIVITY_SERVER_KEY_PATH}"
KONNECTIVITY_SERVER_CERT_PATH="${pki_dir}/konnectivity-server/server.crt"
write-pki-data "${KONNECTIVITY_SERVER_CERT}" "${KONNECTIVITY_SERVER_CERT_PATH}"
KONNECTIVITY_SERVER_CLIENT_KEY_PATH="${pki_dir}/konnectivity-server/client.key"
write-pki-data "${KONNECTIVITY_SERVER_CLIENT_KEY}" "${KONNECTIVITY_SERVER_CLIENT_KEY_PATH}"
KONNECTIVITY_SERVER_CLIENT_CERT_PATH="${pki_dir}/konnectivity-server/client.crt"
write-pki-data "${KONNECTIVITY_SERVER_CLIENT_CERT}" "${KONNECTIVITY_SERVER_CLIENT_CERT_PATH}"
fi
if [[ -n "${KONNECTIVITY_AGENT_CA_CERT:-}" ]]; then
mkdir -p "${pki_dir}"/konnectivity-agent
KONNECTIVITY_AGENT_CA_KEY_PATH="${pki_dir}/konnectivity-agent/ca.key"
write-pki-data "${KONNECTIVITY_AGENT_CA_KEY}" "${KONNECTIVITY_AGENT_CA_KEY_PATH}"
KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt"
write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}"
KONNECTIVITY_AGENT_KEY_PATH="${pki_dir}/konnectivity-agent/server.key"
write-pki-data "${KONNECTIVITY_AGENT_KEY}" "${KONNECTIVITY_AGENT_KEY_PATH}"
KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt"
write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}"
fi
}
# After the first boot and on upgrade, these files exist on the master-pd
@ -953,8 +996,12 @@ egressSelections:
connection:
proxyProtocol: HTTPConnect
transport:
uds:
udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket
tcp:
url: https://127.0.0.1:8131
tlsConfig:
caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt
clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key
clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt
- name: controlplane
connection:
proxyProtocol: Direct
@ -1461,7 +1508,6 @@ function create-master-etcd-apiserver-auth {
fi
}
function docker-installed {
if systemctl cat docker.service &> /dev/null ; then
return 0
@ -1944,26 +1990,44 @@ function prepare-konnectivity-server-manifest {
params+=("--log-file=/var/log/konnectivity-server.log")
params+=("--logtostderr=false")
params+=("--log-file-max-size=0")
params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket")
if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then
params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket")
elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then
# HTTP-CONNECT can work with either UDS or mTLS.
# Linking them here to make sure we get good coverage with two test configurations.
params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}")
params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}")
params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}")
params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}")
fi
params+=("--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt")
params+=("--cluster-key=/etc/srv/kubernetes/pki/apiserver.key")
if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then
params+=("--mode=grpc")
params+=("--server-port=0")
params+=("--agent-namespace=kube-system")
params+=("--agent-service-account=konnectivity-agent")
params+=("--authentication-audience=system:konnectivity-server")
params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig")
params+=("--proxy-strategies=default")
elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then
# GRPC can work with either UDS or mTLS.
params+=("--mode=http-connect")
params+=("--server-port=8131")
params+=("--agent-namespace=")
params+=("--agent-service-account=")
params+=("--authentication-audience=")
# Need to fix ANP code to allow kubeconfig to be set with mtls.
params+=("--kubeconfig=")
params+=("--proxy-strategies=destHost,default")
else
echo "KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE must be set to either grpc or http-connect"
exit 1
fi
params+=("--server-port=0")
params+=("--agent-port=$1")
params+=("--health-port=$2")
params+=("--admin-port=$3")
params+=("--agent-namespace=kube-system")
params+=("--agent-service-account=konnectivity-agent")
params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig")
params+=("--authentication-audience=system:konnectivity-server")
params+=("--kubeconfig-qps=75")
params+=("--kubeconfig-burst=150")
params+=("--keepalive-time=60s")
@ -2834,6 +2898,15 @@ EOF
function setup-konnectivity-agent-manifest {
local -r manifest="/etc/kubernetes/addons/konnectivity-agent/konnectivity-agent-ds.yaml"
sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}"
if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then
sed -i "s|__EXTRA_PARAMS__|\t\t\"--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt\",\n\t\t\"--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key\",|g" "${manifest}"
sed -i "s|__EXTRA_VOL_MNTS__| - name: pki\n mountPath: /etc/srv/kubernetes/pki/konnectivity-agent|g" "${manifest}"
sed -i "s|__EXTRA_VOLS__| - name: pki\n hostPath:\n path: /etc/srv/kubernetes/pki/konnectivity-agent|g" "${manifest}"
else
sed -i "s|__EXTRA_PARAMS__||g" "${manifest}"
sed -i "s|__EXTRA_VOL_MNTS__||g" "${manifest}"
sed -i "s|__EXTRA_VOLS__||g" "${manifest}"
fi
}
# Setups manifests for ingress controller and gce-specific policies for service controller.

View File

@ -1111,6 +1111,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}")
ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}")
ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}")
ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}")
KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}")
KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}")
KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}")
KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}")
KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}")
KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}")
KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}")
KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}")
KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}")
KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
EOF
}
@ -1261,6 +1273,9 @@ EOF
fi
if [[ "${master}" == "false" ]]; then
cat >>"$file" <<EOF
KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
EOF
fi
if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
@ -1675,6 +1690,7 @@ function create-certs {
setup-easyrsa
PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs
KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs
# By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
# Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
@ -1696,6 +1712,22 @@ function create-certs {
REQUESTHEADER_CA_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
PROXY_CLIENT_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | tr -d '\r\n')
PROXY_CLIENT_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | tr -d '\r\n')
# Setting up the Kubernetes API Server Konnectivity Server auth.
# This includes certs for both API Server to Konnectivity Server and
# Konnectivity Agent to Konnectivity Server.
KONNECTIVITY_SERVER_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
KONNECTIVITY_SERVER_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
KONNECTIVITY_SERVER_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
KONNECTIVITY_SERVER_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
KONNECTIVITY_AGENT_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
KONNECTIVITY_AGENT_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
KONNECTIVITY_AGENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
}
# Set up easy-rsa directory structure.
@ -1716,9 +1748,15 @@ function setup-easyrsa {
mkdir easy-rsa-master/kubelet
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet
mkdir easy-rsa-master/aggregator
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator) &>"${cert_create_debug_output}" || true
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator
mkdir easy-rsa-master/konnectivity-server
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server
mkdir easy-rsa-master/konnectivity-agent
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true
CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3"
AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator"
KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then
# TODO(roberthbailey,porridge): add better error handling here,
# see https://github.com/kubernetes/kubernetes/issues/55229
@ -1856,6 +1894,86 @@ function generate-aggregator-certs {
fi
}
# Runs the easy RSA commands to generate server side certificate files
# for the konnectivity server. This includes both server side to both
# konnectivity-server and konnectivity-agent.
# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and
# ${KONNECTIVITY_AGENT_CERT_DIR}
#
# Assumed vars
# KUBE_TEMP
# KONNECTIVITY_SERVER_CERT_DIR
# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name
# KONNECTIVITY_SERVER_SANS: Subject alternate names
#
function generate-konnectivity-server-certs {
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
# Note: This was heavily cribbed from make-ca-cert.sh
(set -x
# Make the client <-> konnectivity server side certificates.
cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
./easyrsa init-pki
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
./easyrsa build-client-full client nopass
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
# make the config for the signer
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
# create the konnectivity server cert with the correct groups
echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server
rm -f "konnectivity-server.csr"
# Make the agent <-> konnectivity server side certificates.
cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
./easyrsa init-pki
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
./easyrsa build-client-full client nopass
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
# make the config for the signer
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
# create the konnectivity server cert with the correct groups
echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
rm -f "konnectivity-agent.csr"
echo "completed main certificate section") &>"${cert_create_debug_output}" || true
local output_file_missing=0
local output_file
for output_file in \
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key"
do
if [[ ! -s "${output_file}" ]]; then
echo "Expected file ${output_file} not created" >&2
output_file_missing=1
fi
done
if (( output_file_missing )); then
# TODO(roberthbailey,porridge): add better error handling here,
# see https://github.com/kubernetes/kubernetes/issues/55229
cat "${cert_create_debug_output}" >&2
echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2
exit 2
fi
}
# Using provided master env, extracts value from provided key.
#
# Args:
@ -1897,6 +2015,16 @@ function parse-master-env() {
ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT")
ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY")
ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT")
KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY")
KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT")
KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")
KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY")
KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT")
KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY")
KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY")
KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT")
KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT")
KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY")
}
# Update or verify required gcloud components are installed