mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-15 14:14:39 +00:00
Add mTLS as default HTTPConnect egress configuration for GCP.
We currently have UDS as the configuration with GRPC. Some users are setting up egress to remote konnectivity servers. Cannot use UDS for this configuration. Should have a config setup which validates the mTLS configuration. Fixed lint errors from shell check. Fix volumes to not include pki for ANP in grpc mode.
This commit is contained in:
Walter Fender
@@ -1111,6 +1111,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}")
|
||||
ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}")
|
||||
ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}")
|
||||
ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}")
|
||||
KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}")
|
||||
KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}")
|
||||
KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}")
|
||||
KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}")
|
||||
KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}")
|
||||
KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -1261,6 +1273,9 @@ EOF
|
||||
fi
|
||||
if [[ "${master}" == "false" ]]; then
|
||||
cat >>"$file" <<EOF
|
||||
KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}")
|
||||
KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}")
|
||||
EOF
|
||||
fi
|
||||
if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
|
||||
@@ -1675,6 +1690,7 @@ function create-certs {
|
||||
setup-easyrsa
|
||||
PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs
|
||||
AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs
|
||||
KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs
|
||||
|
||||
# By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces.
|
||||
# Note 'base64 -w0' doesn't work on Mac OS X, which has different flags.
|
||||
@@ -1696,6 +1712,22 @@ function create-certs {
|
||||
REQUESTHEADER_CA_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
||||
PROXY_CLIENT_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | tr -d '\r\n')
|
||||
PROXY_CLIENT_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | tr -d '\r\n')
|
||||
|
||||
# Setting up the Kubernetes API Server Konnectivity Server auth.
|
||||
# This includes certs for both API Server to Konnectivity Server and
|
||||
# Konnectivity Agent to Konnectivity Server.
|
||||
KONNECTIVITY_SERVER_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
||||
KONNECTIVITY_SERVER_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
||||
KONNECTIVITY_SERVER_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
|
||||
KONNECTIVITY_SERVER_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
|
||||
KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
|
||||
KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
|
||||
KONNECTIVITY_AGENT_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n')
|
||||
KONNECTIVITY_AGENT_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | tr -d '\r\n')
|
||||
KONNECTIVITY_AGENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n')
|
||||
KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n')
|
||||
KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n')
|
||||
KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n')
|
||||
}
|
||||
|
||||
# Set up easy-rsa directory structure.
|
||||
@@ -1716,9 +1748,15 @@ function setup-easyrsa {
|
||||
mkdir easy-rsa-master/kubelet
|
||||
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet
|
||||
mkdir easy-rsa-master/aggregator
|
||||
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator) &>"${cert_create_debug_output}" || true
|
||||
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator
|
||||
mkdir easy-rsa-master/konnectivity-server
|
||||
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server
|
||||
mkdir easy-rsa-master/konnectivity-agent
|
||||
cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true
|
||||
CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3"
|
||||
AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator"
|
||||
KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
|
||||
KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
|
||||
if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then
|
||||
# TODO(roberthbailey,porridge): add better error handling here,
|
||||
# see https://github.com/kubernetes/kubernetes/issues/55229
|
||||
@@ -1856,6 +1894,86 @@ function generate-aggregator-certs {
|
||||
fi
|
||||
}
|
||||
|
||||
# Runs the easy RSA commands to generate server side certificate files
|
||||
# for the konnectivity server. This includes both server side to both
|
||||
# konnectivity-server and konnectivity-agent.
|
||||
# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and
|
||||
# ${KONNECTIVITY_AGENT_CERT_DIR}
|
||||
#
|
||||
# Assumed vars
|
||||
# KUBE_TEMP
|
||||
# KONNECTIVITY_SERVER_CERT_DIR
|
||||
# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name
|
||||
# KONNECTIVITY_SERVER_SANS: Subject alternate names
|
||||
#
|
||||
function generate-konnectivity-server-certs {
|
||||
local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
|
||||
# Note: This was heavily cribbed from make-ca-cert.sh
|
||||
(set -x
|
||||
# Make the client <-> konnectivity server side certificates.
|
||||
cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server"
|
||||
./easyrsa init-pki
|
||||
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
||||
./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
||||
./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
|
||||
./easyrsa build-client-full client nopass
|
||||
|
||||
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
||||
|
||||
# make the config for the signer
|
||||
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
|
||||
# create the konnectivity server cert with the correct groups
|
||||
echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server
|
||||
rm -f "konnectivity-server.csr"
|
||||
|
||||
# Make the agent <-> konnectivity server side certificates.
|
||||
cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent"
|
||||
./easyrsa init-pki
|
||||
# this puts the cert into pki/ca.crt and the key into pki/private/ca.key
|
||||
./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass
|
||||
./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass
|
||||
./easyrsa build-client-full client nopass
|
||||
|
||||
kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl"
|
||||
|
||||
# make the config for the signer
|
||||
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json"
|
||||
# create the konnectivity server cert with the correct groups
|
||||
echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent
|
||||
rm -f "konnectivity-agent.csr"
|
||||
|
||||
echo "completed main certificate section") &>"${cert_create_debug_output}" || true
|
||||
|
||||
local output_file_missing=0
|
||||
local output_file
|
||||
for output_file in \
|
||||
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \
|
||||
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \
|
||||
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \
|
||||
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \
|
||||
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \
|
||||
"${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \
|
||||
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \
|
||||
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \
|
||||
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \
|
||||
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \
|
||||
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \
|
||||
"${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key"
|
||||
do
|
||||
if [[ ! -s "${output_file}" ]]; then
|
||||
echo "Expected file ${output_file} not created" >&2
|
||||
output_file_missing=1
|
||||
fi
|
||||
done
|
||||
if (( output_file_missing )); then
|
||||
# TODO(roberthbailey,porridge): add better error handling here,
|
||||
# see https://github.com/kubernetes/kubernetes/issues/55229
|
||||
cat "${cert_create_debug_output}" >&2
|
||||
echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
# Using provided master env, extracts value from provided key.
|
||||
#
|
||||
# Args:
|
||||
@@ -1897,6 +2015,16 @@ function parse-master-env() {
|
||||
ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT")
|
||||
ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY")
|
||||
ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT")
|
||||
KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY")
|
||||
KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT")
|
||||
KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")
|
||||
KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY")
|
||||
KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT")
|
||||
KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY")
|
||||
KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY")
|
||||
KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT")
|
||||
KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT")
|
||||
KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY")
|
||||
}
|
||||
|
||||
# Update or verify required gcloud components are installed
|
||||
|
||||
Reference in New Issue
Block a user