From f7185b0be1a70ee816c47e15f3abb0ca483a8622 Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Mon, 6 Sep 2021 17:31:44 -0700 Subject: [PATCH] Add mTLS as default HTTPConnect egress configuration for GCP. We currently have UDS as the configuration with GRPC. Some users are setting up egress to remote konnectivity servers. Cannot use UDS for this configuration. Should have a config setup which validates the mTLS configuration. Fixed lint errors from shell check. Fix volumes to not include pki for ANP in grpc mode. --- .../konnectivity-agent-ds.yaml | 10 +- cluster/gce/gci/configure-helper.sh | 90 ++++++++++-- cluster/gce/util.sh | 130 +++++++++++++++++- 3 files changed, 219 insertions(+), 11 deletions(-) diff --git a/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml b/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml index 7805acbdc66..cef0e685c58 100644 --- a/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml +++ b/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml @@ -33,12 +33,14 @@ spec: args: [ "--logtostderr=true", "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", +__EXTRA_PARAMS__ "--proxy-server-host=__APISERVER_IP__", "--proxy-server-port=8132", "--sync-interval=5s", "--sync-interval-cap=30s", "--probe-interval=5s", - "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token" + "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", + "--agent-identifiers=ipv4=$(HOST_IP)" ] env: - name: POD_NAME @@ -49,11 +51,16 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP resources: limits: cpu: 50m memory: 30Mi volumeMounts: +__EXTRA_VOL_MNTS__ - mountPath: /var/run/secrets/tokens name: konnectivity-agent-token livenessProbe: @@ -64,6 +71,7 @@ spec: timeoutSeconds: 15 serviceAccountName: konnectivity-agent volumes: +__EXTRA_VOLS__ - name: konnectivity-agent-token projected: sources: diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 358811cf615..177184b26fb 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -657,6 +657,16 @@ function create-node-pki { KUBELET_KEY_PATH="${pki_dir}/kubelet.key" write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}" fi + + if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then + mkdir -p "${pki_dir}/konnectivity-agent" + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + KONNECTIVITY_AGENT_CLIENT_KEY_PATH="${pki_dir}/konnectivity-agent/client.key" + KONNECTIVITY_AGENT_CLIENT_CERT_PATH="${pki_dir}/konnectivity-agent/client.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_KEY}" "${KONNECTIVITY_AGENT_CLIENT_KEY_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_CERT}" "${KONNECTIVITY_AGENT_CLIENT_CERT_PATH}" + fi } function create-master-pki { @@ -724,6 +734,42 @@ function create-master-pki { PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt" write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}" fi + + if [[ -n "${KONNECTIVITY_SERVER_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-server + #KONNECTIVITY_SERVER_CA_KEY_PATH="${pki_dir}/konnectivity-server/ca.key" + #write-pki-data "${KONNECTIVITY_SERVER_CA_KEY}" "${KONNECTIVITY_SERVER_CA_KEY_PATH}" + + KONNECTIVITY_SERVER_CA_CERT_PATH="${pki_dir}/konnectivity-server/ca.crt" + write-pki-data "${KONNECTIVITY_SERVER_CA_CERT}" "${KONNECTIVITY_SERVER_CA_CERT_PATH}" + + KONNECTIVITY_SERVER_KEY_PATH="${pki_dir}/konnectivity-server/server.key" + write-pki-data "${KONNECTIVITY_SERVER_KEY}" "${KONNECTIVITY_SERVER_KEY_PATH}" + + KONNECTIVITY_SERVER_CERT_PATH="${pki_dir}/konnectivity-server/server.crt" + write-pki-data "${KONNECTIVITY_SERVER_CERT}" "${KONNECTIVITY_SERVER_CERT_PATH}" + + KONNECTIVITY_SERVER_CLIENT_KEY_PATH="${pki_dir}/konnectivity-server/client.key" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_KEY}" "${KONNECTIVITY_SERVER_CLIENT_KEY_PATH}" + + KONNECTIVITY_SERVER_CLIENT_CERT_PATH="${pki_dir}/konnectivity-server/client.crt" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_CERT}" "${KONNECTIVITY_SERVER_CLIENT_CERT_PATH}" + fi + + if [[ -n "${KONNECTIVITY_AGENT_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-agent + KONNECTIVITY_AGENT_CA_KEY_PATH="${pki_dir}/konnectivity-agent/ca.key" + write-pki-data "${KONNECTIVITY_AGENT_CA_KEY}" "${KONNECTIVITY_AGENT_CA_KEY_PATH}" + + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + + KONNECTIVITY_AGENT_KEY_PATH="${pki_dir}/konnectivity-agent/server.key" + write-pki-data "${KONNECTIVITY_AGENT_KEY}" "${KONNECTIVITY_AGENT_KEY_PATH}" + + KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt" + write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}" + fi } # After the first boot and on upgrade, these files exist on the master-pd @@ -953,8 +999,12 @@ egressSelections: connection: proxyProtocol: HTTPConnect transport: - uds: - udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket + tcp: + url: https://127.0.0.1:8131 + tlsConfig: + caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt + clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key + clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt - name: controlplane connection: proxyProtocol: Direct @@ -1461,7 +1511,6 @@ function create-master-etcd-apiserver-auth { fi } - function docker-installed { if systemctl cat docker.service &> /dev/null ; then return 0 @@ -1944,30 +1993,44 @@ function prepare-konnectivity-server-manifest { params+=("--log-file=/var/log/konnectivity-server.log") params+=("--logtostderr=false") params+=("--log-file-max-size=0") - params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket") + if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then + params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket") + elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then + params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}") + params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}") + params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}") + params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}") + fi params+=("--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt") params+=("--cluster-key=/etc/srv/kubernetes/pki/apiserver.key") if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then params+=("--mode=grpc") + params+=("--server-port=0") + params+=("--agent-namespace=kube-system") + params+=("--agent-service-account=konnectivity-agent") + params+=("--authentication-audience=system:konnectivity-server") + params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig") elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then params+=("--mode=http-connect") + params+=("--server-port=8131") + params+=("--agent-namespace=") + params+=("--agent-service-account=") + params+=("--authentication-audience=") + # Need to fix ANP code to allow kubeconfig to be set with mtls. + params+=("--kubeconfig=") else echo "KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE must be set to either grpc or http-connect" exit 1 fi - params+=("--server-port=0") params+=("--agent-port=$1") params+=("--health-port=$2") params+=("--admin-port=$3") - params+=("--agent-namespace=kube-system") - params+=("--agent-service-account=konnectivity-agent") - params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig") - params+=("--authentication-audience=system:konnectivity-server") params+=("--kubeconfig-qps=75") params+=("--kubeconfig-burst=150") params+=("--keepalive-time=60s") params+=("--frontend-keepalive-time=60s") + params+=("--proxy-strategies=destHost,default") konnectivity_args="" for param in "${params[@]}"; do konnectivity_args+=", \"${param}\"" @@ -2834,6 +2897,15 @@ EOF function setup-konnectivity-agent-manifest { local -r manifest="/etc/kubernetes/addons/konnectivity-agent/konnectivity-agent-ds.yaml" sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}" + if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then + sed -i "s|__EXTRA_PARAMS__|\t\t\"--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt\",\n\t\t\"--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key\",|g" "${manifest}" + sed -i "s|__EXTRA_VOL_MNTS__| - name: pki\n mountPath: /etc/srv/kubernetes/pki/konnectivity-agent|g" "${manifest}" + sed -i "s|__EXTRA_VOLS__| - name: pki\n hostPath:\n path: /etc/srv/kubernetes/pki/konnectivity-agent|g" "${manifest}" + else + sed -i "s|__EXTRA_PARAMS__||g" "${manifest}" + sed -i "s|__EXTRA_VOL_MNTS__||g" "${manifest}" + sed -i "s|__EXTRA_VOLS__||g" "${manifest}" + fi } # Setups manifests for ingress controller and gce-specific policies for service controller. diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 8cc89894f06..37b91a2f124 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1111,6 +1111,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}") ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}") ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}") ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}") +KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}") +KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}") +KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}") +KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}") +KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}") +KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}") +KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}") +KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}") +KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}") +KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}") +KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}") +KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}") EOF } @@ -1261,6 +1273,9 @@ EOF fi if [[ "${master}" == "false" ]]; then cat >>"$file" <"${cert_create_debug_output}" || true + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator + mkdir easy-rsa-master/konnectivity-server + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server + mkdir easy-rsa-master/konnectivity-agent + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" + KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then # TODO(roberthbailey,porridge): add better error handling here, # see https://github.com/kubernetes/kubernetes/issues/55229 @@ -1856,6 +1894,86 @@ function generate-aggregator-certs { fi } +# Runs the easy RSA commands to generate server side certificate files +# for the konnectivity server. This includes both server side to both +# konnectivity-server and konnectivity-agent. +# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and +# ${KONNECTIVITY_AGENT_CERT_DIR} +# +# Assumed vars +# KUBE_TEMP +# KONNECTIVITY_SERVER_CERT_DIR +# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name +# KONNECTIVITY_SERVER_SANS: Subject alternate names +# +function generate-konnectivity-server-certs { + local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") + # Note: This was heavily cribbed from make-ca-cert.sh + (set -x + # Make the client <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server + rm -f "konnectivity-server.csr" + + # Make the agent <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + rm -f "konnectivity-agent.csr" + + echo "completed main certificate section") &>"${cert_create_debug_output}" || true + + local output_file_missing=0 + local output_file + for output_file in \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" + do + if [[ ! -s "${output_file}" ]]; then + echo "Expected file ${output_file} not created" >&2 + output_file_missing=1 + fi + done + if (( output_file_missing )); then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 + cat "${cert_create_debug_output}" >&2 + echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2 + exit 2 + fi +} + # Using provided master env, extracts value from provided key. # # Args: @@ -1897,6 +2015,16 @@ function parse-master-env() { ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY") + KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT") + KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT") + KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY") + KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY") + KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY") + KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT") + KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT") + KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY") } # Update or verify required gcloud components are installed