diff --git a/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml b/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml index 7805acbdc66..cef0e685c58 100644 --- a/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml +++ b/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml @@ -33,12 +33,14 @@ spec: args: [ "--logtostderr=true", "--ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", +__EXTRA_PARAMS__ "--proxy-server-host=__APISERVER_IP__", "--proxy-server-port=8132", "--sync-interval=5s", "--sync-interval-cap=30s", "--probe-interval=5s", - "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token" + "--service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token", + "--agent-identifiers=ipv4=$(HOST_IP)" ] env: - name: POD_NAME @@ -49,11 +51,16 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP resources: limits: cpu: 50m memory: 30Mi volumeMounts: +__EXTRA_VOL_MNTS__ - mountPath: /var/run/secrets/tokens name: konnectivity-agent-token livenessProbe: @@ -64,6 +71,7 @@ spec: timeoutSeconds: 15 serviceAccountName: konnectivity-agent volumes: +__EXTRA_VOLS__ - name: konnectivity-agent-token projected: sources: diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 358811cf615..177184b26fb 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -657,6 +657,16 @@ function create-node-pki { KUBELET_KEY_PATH="${pki_dir}/kubelet.key" write-pki-data "${KUBELET_KEY}" "${KUBELET_KEY_PATH}" fi + + if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then + mkdir -p "${pki_dir}/konnectivity-agent" + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + KONNECTIVITY_AGENT_CLIENT_KEY_PATH="${pki_dir}/konnectivity-agent/client.key" + KONNECTIVITY_AGENT_CLIENT_CERT_PATH="${pki_dir}/konnectivity-agent/client.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_KEY}" "${KONNECTIVITY_AGENT_CLIENT_KEY_PATH}" + write-pki-data "${KONNECTIVITY_AGENT_CLIENT_CERT}" "${KONNECTIVITY_AGENT_CLIENT_CERT_PATH}" + fi } function create-master-pki { @@ -724,6 +734,42 @@ function create-master-pki { PROXY_CLIENT_CERT_PATH="${pki_dir}/proxy_client.crt" write-pki-data "${PROXY_CLIENT_CERT}" "${PROXY_CLIENT_CERT_PATH}" fi + + if [[ -n "${KONNECTIVITY_SERVER_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-server + #KONNECTIVITY_SERVER_CA_KEY_PATH="${pki_dir}/konnectivity-server/ca.key" + #write-pki-data "${KONNECTIVITY_SERVER_CA_KEY}" "${KONNECTIVITY_SERVER_CA_KEY_PATH}" + + KONNECTIVITY_SERVER_CA_CERT_PATH="${pki_dir}/konnectivity-server/ca.crt" + write-pki-data "${KONNECTIVITY_SERVER_CA_CERT}" "${KONNECTIVITY_SERVER_CA_CERT_PATH}" + + KONNECTIVITY_SERVER_KEY_PATH="${pki_dir}/konnectivity-server/server.key" + write-pki-data "${KONNECTIVITY_SERVER_KEY}" "${KONNECTIVITY_SERVER_KEY_PATH}" + + KONNECTIVITY_SERVER_CERT_PATH="${pki_dir}/konnectivity-server/server.crt" + write-pki-data "${KONNECTIVITY_SERVER_CERT}" "${KONNECTIVITY_SERVER_CERT_PATH}" + + KONNECTIVITY_SERVER_CLIENT_KEY_PATH="${pki_dir}/konnectivity-server/client.key" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_KEY}" "${KONNECTIVITY_SERVER_CLIENT_KEY_PATH}" + + KONNECTIVITY_SERVER_CLIENT_CERT_PATH="${pki_dir}/konnectivity-server/client.crt" + write-pki-data "${KONNECTIVITY_SERVER_CLIENT_CERT}" "${KONNECTIVITY_SERVER_CLIENT_CERT_PATH}" + fi + + if [[ -n "${KONNECTIVITY_AGENT_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/konnectivity-agent + KONNECTIVITY_AGENT_CA_KEY_PATH="${pki_dir}/konnectivity-agent/ca.key" + write-pki-data "${KONNECTIVITY_AGENT_CA_KEY}" "${KONNECTIVITY_AGENT_CA_KEY_PATH}" + + KONNECTIVITY_AGENT_CA_CERT_PATH="${pki_dir}/konnectivity-agent/ca.crt" + write-pki-data "${KONNECTIVITY_AGENT_CA_CERT}" "${KONNECTIVITY_AGENT_CA_CERT_PATH}" + + KONNECTIVITY_AGENT_KEY_PATH="${pki_dir}/konnectivity-agent/server.key" + write-pki-data "${KONNECTIVITY_AGENT_KEY}" "${KONNECTIVITY_AGENT_KEY_PATH}" + + KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt" + write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}" + fi } # After the first boot and on upgrade, these files exist on the master-pd @@ -953,8 +999,12 @@ egressSelections: connection: proxyProtocol: HTTPConnect transport: - uds: - udsName: /etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket + tcp: + url: https://127.0.0.1:8131 + tlsConfig: + caBundle: /etc/srv/kubernetes/pki/konnectivity-server/ca.crt + clientKey: /etc/srv/kubernetes/pki/konnectivity-server/client.key + clientCert: /etc/srv/kubernetes/pki/konnectivity-server/client.crt - name: controlplane connection: proxyProtocol: Direct @@ -1461,7 +1511,6 @@ function create-master-etcd-apiserver-auth { fi } - function docker-installed { if systemctl cat docker.service &> /dev/null ; then return 0 @@ -1944,30 +1993,44 @@ function prepare-konnectivity-server-manifest { params+=("--log-file=/var/log/konnectivity-server.log") params+=("--logtostderr=false") params+=("--log-file-max-size=0") - params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket") + if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then + params+=("--uds-name=/etc/srv/kubernetes/konnectivity-server/konnectivity-server.socket") + elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then + params+=("--server-ca-cert=${KONNECTIVITY_SERVER_CA_CERT_PATH}") + params+=("--server-cert=${KONNECTIVITY_SERVER_CERT_PATH}") + params+=("--server-key=${KONNECTIVITY_SERVER_KEY_PATH}") + params+=("--cluster-ca-cert=${KONNECTIVITY_AGENT_CA_CERT_PATH}") + fi params+=("--cluster-cert=/etc/srv/kubernetes/pki/apiserver.crt") params+=("--cluster-key=/etc/srv/kubernetes/pki/apiserver.key") if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'grpc' ]]; then params+=("--mode=grpc") + params+=("--server-port=0") + params+=("--agent-namespace=kube-system") + params+=("--agent-service-account=konnectivity-agent") + params+=("--authentication-audience=system:konnectivity-server") + params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig") elif [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then params+=("--mode=http-connect") + params+=("--server-port=8131") + params+=("--agent-namespace=") + params+=("--agent-service-account=") + params+=("--authentication-audience=") + # Need to fix ANP code to allow kubeconfig to be set with mtls. + params+=("--kubeconfig=") else echo "KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE must be set to either grpc or http-connect" exit 1 fi - params+=("--server-port=0") params+=("--agent-port=$1") params+=("--health-port=$2") params+=("--admin-port=$3") - params+=("--agent-namespace=kube-system") - params+=("--agent-service-account=konnectivity-agent") - params+=("--kubeconfig=/etc/srv/kubernetes/konnectivity-server/kubeconfig") - params+=("--authentication-audience=system:konnectivity-server") params+=("--kubeconfig-qps=75") params+=("--kubeconfig-burst=150") params+=("--keepalive-time=60s") params+=("--frontend-keepalive-time=60s") + params+=("--proxy-strategies=destHost,default") konnectivity_args="" for param in "${params[@]}"; do konnectivity_args+=", \"${param}\"" @@ -2834,6 +2897,15 @@ EOF function setup-konnectivity-agent-manifest { local -r manifest="/etc/kubernetes/addons/konnectivity-agent/konnectivity-agent-ds.yaml" sed -i "s|__APISERVER_IP__|${KUBERNETES_MASTER_NAME}|g" "${manifest}" + if [[ "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-grpc}" == 'http-connect' ]]; then + sed -i "s|__EXTRA_PARAMS__|\t\t\"--agent-cert=/etc/srv/kubernetes/pki/konnectivity-agent/client.crt\",\n\t\t\"--agent-key=/etc/srv/kubernetes/pki/konnectivity-agent/client.key\",|g" "${manifest}" + sed -i "s|__EXTRA_VOL_MNTS__| - name: pki\n mountPath: /etc/srv/kubernetes/pki/konnectivity-agent|g" "${manifest}" + sed -i "s|__EXTRA_VOLS__| - name: pki\n hostPath:\n path: /etc/srv/kubernetes/pki/konnectivity-agent|g" "${manifest}" + else + sed -i "s|__EXTRA_PARAMS__||g" "${manifest}" + sed -i "s|__EXTRA_VOL_MNTS__||g" "${manifest}" + sed -i "s|__EXTRA_VOLS__||g" "${manifest}" + fi } # Setups manifests for ingress controller and gce-specific policies for service controller. diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 8cc89894f06..37b91a2f124 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1111,6 +1111,18 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}") ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}") ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}") ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}") +KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}") +KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}") +KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}") +KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}") +KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}") +KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}") +KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}") +KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}") +KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}") +KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}") +KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}") +KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}") EOF } @@ -1261,6 +1273,9 @@ EOF fi if [[ "${master}" == "false" ]]; then cat >>"$file" <"${cert_create_debug_output}" || true + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator + mkdir easy-rsa-master/konnectivity-server + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server + mkdir easy-rsa-master/konnectivity-agent + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" + KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then # TODO(roberthbailey,porridge): add better error handling here, # see https://github.com/kubernetes/kubernetes/issues/55229 @@ -1856,6 +1894,86 @@ function generate-aggregator-certs { fi } +# Runs the easy RSA commands to generate server side certificate files +# for the konnectivity server. This includes both server side to both +# konnectivity-server and konnectivity-agent. +# The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and +# ${KONNECTIVITY_AGENT_CERT_DIR} +# +# Assumed vars +# KUBE_TEMP +# KONNECTIVITY_SERVER_CERT_DIR +# KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name +# KONNECTIVITY_SERVER_SANS: Subject alternate names +# +function generate-konnectivity-server-certs { + local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") + # Note: This was heavily cribbed from make-ca-cert.sh + (set -x + # Make the client <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server + rm -f "konnectivity-server.csr" + + # Make the agent <-> konnectivity server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" + # create the konnectivity server cert with the correct groups + echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + rm -f "konnectivity-agent.csr" + + echo "completed main certificate section") &>"${cert_create_debug_output}" || true + + local output_file_missing=0 + local output_file + for output_file in \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \ + "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" + do + if [[ ! -s "${output_file}" ]]; then + echo "Expected file ${output_file} not created" >&2 + output_file_missing=1 + fi + done + if (( output_file_missing )); then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 + cat "${cert_create_debug_output}" >&2 + echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2 + exit 2 + fi +} + # Using provided master env, extracts value from provided key. # # Args: @@ -1897,6 +2015,16 @@ function parse-master-env() { ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY") + KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT") + KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT") + KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY") + KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT") + KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY") + KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY") + KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT") + KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT") + KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY") } # Update or verify required gcloud components are installed