Merge pull request #25561 from andyzheng0831/webhook

Automatic merge from submit-queue Configuration for GCP webhook authentication and authorization This PR adds configuration for GCP webhook authentication and authorization in ContainerVM and GCI. The change of configure-vm.sh and kube-apiserver.manifest is directly copied from @cjcullen's PR #25380 and #25296. The change in GCI script configure-helper.sh includes the support for webhook authentication and authorization, and also some code refactor to improve readability. @cjcullen @roberthbailey @zmerlynn please review it. The original PRs are P1, please mark this as P1. cc/ @fabioy @kubernetes/goog-image FYI. I verified it by running e2e tests on GCI cluster. Without the GCI side change, cluster creation fails as being capture by GKE Jenkins tests. I don't test when the two env GCP_AUTHN_URL and GCP_AUTHZ_URL are set, because they are only set in GKE. After this PR is merged, @cjcullen will test in GKE.
2025-07-29 22:46:12 +00:00 · 2016-05-21 22:30:14 -07:00 · 2016-05-21 22:30:14 -07:00 · 1eb221e96f
commit 1eb221e96f
parent 529135dd01 9811009d9f
3 changed files with 153 additions and 12 deletions
--- a/cluster/gce/configure-vm.sh
+++ b/cluster/gce/configure-vm.sh
@ -798,14 +798,60 @@ EOF
    CLOUD_CONFIG=/etc/gce.conf
  fi

-  if [[ -n ${CLOUD_CONFIG:-} ]]; then
-  cat <<EOF >>/etc/salt/minion.d/grains.conf
+  if [[ -n "${CLOUD_CONFIG:-}" ]]; then
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
  cloud_config: ${CLOUD_CONFIG}
 EOF
  else
    rm -f /etc/gce.conf
  fi

+  if [[ -n "${GCP_AUTHN_URL:-}" ]]; then
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
+  webhook_authentication_config: /etc/gcp_authn.config
+EOF
+    cat <<EOF >/etc/gcp_authn.config
+clusters:
+  - name: gcp-authentication-server
+    cluster:
+      server: ${GCP_AUTHN_URL}
+users:
+  - name: kube-apiserver
+    user:
+      auth-provider:
+        name: gcp
+current-context: webhook
+contexts:
+- context:
+    cluster: gcp-authentication-server
+    user: kube-apiserver
+  name: webhook
+EOF
+  fi
+
+  if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
+  webhook_authorization_config: /etc/gcp_authz.config
+EOF
+    cat <<EOF >/etc/gcp_authz.config
+clusters:
+  - name: gcp-authorization-server
+    cluster:
+      server: ${GCP_AUTHZ_URL}
+users:
+  - name: kube-apiserver
+    user:
+      auth-provider:
+        name: gcp
+current-context: webhook
+contexts:
+- context:
+    cluster: gcp-authorization-server
+    user: kube-apiserver
+  name: webhook
+EOF
+  fi
+
  # If the kubelet on the master is enabled, give it the same CIDR range
  # as a generic node.
  if [[ ! -z "${KUBELET_APISERVER:-}" ]] && [[ ! -z "${KUBELET_CERT:-}" ]] && [[ ! -z "${KUBELET_KEY:-}" ]]; then
--- a/cluster/gce/trusty/configure-helper.sh
+++ b/cluster/gce/trusty/configure-helper.sh
@ -314,6 +314,46 @@ EOF
  if [ -n "${MULTIZONE:-}" ]; then
    cat <<EOF >>/etc/gce.conf
 multizone = ${MULTIZONE}
+EOF
+  fi
+
+  if [ -n "${GCP_AUTHN_URL:-}" ]; then
+    cat <<EOF >/etc/gcp_authn.config
+clusters:
+  - name: gcp-authentication-server
+    cluster:
+      server: ${GCP_AUTHN_URL}
+users:
+  - name: kube-apiserver
+    user:
+      auth-provider:
+        name: gcp
+current-context: webhook
+contexts:
+- context:
+    cluster: gcp-authentication-server
+    user: kube-apiserver
+  name: webhook
+EOF
+  fi
+
+  if [ -n "${GCP_AUTHZ_URL:-}" ]; then
+    cat <<EOF >/etc/gcp_authz.config
+clusters:
+  - name: gcp-authorization-server
+    cluster:
+      server: ${GCP_AUTHZ_URL}
+users:
+  - name: kube-apiserver
+    user:
+      auth-provider:
+        name: gcp
+current-context: webhook
+contexts:
+- context:
+    cluster: gcp-authorization-server
+    user: kube-apiserver
+  name: webhook
 EOF
  fi
 }
@ -413,7 +453,20 @@ start_kube_apiserver() {
  timeout 30 docker load -i /home/kubernetes/kube-docker-files/kube-apiserver.tar

  # Calculate variables and assemble the command line.
-  params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true --authorization-mode=ABAC --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl --etcd-servers-overrides=/events#http://127.0.0.1:4002 ${APISERVER_TEST_ARGS:-}"
+  params="${APISERVER_TEST_ARGS:-} ${API_SERVER_TEST_LOG_LEVEL:-"--v=2"}"
+  params="${params} --cloud-provider=gce"
+  params="${params} --address=127.0.0.1"
+  params="${params} --etcd-servers=http://127.0.0.1:4001"
+  params="${params} --tls-cert-file=/etc/srv/kubernetes/server.cert"
+  params="${params} --tls-private-key-file=/etc/srv/kubernetes/server.key"
+  params="${params} --secure-port=443"
+  params="${params} --client-ca-file=/etc/srv/kubernetes/ca.crt"
+  params="${params} --token-auth-file=/etc/srv/kubernetes/known_tokens.csv"
+  params="${params} --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv"
+  params="${params} --allow-privileged=true"
+  params="${params} --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl"
+  params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
+
  if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then
    params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
  fi
@ -426,18 +479,32 @@ start_kube_apiserver() {
  if [ -n "${RUNTIME_CONFIG:-}" ]; then
    params="${params} --runtime-config=${RUNTIME_CONFIG}"
  fi
-  log_level="--v=2"
-  if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
-    log_level="${API_SERVER_TEST_LOG_LEVEL}"
-  fi
-  params="${params} ${log_level}"
-
  if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
-    readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
-    params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
+    readonly vm_external_ip=$(curl --retry 5 --retry-delay 3 --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
+    params="${params} --cloud-config=/etc/gce.conf"
+    params="${params} --advertise-address=${vm_external_ip}"
+    params="${params} --ssh-user=${PROXY_SSH_USER}"
+    params="${params} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
  fi
  readonly kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)

+  webhook_authn_config_mount=""
+  webhook_authn_config_volume=""
+  if [ -n "${GCP_AUTHN_URL:-}" ]; then
+    params="${params} --authentication-token-webhook-config-file=/etc/gcp_authn.config"
+    webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false},"
+    webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\"}},"
+  fi
+
+  params="${params} --authorization-mode=ABAC"
+  webhook_config_mount=""
+  webhook_config_volume=""
+  if [ -n "${GCP_AUTHZ_URL:-}" ]; then
+    params="${params},Webhook --authorization-webhook-config-file=/etc/gcp_authz.config"
+    webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false},"
+    webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\"}},"
+  fi
+
  src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty"
  cp "${src_dir}/abac-authz-policy.jsonl" /etc/srv/kubernetes/
  src_file="${src_dir}/kube-apiserver.manifest"
@ -455,6 +522,11 @@ start_kube_apiserver() {
  sed -i -e "s@{{secure_port}}@8080@g" "${src_file}"
  sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
  sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
+  sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}"
+  sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
+  sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"
+  sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}"
+
  cp "${src_file}" /etc/kubernetes/manifests
 }

--- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
+++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
@ -83,6 +83,25 @@
  {% set abac_policy_file = " --authorization-policy-file=/srv/kubernetes/abac-authz-policy.jsonl" -%}
 {% endif -%}

+{% set webhook_authentication_config = "" -%}
+{% set webhook_authn_config_mount = "" -%}
+{% set webhook_authn_config_volume = "" -%}
+{% if grains.webhook_authentication_config is defined -%}
+  {% set webhook_authentication_config = " --authentication-token-webhook-config-file=" + grains.webhook_authentication_config -%}
+  {% set webhook_authn_config_mount = "{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"" + grains.webhook_authentication_config + "\", \"readOnly\": false}," -%}
+  {% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\"}}," -%}
+{% endif -%}
+
+{% set webhook_authorization_config = "" -%}
+{% set webhook_config_mount = "" -%}
+{% set webhook_config_volume = "" -%}
+{% if grains.webhook_authorization_config is defined -%}
+  {% set webhook_authorization_config = " --authorization-webhook-config-file=" + grains.webhook_authorization_config -%}
+  {% set webhook_config_mount = "{\"name\": \"webhookconfigmount\",\"mountPath\": \"" + grains.webhook_authorization_config + "\", \"readOnly\": false}," -%}
+  {% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\"}}," -%}
+  {% set authz_mode = authz_mode + ",Webhook" -%}
+{% endif -%}
+
 {% set admission_control = "" -%}
 {% if pillar['admission_control'] is defined -%}
 {% set admission_control = "--admission-control=" + pillar['admission_control'] -%}
@ -99,7 +118,7 @@
 {% endif -%}

 {% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout -%}
-{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address  + " " + proxy_ssh_options + authz_mode + abac_policy_file -%}
+{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address  + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config -%}

 # test_args has to be kept at the end, so they'll overwrite any prior configuration
 {% if pillar['apiserver_test_args'] is defined -%}
@ -153,6 +172,8 @@
    "volumeMounts": [
        {{cloud_config_mount}}
        {{additional_cloud_config_mount}}
+        {{webhook_config_mount}}
+        {{webhook_authn_config_mount}}
        { "name": "srvkube",
        "mountPath": "{{srv_kube_path}}",
        "readOnly": true},
@ -180,6 +201,8 @@
 "volumes":[
  {{cloud_config_volume}}
  {{additional_cloud_config_volume}}
+  {{webhook_config_volume}}
+  {{webhook_authn_config_volume}}
  { "name": "srvkube",
    "hostPath": {
        "path": "{{srv_kube_path}}"}