From 182a4fd0ac6cdc30aed1cf512c869fcd53059be1 Mon Sep 17 00:00:00 2001 From: Quintin Lee Date: Thu, 18 Aug 2016 13:42:57 -0700 Subject: [PATCH 1/2] Scripts to configure image verification admission controller for gce. --- cluster/gce/configure-vm.sh | 43 ++++++++++++ cluster/gce/gci/configure-helper.sh | 65 ++++++++++++++++--- cluster/gce/trusty/configure-helper.sh | 65 ++++++++++++++++--- .../kube-apiserver/kube-apiserver.manifest | 19 +++++- 4 files changed, 175 insertions(+), 17 deletions(-) diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 5335211bc26..23f8844ce2b 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -460,6 +460,11 @@ kube_uid: '$(echo "${KUBE_UID}" | sed -e "s/'/''/g")' initial_etcd_cluster: '$(echo "${INITIAL_ETCD_CLUSTER:-}" | sed -e "s/'/''/g")' hostname: $(hostname -s) EOF + if [ -n "${ADMISSION_CONTROL:-}" ] && [ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +admission-control-config-file: /etc/admission_controller.config +EOF + fi if [ -n "${KUBELET_PORT:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls kubelet_port: '$(echo "$KUBELET_PORT" | sed -e "s/'/''/g")' @@ -896,6 +901,44 @@ contexts: EOF fi + + if [[ -n "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then + # This is the config file for the image review webhook. + cat <>/etc/salt/minion.d/grains.conf + image_review_config: /etc/gcp_image_review.config +EOF + cat </etc/gcp_image_review.config +clusters: + - name: gcp-image-review-server + cluster: + server: ${GCP_IMAGE_VERIFICATION_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-image-review-server + user: kube-apiserver + name: webhook +EOF + # This is the config for the image review admission controller. + cat <>/etc/salt/minion.d/grains.conf + image_review_webhook_config: /etc/admission_controller.config +EOF + cat </etc/admission_controller.config +imagePolicy: + kubeConfigFile: /etc/gcp_image_review.config + allowTTL: 30 + denyTTL: 30 + retryBackoff: 500 + defaultAllow: true +EOF + fi + + # If the kubelet on the master is enabled, give it the same CIDR range # as a generic node. if [[ ! -z "${KUBELET_APISERVER:-}" ]] && [[ ! -z "${KUBELET_CERT:-}" ]] && [[ ! -z "${KUBELET_KEY:-}" ]]; then diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 8b14cbb5a68..5bafed43061 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -279,6 +279,36 @@ contexts: name: webhook EOF fi + +if [[ -n "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then + # This is the config file for the image review webhook. + cat </etc/gcp_image_review.config +clusters: + - name: gcp-image-review-server + cluster: + server: ${GCP_IMAGE_VERIFICATION_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-image-review-server + user: kube-apiserver + name: webhook +EOF + # This is the config for the image review admission controller. + cat </etc/admission_controller.config +imagePolicy: + kubeConfigFile: /etc/gcp_image_review.config + allowTTL: 30 + denyTTL: 30 + retryBackoff: 500 + defaultAllow: true +EOF + fi } function create-kubelet-kubeconfig { @@ -670,9 +700,24 @@ function start-kube-apiserver { if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi + + local admission_controller_config_mount="" + local admission_controller_config_volume="" + local image_policy_webhook_config_mount="" + local image_policy_webhook_config_volume="" if [[ -n "${ADMISSION_CONTROL:-}" ]]; then params+=" --admission-control=${ADMISSION_CONTROL}" + if [[ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]]; then + params+=" --admission-control-config-file=/etc/admission_controller.config" + # Mount the file to configure admission controllers if ImagePolicyWebhook is set. + admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false}," + admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\"}}," + # Mount the file to configure the ImagePolicyWebhook's webhook. + image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," + image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," + fi fi + if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" fi @@ -689,8 +734,8 @@ function start-kube-apiserver { params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile" fi - webhook_authn_config_mount="" - webhook_authn_config_volume="" + local webhook_authn_config_mount="" + local webhook_authn_config_volume="" if [[ -n "${GCP_AUTHN_URL:-}" ]]; then params+=" --authentication-token-webhook-config-file=/etc/gcp_authn.config" webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," @@ -698,8 +743,8 @@ function start-kube-apiserver { fi params+=" --authorization-mode=ABAC" - webhook_config_mount="" - webhook_config_volume="" + local webhook_config_mount="" + local webhook_config_volume="" if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then params+=",Webhook --authorization-webhook-config-file=/etc/gcp_authz.config" webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," @@ -708,10 +753,10 @@ function start-kube-apiserver { local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" if [[ -n "${KUBE_USER:-}" ]]; then - local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" - remove-salt-config-comments "${abac_policy_json}" - sed -i -e "s@{{kube_user}}@${KUBE_USER}@g" "${abac_policy_json}" - cp "${abac_policy_json}" /etc/srv/kubernetes/ + local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" + remove-salt-config-comments "${abac_policy_json}" + sed -i -e "s@{{kube_user}}@${KUBE_USER}@g" "${abac_policy_json}" + cp "${abac_policy_json}" /etc/srv/kubernetes/ fi src_file="${src_dir}/kube-apiserver.manifest" @@ -734,6 +779,10 @@ function start-kube-apiserver { sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}" + sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" + sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}" cp "${src_file}" /etc/kubernetes/manifests } diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 96a69a208f3..74a29ebe673 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -389,6 +389,36 @@ contexts: name: webhook EOF fi + +if [[ -n "${GCP_IMAGE_VERIFICATION_URL:-}" ]]; then + # This is the config file for the image review webhook. + cat </etc/gcp_image_review.config +clusters: + - name: gcp-image-review-server + cluster: + server: ${GCP_IMAGE_VERIFICATION_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-image-review-server + user: kube-apiserver + name: webhook +EOF + # This is the config for the image review admission controller. + cat </etc/admission_controller.config +imagePolicy: + kubeConfigFile: /etc/gcp_image_review.config + allowTTL: 30 + denyTTL: 30 + retryBackoff: 500 + defaultAllow: true +EOF + fi } # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY @@ -537,9 +567,24 @@ start_kube_apiserver() { if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi + + local admission_controller_config_mount="" + local admission_controller_config_volume="" + local image_policy_webhook_config_mount="" + local image_policy_webhook_config_volume="" if [ -n "${ADMISSION_CONTROL:-}" ]; then params="${params} --admission-control=${ADMISSION_CONTROL}" + if [ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]; then + params+=" --admission-control-config-file=/etc/admission_controller.config" + # Mount the file to configure admission controllers if ImagePolicyWebhook is set. + admission_controller_config_mount="{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"/etc/admission_controller.config\", \"readOnly\": false}," + admission_controller_config_volume="{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"/etc/admission_controller.config\"}}," + # Mount the file to configure the ImagePolicyWebhook's webhook. + image_policy_webhook_config_mount="{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," + image_policy_webhook_config_volume="{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," + fi fi + if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then params="${params} --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" fi @@ -554,8 +599,8 @@ start_kube_apiserver() { fi readonly kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag) - webhook_authn_config_mount="" - webhook_authn_config_volume="" + local webhook_authn_config_mount="" + local webhook_authn_config_volume="" if [ -n "${GCP_AUTHN_URL:-}" ]; then params="${params} --authentication-token-webhook-config-file=/etc/gcp_authn.config" webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false}," @@ -563,8 +608,8 @@ start_kube_apiserver() { fi params="${params} --authorization-mode=ABAC" - webhook_config_mount="" - webhook_config_volume="" + local webhook_config_mount="" + local webhook_config_volume="" if [ -n "${GCP_AUTHZ_URL:-}" ]; then params="${params},Webhook --authorization-webhook-config-file=/etc/gcp_authz.config" webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false}," @@ -574,10 +619,10 @@ start_kube_apiserver() { src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty" if [[ -n "${KUBE_USER:-}" ]]; then - local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" - remove_salt_config_comments "${abac_policy_json}" - sed -i -e "s@{{kube_user}}@${KUBE_USER}@g" "${abac_policy_json}" - cp "${abac_policy_json}" /etc/srv/kubernetes/ + local -r abac_policy_json="${src_dir}/abac-authz-policy.jsonl" + remove_salt_config_comments "${abac_policy_json}" + sed -i -e "s@{{kube_user}}@${KUBE_USER}@g" "${abac_policy_json}" + cp "${abac_policy_json}" /etc/srv/kubernetes/ fi src_file="${src_dir}/kube-apiserver.manifest" @@ -599,6 +644,10 @@ start_kube_apiserver() { sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}" sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" + sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}" + sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" + sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}" cp "${src_file}" /etc/kubernetes/manifests } diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 7771a14f1da..ba1080edea4 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -120,6 +120,19 @@ {% set authz_mode = authz_mode + ",Webhook" -%} {% endif -%} +{% set image_review_config = "" -%} +{% set admission_controller_config_mount = "" -%} +{% set admission_controller_config_volume = "" -%} +{% set image_policy_webhook_config_mount = "" -%} +{% set image_policy_webhook_config_volume = "" -%} +{% if grains.image_review_config is defined -%} + {% set image_review_config = " --admission-control-config-file=" + grains.image_review_config -%} + {% set admission_controller_config_mount = "{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"" + grains.image_review_config + "\", \"readOnly\": false}," -%} + {% set admission_controller_config_volume = "{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"" + grains.image_review_config + "\"}}," -%} + {% set image_policy_webhook_config_mount = "{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," -%} + {% set image_policy_webhook_config_volume = "{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\"}}," -%} +{% endif -%} + {% set admission_control = "" -%} {% if pillar['admission_control'] is defined -%} {% set admission_control = "--admission-control=" + pillar['admission_control'] -%} @@ -146,7 +159,7 @@ {% endif -%} {% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector -%} -{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config -%} +{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config + image_review_config -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration {% if pillar['apiserver_test_args'] is defined -%} @@ -202,6 +215,8 @@ {{additional_cloud_config_mount}} {{webhook_config_mount}} {{webhook_authn_config_mount}} + {{admission_controller_config_mount}} + {{image_policy_webhook_config_mount}} { "name": "srvkube", "mountPath": "{{srv_kube_path}}", "readOnly": true}, @@ -231,6 +246,8 @@ {{additional_cloud_config_volume}} {{webhook_config_volume}} {{webhook_authn_config_volume}} + {{admission_controller_config_volume}} + {{image_policy_webhook_config_volume}} { "name": "srvkube", "hostPath": { "path": "{{srv_kube_path}}"} From 2be0b40c55f278649f77820eb6f7ef98208346ac Mon Sep 17 00:00:00 2001 From: Quintin Lee Date: Mon, 22 Aug 2016 01:09:37 -0700 Subject: [PATCH 2/2] Actually link in ImagePolicyWebhook and run initialization. --- cmd/kube-apiserver/app/plugins.go | 1 + pkg/master/import_known_versions.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/kube-apiserver/app/plugins.go b/cmd/kube-apiserver/app/plugins.go index ef4dda786ae..7b768d2b510 100644 --- a/cmd/kube-apiserver/app/plugins.go +++ b/cmd/kube-apiserver/app/plugins.go @@ -29,6 +29,7 @@ import ( _ "k8s.io/kubernetes/plugin/pkg/admission/antiaffinity" _ "k8s.io/kubernetes/plugin/pkg/admission/deny" _ "k8s.io/kubernetes/plugin/pkg/admission/exec" + _ "k8s.io/kubernetes/plugin/pkg/admission/imagepolicy" _ "k8s.io/kubernetes/plugin/pkg/admission/initialresources" _ "k8s.io/kubernetes/plugin/pkg/admission/limitranger" _ "k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision" diff --git a/pkg/master/import_known_versions.go b/pkg/master/import_known_versions.go index b670c6f6f1a..926a95fdfc8 100644 --- a/pkg/master/import_known_versions.go +++ b/pkg/master/import_known_versions.go @@ -30,6 +30,7 @@ import ( _ "k8s.io/kubernetes/pkg/apis/certificates/install" _ "k8s.io/kubernetes/pkg/apis/componentconfig/install" _ "k8s.io/kubernetes/pkg/apis/extensions/install" + _ "k8s.io/kubernetes/pkg/apis/imagepolicy/install" _ "k8s.io/kubernetes/pkg/apis/policy/install" _ "k8s.io/kubernetes/pkg/apis/rbac/install" )