diff --git a/cluster/gce/addons/cloud-pvl-admission/mutating-webhook-configuration.yaml b/cluster/gce/addons/cloud-pvl-admission/mutating-webhook-configuration.yaml new file mode 100644 index 00000000000..8232645e9f0 --- /dev/null +++ b/cluster/gce/addons/cloud-pvl-admission/mutating-webhook-configuration.yaml @@ -0,0 +1,22 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: "cloud-pvl-admission.k8s.io" + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: cloud-pvl-admission +webhooks: +- name: "cloud-pvl-admission.k8s.io" + rules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["persistentvolumes"] + scope: "*" + clientConfig: + url: "https://127.0.0.1:9001/admit" + caBundle: "__CLOUD_PVL_ADMISSION_CA_CERT__" + admissionReviewVersions: ["v1"] + sideEffects: None + timeoutSeconds: 5 + failurePolicy: Fail diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index b0e62f9488f..8336fa9d205 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -362,7 +362,7 @@ fi CUSTOM_INGRESS_YAML="${CUSTOM_INGRESS_YAML:-}" # Admission Controllers to invoke prior to persisting objects in cluster -ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,RuntimeClass +ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,RuntimeClass # MutatingAdmissionWebhook should be the last controller that modifies the # request object, otherwise users will be confused if the mutating webhooks' diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index dfb52dbc63d..349f56c4f6b 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -407,7 +407,7 @@ fi CUSTOM_INGRESS_YAML=${CUSTOM_INGRESS_YAML:-} if [[ -z "${KUBE_ADMISSION_CONTROL:-}" ]]; then - ADMISSION_CONTROL='NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,PersistentVolumeClaimResize,RuntimeClass' + ADMISSION_CONTROL='NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,PersistentVolumeClaimResize,RuntimeClass' # ResourceQuota must come last, or a creation is recorded, but the pod may be forbidden. ADMISSION_CONTROL="${ADMISSION_CONTROL},MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" else diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index dcf905f1043..168e2628469 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -781,6 +781,18 @@ function create-master-pki { KONNECTIVITY_AGENT_CERT_PATH="${pki_dir}/konnectivity-agent/server.crt" write-pki-data "${KONNECTIVITY_AGENT_CERT}" "${KONNECTIVITY_AGENT_CERT_PATH}" fi + + if [[ -n "${CLOUD_PVL_ADMISSION_CA_CERT:-}" ]]; then + mkdir -p "${pki_dir}"/cloud-pvl-admission + CLOUD_PVL_ADMISSION_CA_CERT_PATH="${pki_dir}/cloud-pvl-admission/ca.crt" + write-pki-data "${CLOUD_PVL_ADMISSION_CA_CERT}" "${CLOUD_PVL_ADMISSION_CA_CERT_PATH}" + + CLOUD_PVL_ADMISSION_KEY_PATH="${pki_dir}/cloud-pvl-admission/server.key" + write-pki-data "${CLOUD_PVL_ADMISSION_KEY}" "${CLOUD_PVL_ADMISSION_KEY_PATH}" + + CLOUD_PVL_ADMISSION_CERT_PATH="${pki_dir}/cloud-pvl-admission/server.crt" + write-pki-data "${CLOUD_PVL_ADMISSION_CERT}" "${CLOUD_PVL_ADMISSION_CERT_PATH}" + fi } # After the first boot and on upgrade, these files exist on the master-pd @@ -2380,6 +2392,9 @@ function start-cloud-controller-manager { echo "Writing manifest for cloud provider controller-manager" cp "${src_file}" /etc/kubernetes/manifests + + setup-addon-manifests "addons" "cloud-pvl-admission" + setup-cloud-pvl-admission-manifest } # Starts kubernetes scheduler. @@ -2998,6 +3013,11 @@ function setup-konnectivity-agent-manifest { fi } +function setup-cloud-pvl-admission-manifest { + local -r manifest="/etc/kubernetes/addons/cloud-pvl-admission/mutating-webhook-configuration.yaml" + sed -i "s|__CLOUD_PVL_ADMISSION_CA_CERT__|${CLOUD_PVL_ADMISSION_CA_CERT}|g" "${manifest}" +} + # Setups manifests for ingress controller and gce-specific policies for service controller. function start-lb-controller { setup-addon-manifests "addons" "loadbalancing" diff --git a/cluster/gce/manifests/cloud-controller-manager.manifest b/cluster/gce/manifests/cloud-controller-manager.manifest index 2049edbb0e3..a8c25211e8b 100644 --- a/cluster/gce/manifests/cloud-controller-manager.manifest +++ b/cluster/gce/manifests/cloud-controller-manager.manifest @@ -74,6 +74,50 @@ "mountPath": "/etc/pki", "readOnly": true} ] + }, + { + "name": "cloud-pvl-admission", + "image": "gcr.io/k8s-staging-cloud-pv-labeler/cloud-pv-admission-labeler:v0.3.0", + "resources": { + "requests": { + "cpu": "10m" + } + }, + "command": [ + "/cloud-pv-admission-labeler", + "--addr=localhost:9001", + "--tls-cert-path=/etc/srv/kubernetes/pki/cloud-pvl-admission/server.crt", + "--tls-key-path=/etc/srv/kubernetes/pki/cloud-pvl-admission/server.key", + "--cloud-provider=gce", + "--cloud-config=/etc/gce.conf" + ], + "volumeMounts": [ + {{cloud_config_mount}} + {{additional_cloud_config_mount}} + {{pv_recycler_mount}} + { "name": "srvkube", + "mountPath": "/etc/srv/kubernetes", + "readOnly": true}, + {{flexvolume_hostpath_mount}} + { "name": "logfile", + "mountPath": "/var/log/cloud-pvl-admission.log", + "readOnly": false}, + { "name": "etcssl", + "mountPath": "/etc/ssl", + "readOnly": true}, + { "name": "usrsharecacerts", + "mountPath": "/usr/share/ca-certificates", + "readOnly": true}, + { "name": "varssl", + "mountPath": "/var/ssl", + "readOnly": true}, + { "name": "etcopenssl", + "mountPath": "/etc/openssl", + "readOnly": true}, + { "name": "etcpki", + "mountPath": "/etc/pki", + "readOnly": true} + ] } ], "volumes":[ diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 7660b240ecc..7345d2d43ac 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1072,6 +1072,10 @@ ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}") ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}") ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}") ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}") +CLOUD_PVL_ADMISSION_CA_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_KEY_BASE64:-}") +CLOUD_PVL_ADMISSION_CA_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_CERT_BASE64:-}") +CLOUD_PVL_ADMISSION_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CERT_BASE64:-}") +CLOUD_PVL_ADMISSION_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_KEY_BASE64:-}") KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}") KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}") KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}") @@ -1357,6 +1361,9 @@ KUBE_POD_LOG_READERS_GROUP: 2007 KONNECTIVITY_SERVER_RUNASUSER: 2008 KONNECTIVITY_SERVER_RUNASGROUP: 2008 KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP: 2008 +CLOUD_CONTROLLER_MANAGER_RUNASUSER: 2009 +CLOUD_CONTROLLER_MANAGER_RUNASGROUP: 2009 + EOF # KUBE_APISERVER_REQUEST_TIMEOUT_SEC (if set) controls the --request-timeout # flag @@ -1685,6 +1692,7 @@ function create-certs { PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs + CLOUD_PVL_ADMISSION_PRIMARY_CN="${primary_cn}" CLOUD_PVL_ADMISSION_SANS="${sans}" generate-cloud-pvl-admission-certs # By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces. # Note 'base64 -w0' doesn't work on Mac OS X, which has different flags. @@ -1722,6 +1730,11 @@ function create-certs { KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n') KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n') KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n') + + CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n') + CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" | tr -d '\r\n') + CLOUD_PVL_ADMISSION_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n') + CLOUD_PVL_ADMISSION_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" | tr -d '\r\n') } # Set up easy-rsa directory structure. @@ -1743,12 +1756,15 @@ function setup-easyrsa { cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet mkdir easy-rsa-master/aggregator cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator + mkdir easy-rsa-master/cloud-pvl-admission + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/cloud-pvl-admission mkdir easy-rsa-master/konnectivity-server cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server mkdir easy-rsa-master/konnectivity-agent cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" + CLOUD_PVL_ADMISSION_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission" KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server" KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then @@ -1968,6 +1984,78 @@ function generate-konnectivity-server-certs { fi } +# Runs the easy RSA commands to generate server side certificate files +# for the cloud-pvl-admission webhook. +# The generated files are in ${CLOUD_PVL_ADMISSION_CERT_DIR} +# +# Assumed vars +# KUBE_TEMP +# CLOUD_PVL_ADMISSION_CERT_DIR +# CLOUD_PVL_ADMISSION_PRIMARY_CN: Primary canonical name +# CLOUD_PVL_ADMISSION_SANS: Subject alternate names +# +function generate-cloud-pvl-admission-certs { + local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") + # Note: This was heavily cribbed from make-ca-cert.sh + (set -x + # Make the client <-> cloud-pvl-admission server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="IP:127.0.0.1,${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" + # create the cloud-pvl-admission cert with the correct groups + echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission + rm -f "cloud-pvl-admission.csr" + + # Make the cloud-pvl-admission server side certificates. + cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission" + ./easyrsa init-pki + # this puts the cert into pki/ca.crt and the key into pki/private/ca.key + ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass + ./easyrsa --subject-alt-name="${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass + ./easyrsa build-client-full client nopass + + kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" + + # make the config for the signer + echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" + # create the cloud-pvl-admission server cert with the correct groups + echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + rm -f "konnectivity-agent.csr" + + echo "completed main certificate section") &>"${cert_create_debug_output}" || true + + local output_file_missing=0 + local output_file + for output_file in \ + "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" \ + "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" \ + "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" \ + "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" \ + "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/client.crt" \ + "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/client.key" + do + if [[ ! -s "${output_file}" ]]; then + echo "Expected file ${output_file} not created" >&2 + output_file_missing=1 + fi + done + if (( output_file_missing )); then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 + cat "${cert_create_debug_output}" >&2 + echo "=== Failed to generate cloud-pvl-admission certificates: Aborting ===" >&2 + exit 2 + fi +} + # Using provided master env, extracts value from provided key. # # Args: @@ -2009,6 +2097,10 @@ function parse-master-env() { ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") + CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_KEY") + CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_CERT") + CLOUD_PVL_ADMISSION_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CERT") + CLOUD_PVL_ADMISSION_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_KEY") KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY") KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT") KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT")