From 192392bddd2dbf97b29f389ea7612d0a0f862210 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Fri, 17 Feb 2017 15:06:55 -0800 Subject: [PATCH] refactor certs in GCE --- cluster/gce/configure-vm.sh | 40 ++++++--- cluster/gce/gci/configure-helper.sh | 116 ++++++++++++++++++------- cluster/saltbase/salt/kubelet/default | 2 +- cluster/saltbase/salt/kubelet/init.sls | 6 ++ 4 files changed, 119 insertions(+), 45 deletions(-) diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 04c076f3ebf..092f9c80af8 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -53,6 +53,27 @@ ensure-packages() { : } +function create-node-pki { + echo "Creating node pki files" + + local -r pki_dir="/etc/kubernetes/pki" + mkdir -p "${pki_dir}" + + if [[ -z "${CA_CERT_BUNDLE:-}" ]]; then + CA_CERT_BUNDLE="${CA_CERT}" + CA_CERT_BUNDLE_PATH="${pki_dir}/ca-certificates.crt" + echo "${CA_CERT_BUNDLE}" | base64 --decode > "${CA_CERT_BUNDLE_PATH}" + fi + + if [[ ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then + KUBELET_CERT_PATH="${pki_dir}/kubelet.crt" + echo "${KUBELET_CERT}" | base64 --decode > "${KUBELET_CERT_PATH}" + + KUBELET_KEY_PATH="${pki_dir}/kubelet.key" + echo "${KUBELET_KEY}" | base64 --decode > "${KUBELET_KEY_PATH}" + fi +} + # A hookpoint for setting up local devices ensure-local-disks() { for ssd in /dev/disk/by-id/google-local-ssd-*; do @@ -476,7 +497,7 @@ e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e " kube_uid: '$(echo "${KUBE_UID}" | sed -e "s/'/''/g")' initial_etcd_cluster: '$(echo "${INITIAL_ETCD_CLUSTER:-}" | sed -e "s/'/''/g")' initial_etcd_cluster_state: '$(echo "${INITIAL_ETCD_CLUSTER_STATE:-}" | sed -e "s/'/''/g")' - +ca_cert_bundle_path: '$(echo "${CA_CERT_BUNDLE_PATH:-}" | sed -e "s/'/''/g")' hostname: $(hostname -s) enable_default_storage_class: '$(echo "$ENABLE_DEFAULT_STORAGE_CLASS" | sed -e "s/'/''/g")' EOF @@ -727,11 +748,6 @@ function create-salt-master-kubelet-auth() { function create-salt-kubelet-auth() { local -r kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig" if [ ! -e "${kubelet_kubeconfig_file}" ]; then - # If there isn't a CA certificate set specifically for the kubelet, use - # the cluster CA certificate. - if [[ -z "${KUBELET_CA_CERT:-}" ]]; then - KUBELET_CA_CERT="${CA_CERT}" - fi mkdir -p /srv/salt-overlay/salt/kubelet (umask 077; cat > "${kubelet_kubeconfig_file}" < "${client_ca_file}") } # This should happen both on cluster initialization and node upgrades. @@ -780,7 +793,7 @@ users: clusters: - name: local cluster: - certificate-authority-data: ${CA_CERT} + certificate-authority-data: ${CA_CERT_BUNDLE} contexts: - context: cluster: local @@ -1129,6 +1142,7 @@ if [[ -z "${is_push}" ]]; then set-kube-env auto-upgrade ensure-local-disks + create-node-pki [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd create-salt-pillar if [[ "${KUBERNETES_MASTER}" == "true" ]]; then diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 94e71f272d8..f7df2206595 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -204,24 +204,79 @@ function replace_prefixed_line { echo "${prefix}${suffix}" >> "${file}" } +function create-node-pki { + echo "Creating node pki files" + + local -r pki_dir="/etc/srv/kubernetes/pki" + mkdir -p "${pki_dir}" + + if [[ -z "${CA_CERT_BUNDLE:-}" ]]; then + CA_CERT_BUNDLE="${CA_CERT}" + CA_CERT_BUNDLE_PATH="${pki_dir}/ca-certificates.crt" + echo "${CA_CERT_BUNDLE}" | base64 --decode > "${CA_CERT_BUNDLE_PATH}" + fi + + if [[ ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then + KUBELET_CERT_PATH="${pki_dir}/kubelet.crt" + echo "${KUBELET_CERT}" | base64 --decode > "${KUBELET_CERT_PATH}" + + KUBELET_KEY_PATH="${pki_dir}/kubelet.key" + echo "${KUBELET_KEY}" | base64 --decode > "${KUBELET_KEY_PATH}" + fi +} + +function create-master-pki { + echo "Creating master pki files" + + local -r pki_dir="/etc/srv/kubernetes/pki" + mkdir -p "${pki_dir}" + + CA_CERT_PATH="${pki_dir}/ca.crt" + echo "${CA_CERT}" | base64 --decode > "${CA_CERT_PATH}" + + # this is not true on GKE + if [[ ! -z "${CA_KEY:-}" ]]; then + CA_KEY_PATH="${pki_dir}/ca.key" + echo "${CA_KEY}" | base64 --decode > "${CA_KEY_PATH}" + fi + + if [[ -z "${APISERVER_SERVER_CERT:-}" || -z "${APISERVER_SERVER_KEY:-}" ]]; then + APISERVER_SERVER_CERT="${MASTER_CERT}" + APISERVER_SERVER_CERT_PATH="${pki_dir}/apiserver.crt" + echo "${APISERVER_SERVER_CERT}" | base64 --decode > "${APISERVER_SERVER_CERT_PATH}" + + APISERVER_SERVER_KEY="${MASTER_KEY}" + APISERVER_SERVER_KEY_PATH="${pki_dir}/apiserver.key" + echo "${APISERVER_SERVER_KEY}" | base64 --decode > "${APISERVER_SERVER_KEY_PATH}" + fi + + if [[ -z "${APISERVER_CLIENT_CERT:-}" || -z "${APISERVER_CLIENT_KEY:-}" ]]; then + APISERVER_CLIENT_CERT="${KUBEAPISERVER_CERT}" + APISERVER_CLIENT_CERT_PATH="${pki_dir}/apiserver-client.crt" + echo "${APISERVER_CLIENT_CERT}" | base64 --decode > "${APISERVER_CLIENT_CERT_PATH}" + + APISERVER_CLIENT_KEY="${KUBEAPISERVER_KEY}" + APISERVER_CLIENT_KEY_PATH="${pki_dir}/apiserver-client.key" + echo "${APISERVER_CLIENT_KEY}" | base64 --decode > "${APISERVER_CLIENT_KEY_PATH}" + fi + + if [[ -z "${SERVICEACCOUNT_CERT:-}" || -z "${SERVICEACCOUNT_KEY:-}" ]]; then + SERVICEACCOUNT_CERT="${MASTER_CERT}" + SERVICEACCOUNT_CERT_PATH="${pki_dir}/serviceaccount.crt" + echo "${SERVICEACCOUNT_CERT}" | base64 --decode > "${SERVICEACCOUNT_CERT_PATH}" + + SERVICEACCOUNT_KEY="${MASTER_KEY}" + SERVICEACCOUNT_KEY_PATH="${pki_dir}/serviceaccount.key" + echo "${SERVICEACCOUNT_KEY}" | base64 --decode > "${SERVICEACCOUNT_KEY_PATH}" + fi +} + # After the first boot and on upgrade, these files exist on the master-pd # and should never be touched again (except perhaps an additional service # account, see NB below.) function create-master-auth { echo "Creating master auth files" local -r auth_dir="/etc/srv/kubernetes" - if [[ ! -e "${auth_dir}/ca.crt" && ! -z "${CA_CERT:-}" && ! -z "${MASTER_CERT:-}" && ! -z "${MASTER_KEY:-}" ]]; then - echo "${CA_CERT}" | base64 --decode > "${auth_dir}/ca.crt" - echo "${MASTER_CERT}" | base64 --decode > "${auth_dir}/server.cert" - echo "${MASTER_KEY}" | base64 --decode > "${auth_dir}/server.key" - fi - if [[ ! -z "${CA_KEY:-}" ]]; then - echo "${CA_KEY}" | base64 --decode > "${auth_dir}/ca.key" - fi - if [ ! -e "${auth_dir}/kubeapiserver.cert" ] && [[ ! -z "${KUBEAPISERVER_CERT:-}" ]] && [[ ! -z "${KUBEAPISERVER_KEY:-}" ]]; then - echo "${KUBEAPISERVER_CERT}" | base64 --decode > "${auth_dir}/kubeapiserver.cert" - echo "${KUBEAPISERVER_KEY}" | base64 --decode > "${auth_dir}/kubeapiserver.key" - fi local -r basic_auth_csv="${auth_dir}/basic_auth.csv" if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then replace_prefixed_line "${basic_auth_csv}" "${KUBE_PASSWORD},${KUBE_USER}," "admin,system:masters" @@ -353,21 +408,18 @@ EOF function create-kubelet-kubeconfig { echo "Creating kubelet kubeconfig file" - if [[ -z "${KUBELET_CA_CERT:-}" ]]; then - KUBELET_CA_CERT="${CA_CERT}" - fi cat </var/lib/kubelet/kubeconfig apiVersion: v1 kind: Config users: - name: kubelet user: - client-certificate-data: ${KUBELET_CERT} - client-key-data: ${KUBELET_KEY} + client-certificate: ${KUBELET_CERT_PATH} + client-key: ${KUBELET_KEY_PATH} clusters: - name: local cluster: - certificate-authority-data: ${KUBELET_CA_CERT} + certificate-authority: ${CA_CERT_BUNDLE_PATH} contexts: - context: cluster: local @@ -375,7 +427,6 @@ contexts: name: service-account-context current-context: service-account-context EOF - echo "${KUBELET_CA_CERT}" | base64 -d > /var/lib/kubelet/ca.crt } # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY @@ -403,7 +454,7 @@ users: clusters: - name: local cluster: - certificate-authority-data: ${CA_CERT} + certificate-authority-data: ${CA_CERT_BUNDLE} contexts: - context: cluster: local @@ -629,7 +680,7 @@ function start-kubelet { [[ "${HAIRPIN_MODE:-}" == "none" ]]; then flags+=" --hairpin-mode=${HAIRPIN_MODE}" fi - flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=/var/lib/kubelet/ca.crt" + flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=${CA_CERT_BUNDLE_PATH}" fi # Network plugin if [[ -n "${NETWORK_PROVIDER:-}" ]]; then @@ -910,15 +961,15 @@ function start-kube-apiserver { params+=" --address=127.0.0.1" params+=" --allow-privileged=true" params+=" --cloud-provider=gce" - params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt" + params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" params+=" --etcd-servers=http://127.0.0.1:2379" params+=" --etcd-servers-overrides=/events#http://127.0.0.1:4002" params+=" --secure-port=443" - params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" - params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" - if [[ -e /etc/srv/kubernetes/kubeapiserver.cert ]] && [[ -e /etc/srv/kubernetes/kubeapiserver.key ]]; then - params+=" --kubelet-client-certificate=/etc/srv/kubernetes/kubeapiserver.cert" - params+=" --kubelet-client-key=/etc/srv/kubernetes/kubeapiserver.key" + params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" + params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" + if [[ -e "${APISERVER_CLIENT_CERT_PATH}" ]] && [[ -e "${APISERVER_CLIENT_KEY_PATH}" ]]; then + params+=" --kubelet-client-certificate=${APISERVER_CLIENT_CERT_PATH}" + params+=" --kubelet-client-key=${APISERVER_CLIENT_KEY_PATH}" fi params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then @@ -1079,8 +1130,8 @@ function start-kube-controller-manager { params+=" --use-service-account-credentials" params+=" --cloud-provider=gce" params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig" - params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt" - params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key" + params+=" --root-ca-file=${CA_CERT_BUNDLE_PATH}" + params+=" --service-account-private-key-file=${SERVICEACCOUNT_KEY_PATH}" if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" fi @@ -1091,8 +1142,8 @@ function start-kube-controller-manager { params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" fi if [[ -n "${CA_KEY:-}" ]]; then - params+=" --cluster-signing-cert-file=/etc/srv/kubernetes/ca.crt" - params+=" --cluster-signing-key-file=/etc/srv/kubernetes/ca.key" + params+=" --cluster-signing-cert-file=${CA_CERT_PATH}" + params+=" --cluster-signing-key-file=${CA_KEY_PATH}" fi if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" @@ -1470,10 +1521,13 @@ ensure-local-ssds setup-logrotate if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then mount-master-pd + create-node-pki + create-master-pki create-master-auth create-master-kubelet-auth create-master-etcd-auth else + create-node-pki create-kubelet-kubeconfig create-kubeproxy-kubeconfig if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index c3a9ed8b255..02c4ba186e1 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -188,7 +188,7 @@ {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %} {% endif -%} -{% set kubelet_auth = "--anonymous-auth=false --authorization-mode=Webhook --client-ca-file=/var/lib/kubelet/ca.crt" %} +{% set kubelet_auth = "--anonymous-auth=false --authorization-mode=Webhook --client-ca-file=" + pillar.get('ca_cert_bundle_path', '/var/lib/kubelet/ca.crt') %} # test_args has to be kept at the end, so they'll overwrite any prior configuration DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{kubelet_auth}} {{feature_gates}} {{test_args}}" diff --git a/cluster/saltbase/salt/kubelet/init.sls b/cluster/saltbase/salt/kubelet/init.sls index eb31fd6f13e..86c1f82bbd3 100644 --- a/cluster/saltbase/salt/kubelet/init.sls +++ b/cluster/saltbase/salt/kubelet/init.sls @@ -31,6 +31,7 @@ - mode: 400 - makedirs: true +{% if grains.cloud != 'gce' %} /var/lib/kubelet/ca.crt: file.managed: - source: salt://kubelet/ca.crt @@ -38,6 +39,7 @@ - group: root - mode: 400 - makedirs: true +{% endif %} {% if pillar.get('is_systemd') %} @@ -59,7 +61,9 @@ fix-service-kubelet: - file: {{ pillar.get('systemd_system_path') }}/kubelet.service - file: {{ environment_file }} - file: /var/lib/kubelet/kubeconfig +{% if grains.cloud != 'gce' %} - file: /var/lib/kubelet/ca.crt +{% endif %} {% else %} @@ -87,7 +91,9 @@ kubelet: {% endif %} - file: {{ environment_file }} - file: /var/lib/kubelet/kubeconfig +{% if grains.cloud != 'gce' %} - file: /var/lib/kubelet/ca.crt +{% endif %} {% if pillar.get('is_systemd') %} - provider: - service: systemd