diff --git a/cluster/gce/templates/format-and-mount-pd.sh b/cluster/gce/templates/mount-pd.sh similarity index 61% rename from cluster/gce/templates/format-and-mount-pd.sh rename to cluster/gce/templates/mount-pd.sh index 28b453ad994..7233ed1f2a0 100755 --- a/cluster/gce/templates/format-and-mount-pd.sh +++ b/cluster/gce/templates/mount-pd.sh @@ -14,20 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Formats and mounts a persistent disk to store the persistent data on the -# master -- etcd's data and the security certs/keys. +# Mounts a persistent disk (formatting if needed) to store the persistent data +# on the master -- etcd's data, a few settings, and security certs/keys/tokens. +# +# This script can be reused to mount an existing PD because all of its +# operations modifying the disk are idempotent -- safe_format_and_mount only +# formats an unformatted disk, and mkdir -p will leave a directory be if it +# already exists. device_info=$(ls -l /dev/disk/by-id/google-master-pd) relative_path=${device_info##* } device_path="/dev/disk/by-id/${relative_path}" -# Format and mount the disk to the directory used by etcd. +# Format and mount the disk, create directories on it for all of the master's +# persistent data, and link them to where they're used. mkdir -p /mnt/master-pd /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd +# Contains all the data stored in etcd mkdir -m 700 -p /mnt/master-pd/var/etcd +# Contains the dynamically generated apiserver auth certs and keys mkdir -p /mnt/master-pd/srv/kubernetes +# Contains the cluster's initial config parameters and auth tokens +mkdir -p /mnt/master-pd/srv/salt-overlay +# Contains salt's dynamically generated RSA keys +mkdir -m 770 -p /mnt/master-pd/etc/salt/pki ln -s /mnt/master-pd/var/etcd /var/etcd ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes +ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay +ln -s /mnt/master-pd/etc/salt/pki /etc/salt/pki # This is a bit of a hack to get around the fact that salt has to run after the # PD and mounted directory are already set up. We can't give ownership of the diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index e92a0df9f9c..d462659f5f5 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -433,6 +433,13 @@ function kube-up { --target-tags "${MASTER_TAG}" \ --allow tcp:443 & + # We have to make sure the disk is created before creating the master VM, so + # run this in the foreground. + gcloud compute disks create "${MASTER_NAME}-pd" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --size "10GB" + ( echo "#! /bin/bash" echo "mkdir -p /var/cache/kubernetes-install" @@ -454,28 +461,12 @@ function kube-up { echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/format-and-mount-pd.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/mount-pd.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh" ) > "${KUBE_TEMP}/master-start.sh" - # Report logging choice (if any). - if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then - echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}" - # For logging to GCP we need to enable some minion scopes. - if [[ "${LOGGING_DESTINATION-}" == "gcp" ]]; then - MINION_SCOPES+=('https://www.googleapis.com/auth/logging.write') - fi - fi - - # We have to make sure the disk is created before creating the master VM, so - # run this in the foreground. - gcloud compute disks create "${MASTER_NAME}-pd" \ - --project "${PROJECT}" \ - --zone "${ZONE}" \ - --size "10GB" - gcloud compute instances create "${MASTER_NAME}" \ --project "${PROJECT}" \ --zone "${ZONE}" \ @@ -491,6 +482,15 @@ function kube-up { # Create a single firewall rule for all minions. create-firewall-rule "${MINION_TAG}-all" "${CLUSTER_IP_RANGE}" "${MINION_TAG}" & + # Report logging choice (if any). + if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then + echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}" + # For logging to GCP we need to enable some minion scopes. + if [[ "${LOGGING_DESTINATION-}" == "gcp" ]]; then + MINION_SCOPES+=('https://www.googleapis.com/auth/logging.write') + fi + fi + # Wait for last batch of jobs. wait-for-jobs @@ -548,6 +548,16 @@ function kube-up { detect-master + # Reserve the master's IP so that it can later be transferred to another VM + # without disrupting the kubelets. IPs are associated with regions, not zones, + # so extract the region name, which is the same as the zone but with the final + # dash and characters trailing the dash removed. + local REGION=${ZONE%-*} + gcloud compute addresses create "${MASTER_NAME}-ip" \ + --project "${PROJECT}" \ + --addresses "${KUBE_MASTER_IP}" \ + --region "${REGION}" + echo "Waiting for cluster initialization." echo echo " This will continually check to see if the API for kubernetes is reachable." @@ -727,6 +737,14 @@ function kube-down { routes=( "${routes[@]:10}" ) done + # Delete the master's reserved IP + local REGION=${ZONE%-*} + gcloud compute addresses delete \ + --project "${PROJECT}" \ + --region "${REGION}" \ + --quiet \ + "${MASTER_NAME}-ip" || true + } # Update a kubernetes cluster with latest source