From cf23d4c939fee32bed17497ccf7d4f5640011db4 Mon Sep 17 00:00:00 2001 From: Chris Glass Date: Tue, 18 Apr 2017 20:47:23 +0200 Subject: [PATCH 1/2] Add Ubuntu as a platform for e2e tests on GCE Using Ubuntu on GCE to run cluster e2e tests requires slightly different node.yaml and master.yaml files than GCI, because Ubuntu uses systemd as PID 1, wheras GCI uses upstart with a systemd delegate. Therefore the e2e tests fail using those files since the kubernetes services are not brought back up after a node/master reboot. --- cluster/common.sh | 12 +-- cluster/gce/ubuntu/README.md | 12 +++ cluster/gce/ubuntu/master-helper.sh | 111 ++++++++++++++++++++++++ cluster/gce/ubuntu/master.yaml | 128 ++++++++++++++++++++++++++++ cluster/gce/ubuntu/node-helper.sh | 32 +++++++ cluster/gce/ubuntu/node.yaml | 128 ++++++++++++++++++++++++++++ cluster/gce/util.sh | 16 ++-- 7 files changed, 429 insertions(+), 10 deletions(-) create mode 100644 cluster/gce/ubuntu/README.md create mode 100644 cluster/gce/ubuntu/master-helper.sh create mode 100644 cluster/gce/ubuntu/master.yaml create mode 100644 cluster/gce/ubuntu/node-helper.sh create mode 100644 cluster/gce/ubuntu/node.yaml diff --git a/cluster/common.sh b/cluster/common.sh index b89fdd53455..58e77d38341 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -450,8 +450,8 @@ function find-release-tars() { # This tarball is used by GCI, Ubuntu Trusty, and Container Linux. KUBE_MANIFESTS_TAR= - if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" ]] || \ - [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" ]] ; then + if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \ + [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" ]] ; then KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz) fi } @@ -587,7 +587,9 @@ function build-kube-env { local salt_tar_url=$SALT_TAR_URL local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}" if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "container-linux" ]] || \ - [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] ; then + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] || \ + [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]] ; then # TODO: Support fallback .tar.gz settings on Container Linux server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}") salt_tar_url=$(split_csv "${SALT_TAR_URL}") @@ -671,8 +673,8 @@ EOF TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD}) EOF fi - if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") ]] || \ - [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") ]] ; then + if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") || "${NODE_OS_DISTRIBUTION}" = "ubuntu" ]] ; then cat >>$file < ${KUBE_TEMP}/master-kube-env.yaml + get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" kube-master-certs > "${KUBE_TEMP}/kube-master-certs.yaml" + + create-master-instance-internal "${REPLICA_NAME}" +} + + +function create-master-instance-internal() { + local -r master_name="${1}" + local -r address_option="${2:-}" + + local preemptible_master="" + if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then + preemptible_master="--preemptible --maintenance-policy TERMINATE" + fi + + gcloud compute instances create "${master_name}" \ + ${address_option} \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --machine-type "${MASTER_SIZE}" \ + --image-project="${MASTER_IMAGE_PROJECT}" \ + --image "${MASTER_IMAGE}" \ + --tags "${MASTER_TAG}" \ + --network "${NETWORK}" \ + --scopes "storage-ro,compute-rw,monitoring,logging-write" \ + --can-ip-forward \ + --metadata-from-file \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/ubuntu/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \ + --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ + --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ + ${preemptible_master} +} + +function get-metadata() { + local zone="${1}" + local name="${2}" + local key="${3}" + gcloud compute ssh "${name}" \ + --project "${PROJECT}" \ + --zone "${zone}" \ + --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null +} diff --git a/cluster/gce/ubuntu/master.yaml b/cluster/gce/ubuntu/master.yaml new file mode 100644 index 00000000000..07d7edc308b --- /dev/null +++ b/cluster/gce/ubuntu/master.yaml @@ -0,0 +1,128 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + After=docker.service + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/cluster/gce/ubuntu/node-helper.sh b/cluster/gce/ubuntu/node-helper.sh new file mode 100644 index 00000000000..38fa0d79817 --- /dev/null +++ b/cluster/gce/ubuntu/node-helper.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for GCI distro +source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" + +# $1: template name (required). +function create-node-instance-template { + local template_name="$1" + ensure-gci-metadata-files + create-node-template "$template_name" "${scope_flags[*]}" \ + "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ + "user-data=${KUBE_ROOT}/cluster/gce/ubuntu/node.yaml" \ + "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \ + "cluster-name=${KUBE_TEMP}/cluster-name.txt" \ + "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \ + "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \ + "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" +} diff --git a/cluster/gce/ubuntu/node.yaml b/cluster/gce/ubuntu/node.yaml new file mode 100644 index 00000000000..ede0943323b --- /dev/null +++ b/cluster/gce/ubuntu/node.yaml @@ -0,0 +1,128 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + After=docker.service + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 6a7cfc7464f..4099430b3d0 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -23,14 +23,14 @@ source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" source "${KUBE_ROOT}/hack/lib/util.sh" -if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then +if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" else echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2 exit 1 fi -if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then +if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" else echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2 @@ -45,10 +45,13 @@ if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then elif [[ "${MASTER_OS_DISTRIBUTION}" == "debian" ]]; then MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${CVM_VERSION}} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} +elif [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then + MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE} + MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT} fi # Sets node image based on the specified os distro. Currently this function only -# supports gci and debian. +# supports gci, debian, and ubuntu. function set-node-image() { if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then # If the node image is not set, we use the latest GCI image. @@ -58,6 +61,9 @@ function set-node-image() { elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} + elif [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then + NODE_IMAGE=${KUBE_GCE_NODE_IMAGE} + NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT} fi } @@ -643,8 +649,8 @@ function kube-up() { create-subnetwork create-nodes elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then - if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then - echo "Master replication supported only for gci and debian" + if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then + echo "Master replication supported only for gci, debian, and ubuntu" return 1 fi create-loadbalancer From 0d66fc97cd73f2e7fcf2ff7152ae4a32550a08c6 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 25 Apr 2017 16:03:27 -0700 Subject: [PATCH 2/2] Makes cluster/gce/ubuntu to be a symlink to cluster/gce/gci and changes the gci's [master|node].yaml to enable kubernetes.target. This enables Ubuntu to use the same provisioning scripts as GCI's. The change for enabling kubernetes.target is needed for Ubuntu but a no-op for GCI. --- cluster/common.sh | 2 +- cluster/gce/gci/master.yaml | 4 + cluster/gce/gci/node.yaml | 4 + cluster/gce/ubuntu | 1 + cluster/gce/ubuntu/README.md | 12 --- cluster/gce/ubuntu/master-helper.sh | 111 ------------------------ cluster/gce/ubuntu/master.yaml | 128 ---------------------------- cluster/gce/ubuntu/node-helper.sh | 32 ------- cluster/gce/ubuntu/node.yaml | 128 ---------------------------- cluster/gce/util.sh | 10 +-- 10 files changed, 12 insertions(+), 420 deletions(-) create mode 120000 cluster/gce/ubuntu delete mode 100644 cluster/gce/ubuntu/README.md delete mode 100644 cluster/gce/ubuntu/master-helper.sh delete mode 100644 cluster/gce/ubuntu/master.yaml delete mode 100644 cluster/gce/ubuntu/node-helper.sh delete mode 100644 cluster/gce/ubuntu/node.yaml diff --git a/cluster/common.sh b/cluster/common.sh index 58e77d38341..7cf354be8b1 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015 The Kubernetes Authors. +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cluster/gce/gci/master.yaml b/cluster/gce/gci/master.yaml index 845973df84b..7854ab4fa93 100644 --- a/cluster/gce/gci/master.yaml +++ b/cluster/gce/gci/master.yaml @@ -112,6 +112,9 @@ write_files: [Unit] Description=Kubernetes + [Install] + WantedBy=multi-user.target + runcmd: - systemctl daemon-reload - systemctl enable kube-master-installation.service @@ -120,4 +123,5 @@ runcmd: - systemctl enable kubelet-monitor.service - systemctl enable kube-logrotate.timer - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target - systemctl start kubernetes.target diff --git a/cluster/gce/gci/node.yaml b/cluster/gce/gci/node.yaml index de59fcac27b..52971e2a076 100644 --- a/cluster/gce/gci/node.yaml +++ b/cluster/gce/gci/node.yaml @@ -112,6 +112,9 @@ write_files: [Unit] Description=Kubernetes + [Install] + WantedBy=multi-user.target + runcmd: - systemctl daemon-reload - systemctl enable kube-node-installation.service @@ -120,4 +123,5 @@ runcmd: - systemctl enable kubelet-monitor.service - systemctl enable kube-logrotate.timer - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target - systemctl start kubernetes.target diff --git a/cluster/gce/ubuntu b/cluster/gce/ubuntu new file mode 120000 index 00000000000..67a1dec2892 --- /dev/null +++ b/cluster/gce/ubuntu @@ -0,0 +1 @@ +gci \ No newline at end of file diff --git a/cluster/gce/ubuntu/README.md b/cluster/gce/ubuntu/README.md deleted file mode 100644 index c7dc6af10f1..00000000000 --- a/cluster/gce/ubuntu/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Ubuntu Image - -The Ubuntu GKE image is an image optimized to run Kubernetes on the Google -Cloud Platform. - -The image is currently made to behave as much as possible like the GCI image, -but since it uses "real" systemd (instead of upstart with a delegate), the systemd -targets need to be changed slightly to ensure services are always started on -boot. - -In practice this means that for now the only changes are in master.yaml and node.yaml, -as well as their injection points in node-helper.sh and master-helper.sh. diff --git a/cluster/gce/ubuntu/master-helper.sh b/cluster/gce/ubuntu/master-helper.sh deleted file mode 100644 index 4d0a7d66fc9..00000000000 --- a/cluster/gce/ubuntu/master-helper.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions and constant for GCI distro -source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" - -# create-master-instance creates the master instance. If called with -# an argument, the argument is used as the name to a reserved IP -# address for the master. (In the case of upgrade/repair, we re-use -# the same IP.) -# -# It requires a whole slew of assumed variables, partially due to to -# the call to write-master-env. Listing them would be rather -# futile. Instead, we list the required calls to ensure any additional -# -# variables are set: -# ensure-temp-dir -# detect-project -# get-bearer-token -function create-master-instance { - local address_opt="" - [[ -n ${1:-} ]] && address_opt="--address ${1}" - - write-master-env - ensure-gci-metadata-files - create-master-instance-internal "${MASTER_NAME}" "${address_opt}" -} - -function replicate-master-instance() { - local existing_master_zone="${1}" - local existing_master_name="${2}" - local existing_master_replicas="${3}" - - local kube_env="$(get-metadata "${existing_master_zone}" "${existing_master_name}" kube-env)" - # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering. - kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")" - kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")" - - # Substitute INITIAL_ETCD_CLUSTER_STATE - kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER_STATE")" - kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER_STATE: 'existing'")" - - ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")" - ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")" - create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}" - - kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")" - kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")" - kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")" - kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")" - - echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml - get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" kube-master-certs > "${KUBE_TEMP}/kube-master-certs.yaml" - - create-master-instance-internal "${REPLICA_NAME}" -} - - -function create-master-instance-internal() { - local -r master_name="${1}" - local -r address_option="${2:-}" - - local preemptible_master="" - if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then - preemptible_master="--preemptible --maintenance-policy TERMINATE" - fi - - gcloud compute instances create "${master_name}" \ - ${address_option} \ - --project "${PROJECT}" \ - --zone "${ZONE}" \ - --machine-type "${MASTER_SIZE}" \ - --image-project="${MASTER_IMAGE_PROJECT}" \ - --image "${MASTER_IMAGE}" \ - --tags "${MASTER_TAG}" \ - --network "${NETWORK}" \ - --scopes "storage-ro,compute-rw,monitoring,logging-write" \ - --can-ip-forward \ - --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/ubuntu/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \ - --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ - --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ - ${preemptible_master} -} - -function get-metadata() { - local zone="${1}" - local name="${2}" - local key="${3}" - gcloud compute ssh "${name}" \ - --project "${PROJECT}" \ - --zone "${zone}" \ - --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null -} diff --git a/cluster/gce/ubuntu/master.yaml b/cluster/gce/ubuntu/master.yaml deleted file mode 100644 index 07d7edc308b..00000000000 --- a/cluster/gce/ubuntu/master.yaml +++ /dev/null @@ -1,128 +0,0 @@ -#cloud-config - -write_files: - - path: /etc/systemd/system/kube-master-installation.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Download and install k8s binaries and configurations - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/kubernetes/bin - ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin - ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh - ExecStart=/home/kubernetes/bin/configure.sh - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-master-configuration.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Configure kubernetes master - After=kube-master-installation.service - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh - ExecStart=/home/kubernetes/bin/configure-helper.sh - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-docker-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for docker - After=kube-master-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh docker - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubelet-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for kubelet - After=kube-master-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.timer - permissions: 0644 - owner: root - content: | - [Unit] - Description=Hourly kube-logrotate invocation - - [Timer] - OnCalendar=hourly - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes log rotation - After=kube-master-configuration.service - - [Service] - Type=oneshot - ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubernetes.target - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes - - [Install] - WantedBy=multi-user.target - After=docker.service - -runcmd: - - systemctl daemon-reload - - systemctl enable kube-master-installation.service - - systemctl enable kube-master-configuration.service - - systemctl enable kube-docker-monitor.service - - systemctl enable kubelet-monitor.service - - systemctl enable kube-logrotate.timer - - systemctl enable kube-logrotate.service - - systemctl enable kubernetes.target - - systemctl start kubernetes.target diff --git a/cluster/gce/ubuntu/node-helper.sh b/cluster/gce/ubuntu/node-helper.sh deleted file mode 100644 index 38fa0d79817..00000000000 --- a/cluster/gce/ubuntu/node-helper.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions and constant for GCI distro -source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" - -# $1: template name (required). -function create-node-instance-template { - local template_name="$1" - ensure-gci-metadata-files - create-node-template "$template_name" "${scope_flags[*]}" \ - "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ - "user-data=${KUBE_ROOT}/cluster/gce/ubuntu/node.yaml" \ - "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \ - "cluster-name=${KUBE_TEMP}/cluster-name.txt" \ - "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \ - "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \ - "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" -} diff --git a/cluster/gce/ubuntu/node.yaml b/cluster/gce/ubuntu/node.yaml deleted file mode 100644 index ede0943323b..00000000000 --- a/cluster/gce/ubuntu/node.yaml +++ /dev/null @@ -1,128 +0,0 @@ -#cloud-config - -write_files: - - path: /etc/systemd/system/kube-node-installation.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Download and install k8s binaries and configurations - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/kubernetes/bin - ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin - ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh - ExecStart=/home/kubernetes/bin/configure.sh - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-node-configuration.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Configure kubernetes node - After=kube-node-installation.service - - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh - ExecStart=/home/kubernetes/bin/configure-helper.sh - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-docker-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for docker - After=kube-node-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh docker - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubelet-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for kubelet - After=kube-node-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.timer - permissions: 0644 - owner: root - content: | - [Unit] - Description=Hourly kube-logrotate invocation - - [Timer] - OnCalendar=hourly - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes log rotation - After=kube-node-configuration.service - - [Service] - Type=oneshot - ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubernetes.target - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes - - [Install] - WantedBy=multi-user.target - After=docker.service - -runcmd: - - systemctl daemon-reload - - systemctl enable kube-node-installation.service - - systemctl enable kube-node-configuration.service - - systemctl enable kube-docker-monitor.service - - systemctl enable kubelet-monitor.service - - systemctl enable kube-logrotate.timer - - systemctl enable kube-logrotate.service - - systemctl enable kubernetes.target - - systemctl start kubernetes.target diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 4099430b3d0..27d47e925cc 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2014 The Kubernetes Authors. +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -45,13 +45,10 @@ if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then elif [[ "${MASTER_OS_DISTRIBUTION}" == "debian" ]]; then MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${CVM_VERSION}} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} -elif [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then - MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE} - MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT} fi # Sets node image based on the specified os distro. Currently this function only -# supports gci, debian, and ubuntu. +# supports gci and debian. function set-node-image() { if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then # If the node image is not set, we use the latest GCI image. @@ -61,9 +58,6 @@ function set-node-image() { elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} - elif [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then - NODE_IMAGE=${KUBE_GCE_NODE_IMAGE} - NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT} fi }