From 13afe18ab4727eb9b15a2ab5356d17d835aca6db Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Tue, 1 Nov 2016 14:29:04 -0700 Subject: [PATCH] cluster/coreos: update to gci based implementation This update includes significant refactoring. It moves almost all of the logic into bash scripts, modeled after the `gci` cluster scripts. The primary differences between the two are the following: 1. Use of the `/opt/kubernetes` directory over `/home/kubernetes` 2. Support for rkt as a runtime 3. No use of logrotate 4. No use of `/etc/default/` 5. No logic related to noexec mounts or gci-specific firewall-stuff --- build/lib/release.sh | 4 +- .../dashboard/dashboard-controller.yaml | 1 - .../addons/dashboard/dashboard-service.yaml | 1 - cluster/gce/coreos/README.md | 13 +- cluster/gce/coreos/configure-helper.sh | 322 ++++++++---------- cluster/gce/coreos/configure.sh | 63 +--- cluster/gce/coreos/helper.sh | 17 +- cluster/gce/coreos/master-helper.sh | 12 +- cluster/gce/coreos/master.yaml | 166 +++------ cluster/gce/coreos/node-helper.sh | 16 +- cluster/gce/coreos/node.yaml | 166 +++------ 11 files changed, 276 insertions(+), 505 deletions(-) mode change 100644 => 100755 cluster/gce/coreos/configure-helper.sh mode change 100644 => 100755 cluster/gce/coreos/configure.sh diff --git a/build/lib/release.sh b/build/lib/release.sh index acf43de985b..42cc019ce71 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -331,14 +331,12 @@ function kube::release::package_kube_manifests_tarball() { cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh" cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter" cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh" + cp "${KUBE_ROOT}/cluster/gce/coreos/configure-helper.sh" "${dst_dir}/coreos-configure-helper.sh" cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}" local objects objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo) tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}" - # This is for coreos only. ContainerVM, GCI, or Trusty does not use it. - cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/" - kube::release::clean_cruft local package_name="${RELEASE_DIR}/kubernetes-manifests.tar.gz" diff --git a/cluster/addons/dashboard/dashboard-controller.yaml b/cluster/addons/dashboard/dashboard-controller.yaml index 8f0eebe88cc..78f942f5231 100644 --- a/cluster/addons/dashboard/dashboard-controller.yaml +++ b/cluster/addons/dashboard/dashboard-controller.yaml @@ -1,4 +1,3 @@ -# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml apiVersion: extensions/v1beta1 kind: Deployment metadata: diff --git a/cluster/addons/dashboard/dashboard-service.yaml b/cluster/addons/dashboard/dashboard-service.yaml index 2c1c02f1139..195b503de10 100644 --- a/cluster/addons/dashboard/dashboard-service.yaml +++ b/cluster/addons/dashboard/dashboard-service.yaml @@ -1,4 +1,3 @@ -# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml apiVersion: v1 kind: Service metadata: diff --git a/cluster/gce/coreos/README.md b/cluster/gce/coreos/README.md index 64d36a8a5ae..e285379d437 100644 --- a/cluster/gce/coreos/README.md +++ b/cluster/gce/coreos/README.md @@ -1,11 +1,8 @@ -# Container-VM Image +# CoreOS image -[Container-VM Image](https://cloud.google.com/compute/docs/containers/vm-image/) -is a container-optimized OS image for the Google Cloud Platform (GCP). It is -primarily for running Google services on GCP. Unlike the open preview version -of container-vm, the new Container-VM Image is based on the open source -ChromiumOS project, allowing us greater control over the build management, -security compliance, and customizations for GCP. +The [CoreOS operating system](https://coreos.com/why/) is a Linux distribution optimized for running containers securely at scale. +CoreOS provides [an image](https://coreos.com/os/docs/latest/booting-on-google-compute-engine.html) for Google Cloud Platform (GCP). +This folder contains configuration and tooling to allow kube-up to create a Kubernetes cluster on Google Cloud Platform running on the official CoreOS image. -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/gci/README.md?pixel)]() +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/coreos/README.md?pixel)]() diff --git a/cluster/gce/coreos/configure-helper.sh b/cluster/gce/coreos/configure-helper.sh old mode 100644 new mode 100755 index 01bfbff6e96..c3beed73f65 --- a/cluster/gce/coreos/configure-helper.sh +++ b/cluster/gce/coreos/configure-helper.sh @@ -19,37 +19,12 @@ # TODO: this script duplicates templating logic from cluster/saltbase/salt # using sed. It should use an actual template parser on the manifest -# files. +# files, or the manifest files should not be templated salt set -o errexit set -o nounset set -o pipefail -function setup-os-params { - # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to - # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for - # now, set a generic core_pattern that users can work with. - echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern -} - -function config-ip-firewall { - echo "Configuring IP firewall rules" - # The GCI image has host firewall which drop most inbound/forwarded packets. - # We need to add rules to accept all TCP/UDP/ICMP packets. - if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then - echo "Add rules to accept all inbound TCP/UDP/ICMP packets" - iptables -A INPUT -w -p TCP -j ACCEPT - iptables -A INPUT -w -p UDP -j ACCEPT - iptables -A INPUT -w -p ICMP -j ACCEPT - fi - if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then - echo "Add rules to accept all forwarded TCP/UDP/ICMP packets" - iptables -A FORWARD -w -p TCP -j ACCEPT - iptables -A FORWARD -w -p UDP -j ACCEPT - iptables -A FORWARD -w -p ICMP -j ACCEPT - fi -} - function create-dirs { echo "Creating required directories" mkdir -p /var/lib/kubelet @@ -59,6 +34,13 @@ function create-dirs { fi } +# Create directories referenced in the kube-controller-manager manifest for +# bindmounts. This is used under the rkt runtime to work around +# https://github.com/kubernetes/kubernetes/issues/26816 +function create-kube-controller-manager-dirs { + mkdir -p /etc/srv/kubernetes /var/ssl /etc/{ssl,openssl,pki} +} + # Formats the given device ($1) if needed and mounts it at given mount point # ($2). function safe-format-and-mount() { @@ -92,51 +74,6 @@ function ensure-local-ssds() { done } -# Installs logrotate configuration files -function setup-logrotate() { - mkdir -p /etc/logrotate.d/ - cat >/etc/logrotate.d/docker-containers < 100Mb OR if one day has elapsed - # * save rotated logs into a gzipped timestamped backup - # * log file timestamp (controlled by 'dateformat') includes seconds too. This - # ensures that logrotate can generate unique logfiles during each rotation - # (otherwise it skips rotation if 'maxsize' is reached multiple times in a - # day). - # * keep only 5 old (rotated) logs, and will discard older logs. - cat > /etc/logrotate.d/allvarlogs < /etc/default/docker - - if [[ "${use_net_plugin}" == "true" ]]; then - # If using a network plugin, extend the docker configuration to always remove - # the network checkpoint to avoid corrupt checkpoints. - # (https://github.com/docker/docker/issues/18283). - echo "Extend the default docker.service configuration" - mkdir -p /etc/systemd/system/docker.service.d - cat </etc/systemd/system/docker.service.d/01network.conf + mkdir -p /etc/systemd/system/docker.service.d/ + local kubernetes_conf_dropin="/etc/systemd/system/docker.service.d/00_kubelet.conf" + cat > "${kubernetes_conf_dropin}" < "${kubelet_env_file}" # Write the systemd service file for kubelet. @@ -600,7 +539,7 @@ function start-kube-proxy { if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then kube_docker_registry=${KUBE_DOCKER_REGISTRY} fi - local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag) + local -r kube_proxy_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-proxy.docker_tag) local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}" if [[ -n "${FEATURE_GATES:-}" ]]; then @@ -618,6 +557,18 @@ function start-kube-proxy { if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file} fi + if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then + # Work arounds for https://github.com/coreos/rkt/issues/3245 and https://github.com/coreos/rkt/issues/3264 + # This is an incredibly hacky workaround. It's fragile too. If the kube-proxy command changes too much, this breaks + # TODO, this could be done much better in many other places, such as an + # init script within the container, or even within kube-proxy's code. + local extra_workaround_cmd="ln -sf /proc/self/mounts /etc/mtab; \ + mount -o remount,rw /proc; \ + mount -o remount,rw /proc/sys; \ + mount -o remount,rw /sys; " + sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}" + fi + cp "${src_file}" /etc/kubernetes/manifests } @@ -629,7 +580,7 @@ function start-kube-proxy { # $4: value for variable 'cpulimit' # $5: pod name, which should be either etcd or etcd-events function prepare-etcd-manifest { - local host_name=$(hostname) + local host_name=$(hostname -s) local etcd_cluster="" local cluster_state="new" local etcd_protocol="http" @@ -671,6 +622,7 @@ function prepare-etcd-manifest { else sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}" fi + sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}" sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}" if [[ -n "${ETCD_VERSION:-}" ]]; then @@ -862,7 +814,7 @@ function start-kube-apiserver { src_file="${src_dir}/kube-apiserver.manifest" remove-salt-config-comments "${src_file}" # Evaluate variables. - local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag) + local -r kube_apiserver_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-apiserver.docker_tag) sed -i -e "s@{{params}}@${params}@g" "${src_file}" sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}" @@ -927,7 +879,7 @@ function start-kube-controller-manager { if [[ -n "${FEATURE_GATES:-}" ]]; then params+=" --feature-gates=${FEATURE_GATES}" fi - local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) + local -r kube_rc_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest" remove-salt-config-comments "${src_file}" @@ -1182,63 +1134,74 @@ function start-rescheduler { fi } -# Setup working directory for kubelet. -function setup-kubelet-dir { - echo "Making /var/lib/kubelet executable for kubelet" - mount -B /var/lib/kubelet /var/lib/kubelet/ - mount -B -o remount,exec,suid,dev /var/lib/kubelet -} +# Install and setup rkt +# TODO(euank): There should be a toggle to use the distro-provided rkt binary +# Sets the following variables: +# RKT_BIN: the path to the rkt binary +function setup-rkt { + local rkt_bin="${KUBE_HOME}/bin/rkt" + if [[ -x "${rkt_bin}" ]]; then + # idempotency, skip downloading this time + # TODO(euank): this might get in the way of updates, but 'file busy' + # because of rkt-api would too + RKT_BIN="${rkt_bin}" + return + fi + mkdir -p /etc/rkt "${KUBE_HOME}/download/" + local rkt_tar="${KUBE_HOME}/download/rkt.tar.gz" + local rkt_tmpdir=$(mktemp -d "${KUBE_HOME}/rkt_download.XXXXX") + curl --retry 5 --retry-delay 3 --fail --silent --show-error \ + --location --create-dirs --output "${rkt_tar}" \ + https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz + tar --strip-components=1 -xf "${rkt_tar}" -C "${rkt_tmpdir}" --overwrite + mv "${rkt_tmpdir}/rkt" "${rkt_bin}" + if [[ ! -x "${rkt_bin}" ]]; then + echo "Could not download requested rkt binary" + exit 1 + fi + RKT_BIN="${rkt_bin}" + # Cache rkt stage1 images for speed + "${RKT_BIN}" fetch --insecure-options=image "${rkt_tmpdir}"/*.aci + rm -rf "${rkt_tmpdir}" -function reset-motd { - # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl) - local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")" - # This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1), - # or the git hash that's in the build info. - local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")" - local devel="" - if [[ "${gitref}" != "${version}" ]]; then - devel=" -Note: This looks like a development version, which might not be present on GitHub. -If it isn't, the closest tag is at: - https://github.com/kubernetes/kubernetes/tree/${gitref} -" - gitref="${version//*+/}" - fi - cat > /etc/motd < /etc/systemd/system/rkt-api.service < /etc/profile.d/kube_env.sh -} - -function pre-warm-mounter { - echo "prewarming mounter" - ${KUBE_HOME}/bin/mounter &> /dev/null +# Install docker2aci, needed to load server images if using rkt runtime +# This should be removed once rkt can fetch on-disk docker tarballs directly +# Sets the following variables: +# DOCKER2ACI_BIN: the path to the docker2aci binary +function install-docker2aci { + local tar_path="${KUBE_HOME}/download/docker2aci.tar.gz" + local tmp_path="${KUBE_HOME}/docker2aci" + mkdir -p "${KUBE_HOME}/download/" "${tmp_path}" + curl --retry 5 --retry-delay 3 --fail --silent --show-error \ + --location --create-dirs --output "${tar_path}" \ + https://github.com/appc/docker2aci/releases/download/v0.14.0/docker2aci-v0.14.0.tar.gz + tar --strip-components=1 -xf "${tar_path}" -C "${tmp_path}" --overwrite + DOCKER2ACI_BIN="${KUBE_HOME}/bin/docker2aci" + mv "${tmp_path}/docker2aci" "${DOCKER2ACI_BIN}" } ########### Main Function ########### echo "Start to configure instance for kubernetes" -KUBE_HOME="/home/kubernetes" +# Note: this name doesn't make as much sense here as in gci where it's actually +# /home/kubernetes, but for ease of diff-ing, retain the same variable name +KUBE_HOME="/opt/kubernetes" if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization." exit 1 @@ -1253,12 +1216,13 @@ if [[ -n "${KUBE_USER:-}" ]]; then fi fi -setup-os-params -config-ip-firewall +# KUBERNETES_CONTAINER_RUNTIME is set by the `kube-env` file, but it's a bit of a mouthful +if [[ "${CONTAINER_RUNTIME:-}" == "" ]]; then + CONTAINER_RUNTIME="${KUBERNETES_CONTAINER_RUNTIME:-docker}" +fi + create-dirs -setup-kubelet-dir ensure-local-ssds -setup-logrotate if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then mount-master-pd create-master-auth @@ -1269,10 +1233,16 @@ else create-kubeproxy-kubeconfig fi -override-kubectl -# Run the containerized mounter once to pre-cache the container image. -pre-warm-mounter -assemble-docker-flags +if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then + systemctl stop docker + systemctl disable docker + setup-rkt + install-docker2aci + create-kube-controller-manager-dirs +else + configure-docker-daemon +fi + load-docker-images start-kubelet @@ -1298,5 +1268,5 @@ else start-image-puller fi fi -reset-motd +start-fluentd echo "Done for the configuration for kubernetes" diff --git a/cluster/gce/coreos/configure.sh b/cluster/gce/coreos/configure.sh old mode 100644 new mode 100755 index 11fe6f08d67..00a1602c5a2 --- a/cluster/gce/coreos/configure.sh +++ b/cluster/gce/coreos/configure.sh @@ -14,30 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Due to the GCE custom metadata size limit, we split the entire script into two -# files configure.sh and configure-helper.sh. The functionality of downloading -# kubernetes configuration, manifests, docker images, and binary files are -# put in configure.sh, which is uploaded via GCE custom metadata. - set -o errexit set -o nounset set -o pipefail -function set-broken-motd { - cat > /etc/motd < "${KUBE_HOME}/kube-env") + sed 's/: /=/' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env" rm -f "${tmp_kube_env}" } @@ -65,6 +41,7 @@ function validate-hash { fi } + # Retry a download until we get it. Takes a hash and a set of URLs. # # $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. @@ -98,24 +75,8 @@ function split-commas { echo $1 | tr "," "\n" } -function install-gci-mounter-tools { - local -r rkt_version="v1.18.0" - local -r gci_mounter_version="v2" - local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967" - local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091" - local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561" - download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt" - download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci" - download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci" - local -r rkt_dst="${KUBE_HOME}/bin/" - mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt" - mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci" - mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci" - chmod a+x "${rkt_dst}/rkt" -} - # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them, -# and places them into suitable directories. Files are placed in /home/kubernetes. +# and places them into suitable directories. Files are placed in /opt/kubernetes. function install-kube-binary-config { cd "${KUBE_HOME}" local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") ) @@ -186,14 +147,9 @@ function install-kube-binary-config { find "${dst_dir}" -name \*.manifest -or -name \*.json | \ xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" fi - cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" - cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter" - cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh" + cp "${dst_dir}/kubernetes/gci-trusty/coreos-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" chmod -R 755 "${kube_bin}" - # Install gci mounter related artifacts to allow mounting storage volumes in GCI - install-gci-mounter-tools - # Clean up. rm -rf "${KUBE_HOME}/kubernetes" rm -f "${KUBE_HOME}/${server_binary_tar}" @@ -204,10 +160,17 @@ function install-kube-binary-config { ######### Main Function ########## echo "Start to install kubernetes files" -set-broken-motd -KUBE_HOME="/home/kubernetes" +KUBE_HOME="/opt/kubernetes" +mkdir -p "${KUBE_HOME}" download-kube-env source "${KUBE_HOME}/kube-env" install-kube-binary-config echo "Done for installing kubernetes files" +# On CoreOS, the hosts is in /usr/share/baselayout/hosts +# So we need to manually populdate the hosts file here on gce. +echo "127.0.0.1 localhost" >> /etc/hosts +echo "::1 localhost" >> /etc/hosts + +echo "Configuring hostname" +hostnamectl set-hostname $(hostname | cut -f1 -d.) diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index e0486c80c7b..714f49d231c 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -14,19 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A library of helper functions and constant for GCI distro +# A library of helper functions and constants for the CoreOS distro -# Creates the GCI specific metadata files if they do not exit. -# Assumed var -# KUBE_TEMP -function ensure-gci-metadata-files { - if [[ ! -f "${KUBE_TEMP}/gci-update.txt" ]]; then - echo -n "update_disabled" > "${KUBE_TEMP}/gci-update.txt" - fi - if [[ ! -f "${KUBE_TEMP}/gci-ensure-gke-docker.txt" ]]; then - echo -n "true" > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" - fi - if [[ ! -f "${KUBE_TEMP}/gci-docker-version.txt" ]]; then - echo -n "${GCI_DOCKER_VERSION:-}" > "${KUBE_TEMP}/gci-docker-version.txt" - fi -} +# This file intentionally left blank diff --git a/cluster/gce/coreos/master-helper.sh b/cluster/gce/coreos/master-helper.sh index db67224f84e..c43ce748510 100755 --- a/cluster/gce/coreos/master-helper.sh +++ b/cluster/gce/coreos/master-helper.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A library of helper functions and constant for GCI distro -source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" +# A library of helper functions and constant for coreos os distro +source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh" # create-master-instance creates the master instance. If called with # an argument, the argument is used as the name to a reserved IP @@ -35,7 +35,6 @@ function create-master-instance { [[ -n ${1:-} ]] && address_opt="--address ${1}" write-master-env - ensure-gci-metadata-files create-master-instance-internal "${MASTER_NAME}" "${address_opt}" } @@ -60,9 +59,6 @@ function replicate-master-instance() { echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" - get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt" create-master-instance-internal "${REPLICA_NAME}" } @@ -89,9 +85,9 @@ function create-master-instance-internal() { --scopes "storage-ro,compute-rw,monitoring,logging-write" \ --can-ip-forward \ --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ - --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ + --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \ ${preemptible_master} } diff --git a/cluster/gce/coreos/master.yaml b/cluster/gce/coreos/master.yaml index 78b39164ae4..4dec695c9d7 100644 --- a/cluster/gce/coreos/master.yaml +++ b/cluster/gce/coreos/master.yaml @@ -1,124 +1,56 @@ #cloud-config -write_files: - - path: /etc/systemd/system/kube-master-installation.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Download and install k8s binaries and configurations - After=network-online.target +coreos: + update: + reboot-strategy: off + units: + - name: locksmithd.service + mask: true + - name: kube-master-installation.service + command: start + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/kubernetes/bin - ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin - ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh - ExecStart=/home/kubernetes/bin/configure.sh + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh + ExecStart=/opt/kubernetes/bin/configure.sh - [Install] - WantedBy=kubernetes.target + [Install] + WantedBy=kubernetes.target + - name: kube-master-configuration.service + command: start + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service - - path: /etc/systemd/system/kube-master-configuration.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Configure kubernetes master - After=kube-master-installation.service + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh + ExecStart=/opt/kubernetes/bin/configure-helper.sh - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter - ExecStart=/home/kubernetes/bin/configure-helper.sh + [Install] + WantedBy=kubernetes.target + - name: kubernetes.target + enable: true + command: start + content: | + [Unit] + Description=Kubernetes - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-docker-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for docker - After=kube-master-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh docker - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubelet-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for kubelet - After=kube-master-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.timer - permissions: 0644 - owner: root - content: | - [Unit] - Description=Hourly kube-logrotate invocation - - [Timer] - OnCalendar=hourly - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes log rotation - After=kube-master-configuration.service - - [Service] - Type=oneshot - ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubernetes.target - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes - -runcmd: - - systemctl daemon-reload - - systemctl enable kube-master-installation.service - - systemctl enable kube-master-configuration.service - - systemctl enable kube-docker-monitor.service - - systemctl enable kubelet-monitor.service - - systemctl enable kube-logrotate.timer - - systemctl enable kube-logrotate.service - - systemctl start kubernetes.target + [Install] + WantedBy=multi-user.target + - name: docker.service + drop-ins: + - name: "use-cgroupfs-driver.conf" + # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl + content: | + [Service] + Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver=" diff --git a/cluster/gce/coreos/node-helper.sh b/cluster/gce/coreos/node-helper.sh index 3217b4e09da..347a41524d5 100755 --- a/cluster/gce/coreos/node-helper.sh +++ b/cluster/gce/coreos/node-helper.sh @@ -14,19 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A library of helper functions and constant for GCI distro -source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" +# A library of helper functions and constant for the CoreOS distro +source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh" # $1: template name (required). function create-node-instance-template { local template_name="$1" - ensure-gci-metadata-files + create-node-template "$template_name" "${scope_flags[*]}" \ "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ - "user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml" \ - "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \ - "cluster-name=${KUBE_TEMP}/cluster-name.txt" \ - "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \ - "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \ - "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" + "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \ + "configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh" \ + "cluster-name=${KUBE_TEMP}/cluster-name.txt" + # TODO(euank): We should include update-strategy here. We should also switch to ignition } diff --git a/cluster/gce/coreos/node.yaml b/cluster/gce/coreos/node.yaml index d0cc12e752d..b203c4fded3 100644 --- a/cluster/gce/coreos/node.yaml +++ b/cluster/gce/coreos/node.yaml @@ -1,124 +1,56 @@ #cloud-config -write_files: - - path: /etc/systemd/system/kube-node-installation.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Download and install k8s binaries and configurations - After=network-online.target +coreos: + update: + reboot-strategy: off + units: + - name: locksmithd.service + mask: true + - name: kube-node-installation.service + command: start + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/kubernetes/bin - ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin - ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh - ExecStart=/home/kubernetes/bin/configure.sh + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh + ExecStart=/opt/kubernetes/bin/configure.sh - [Install] - WantedBy=kubernetes.target + [Install] + WantedBy=kubernetes.target + - name: kube-node-configuration.service + command: start + content: | + [Unit] + Description=Configure kubernetes master + After=kube-node-installation.service - - path: /etc/systemd/system/kube-node-configuration.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Configure kubernetes node - After=kube-node-installation.service + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh + ExecStart=/opt/kubernetes/bin/configure-helper.sh - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter - ExecStart=/home/kubernetes/bin/configure-helper.sh + [Install] + WantedBy=kubernetes.target + - name: kubernetes.target + enable: true + command: start + content: | + [Unit] + Description=Kubernetes - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-docker-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for docker - After=kube-node-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh docker - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubelet-monitor.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes health monitoring for kubelet - After=kube-node-configuration.service - - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh - ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.timer - permissions: 0644 - owner: root - content: | - [Unit] - Description=Hourly kube-logrotate invocation - - [Timer] - OnCalendar=hourly - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kube-logrotate.service - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes log rotation - After=kube-node-configuration.service - - [Service] - Type=oneshot - ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf - - [Install] - WantedBy=kubernetes.target - - - path: /etc/systemd/system/kubernetes.target - permissions: 0644 - owner: root - content: | - [Unit] - Description=Kubernetes - -runcmd: - - systemctl daemon-reload - - systemctl enable kube-node-installation.service - - systemctl enable kube-node-configuration.service - - systemctl enable kube-docker-monitor.service - - systemctl enable kubelet-monitor.service - - systemctl enable kube-logrotate.timer - - systemctl enable kube-logrotate.service - - systemctl start kubernetes.target + [Install] + WantedBy=multi-user.target + - name: docker.service + drop-ins: + - name: "use-cgroupfs-driver.conf" + # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl + content: | + [Service] + Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="