cluster/coreos: update to gci based implementation

This update includes significant refactoring. It moves almost all of the
logic into bash scripts, modeled after the `gci` cluster scripts.

The primary differences between the two are the following:
1. Use of the `/opt/kubernetes` directory over `/home/kubernetes`
2. Support for rkt as a runtime
3. No use of logrotate
4. No use of `/etc/default/`
5. No logic related to noexec mounts or gci-specific firewall-stuff

build/lib/release.sh

@@ -331,14 +331,12 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
   cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
+  cp "${KUBE_ROOT}/cluster/gce/coreos/configure-helper.sh" "${dst_dir}/coreos-configure-helper.sh"
   cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
   local objects
   objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
   tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
 
-  # This is for coreos only. ContainerVM, GCI, or Trusty does not use it.
-  cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
-
   kube::release::clean_cruft
 
   local package_name="${RELEASE_DIR}/kubernetes-manifests.tar.gz"

cluster/addons/dashboard/dashboard-controller.yaml

@@ -1,4 +1,3 @@
-# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:

cluster/addons/dashboard/dashboard-service.yaml

@@ -1,4 +1,3 @@
-# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml
 apiVersion: v1
 kind: Service
 metadata:

cluster/gce/coreos/README.md

@@ -1,11 +1,8 @@
-# Container-VM Image
+# CoreOS image
 
-[Container-VM Image](https://cloud.google.com/compute/docs/containers/vm-image/)
+The [CoreOS operating system](https://coreos.com/why/) is a Linux distribution optimized for running containers securely at scale.
-is a container-optimized OS image for the Google Cloud Platform (GCP). It is
+CoreOS provides [an image](https://coreos.com/os/docs/latest/booting-on-google-compute-engine.html) for Google Cloud Platform (GCP).
-primarily for running Google services on GCP. Unlike the open preview version
-of container-vm, the new Container-VM Image is based on the open source
-ChromiumOS project, allowing us greater control over the build management,
-security compliance, and customizations for GCP.
 
+This folder contains configuration and tooling to allow kube-up to create a Kubernetes cluster on Google Cloud Platform running on the official CoreOS image.
 
-[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/gci/README.md?pixel)]()
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/coreos/README.md?pixel)]()

cluster/gce/coreos/configure-helper.sh

@@ -19,37 +19,12 @@
 
 # TODO: this script duplicates templating logic from cluster/saltbase/salt
 # using sed. It should use an actual template parser on the manifest
-# files.
+# files, or the manifest files should not be templated salt
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
-function setup-os-params {
-  # Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to
-  # /sbin/crash_reporter which is more restrictive in saving crash dumps. So for
-  # now, set a generic core_pattern that users can work with.
-  echo "core.%e.%p.%t" > /proc/sys/kernel/core_pattern
-}
-
-function config-ip-firewall {
-  echo "Configuring IP firewall rules"
-  # The GCI image has host firewall which drop most inbound/forwarded packets.
-  # We need to add rules to accept all TCP/UDP/ICMP packets.
-  if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
-    echo "Add rules to accept all inbound TCP/UDP/ICMP packets"
-    iptables -A INPUT -w -p TCP -j ACCEPT
-    iptables -A INPUT -w -p UDP -j ACCEPT
-    iptables -A INPUT -w -p ICMP -j ACCEPT
-  fi
-  if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
-    echo "Add rules to accept all forwarded TCP/UDP/ICMP packets"
-    iptables -A FORWARD -w -p TCP -j ACCEPT
-    iptables -A FORWARD -w -p UDP -j ACCEPT
-    iptables -A FORWARD -w -p ICMP -j ACCEPT
-  fi
-}
-
 function create-dirs {
   echo "Creating required directories"
   mkdir -p /var/lib/kubelet
@@ -59,6 +34,13 @@ function create-dirs {
   fi
 }
 
+# Create directories referenced in the kube-controller-manager manifest for
+# bindmounts. This is used under the rkt runtime to work around
+# https://github.com/kubernetes/kubernetes/issues/26816
+function create-kube-controller-manager-dirs {
+  mkdir -p /etc/srv/kubernetes /var/ssl /etc/{ssl,openssl,pki}
+}
+
 # Formats the given device ($1) if needed and mounts it at given mount point
 # ($2).
 function safe-format-and-mount() {
@@ -92,51 +74,6 @@ function ensure-local-ssds() {
   done
 }
 
-# Installs logrotate configuration files
-function setup-logrotate() {
-  mkdir -p /etc/logrotate.d/
-  cat >/etc/logrotate.d/docker-containers <<EOF
-/var/lib/docker/containers/*/*-json.log {
-    rotate 5
-    copytruncate
-    missingok
-    notifempty
-    compress
-    maxsize 10M
-    daily
-    dateext
-    dateformat -%Y%m%d-%s
-    create 0644 root root
-}
-EOF
-
-  # Configure log rotation for all logs in /var/log, which is where k8s services
-  # are configured to write their log files. Whenever logrotate is ran, this
-  # config will:
-  # * rotate the log file if its size is > 100Mb OR if one day has elapsed
-  # * save rotated logs into a gzipped timestamped backup
-  # * log file timestamp (controlled by 'dateformat') includes seconds too. This
-  #   ensures that logrotate can generate unique logfiles during each rotation
-  #   (otherwise it skips rotation if 'maxsize' is reached multiple times in a
-  #   day).
-  # * keep only 5 old (rotated) logs, and will discard older logs.
-  cat > /etc/logrotate.d/allvarlogs <<EOF
-/var/log/*.log {
-    rotate 5
-    copytruncate
-    missingok
-    notifempty
-    compress
-    maxsize 100M
-    daily
-    dateext
-    dateformat -%Y%m%d-%s
-    create 0644 root root
-}
-EOF
-
-}
-
 # Finds the master PD device; returns it in MASTER_PD_DEVICE
 function find-master-pd {
   MASTER_PD_DEVICE=""
@@ -387,8 +324,8 @@ function create-master-etcd-auth {
   fi
 }
 
-function assemble-docker-flags {
+function configure-docker-daemon {
-  echo "Assemble docker command line flags"
+  echo "Configuring the Docker daemon"
   local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
   if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
     docker_opts+=" --log-level=debug"
@@ -411,28 +348,17 @@ function assemble-docker-flags {
     docker_opts+=" --registry-mirror=${DOCKER_REGISTRY_MIRROR_URL}"
   fi
 
-  echo "DOCKER_OPTS=\"${docker_opts} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
+  mkdir -p /etc/systemd/system/docker.service.d/
-
+  local kubernetes_conf_dropin="/etc/systemd/system/docker.service.d/00_kubelet.conf"
-  if [[ "${use_net_plugin}" == "true" ]]; then
+  cat > "${kubernetes_conf_dropin}" <<EOF
-    # If using a network plugin, extend the docker configuration to always remove
-    # the network checkpoint to avoid corrupt checkpoints.
-    # (https://github.com/docker/docker/issues/18283).
-    echo "Extend the default docker.service configuration"
-    mkdir -p /etc/systemd/system/docker.service.d
-    cat <<EOF >/etc/systemd/system/docker.service.d/01network.conf
 [Service]
-ExecStartPre=/bin/sh -x -c "rm -rf /var/lib/docker/network"
+Environment="DOCKER_OPTS=${docker_opts} ${EXTRA_DOCKER_OPTS:-}"
 EOF
-
+  # Always restart to get the cbr0 change
-    systemctl daemon-reload
+  echo "Docker daemon options updated. Restarting docker..."
-
+  systemctl daemon-reload
-    # If using a network plugin, we need to explicitly restart docker daemon, because
+  systemctl restart docker
-    # kubelet will not do it.
-    echo "Docker command line is updated. Restart docker to pick it up"
-    systemctl restart docker
-  fi
 }
-
 # A helper function for loading a docker image. It keeps trying up to 5 times.
 #
 # $1: Full path of the docker image
@@ -443,15 +369,37 @@ function try-load-docker-image {
   set +e
   local -r max_attempts=5
   local -i attempt_num=1
-  until timeout 30 docker load -i "${img}"; do
+
-    if [[ "${attempt_num}" == "${max_attempts}" ]]; then
+  if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
-      echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!"
+    for attempt_num in $(seq 1 "${max_attempts}"); do
-      exit 1
+      local aci_tmpdir="$(mktemp -t -d docker2aci.XXXXX)"
-    else
+      (cd "${aci_tmpdir}"; timeout 40 "${DOCKER2ACI_BIN}" "$1")
-      attempt_num=$((attempt_num+1))
+      local aci_success=$?
+      timeout 40 "${RKT_BIN}" fetch --insecure-options=image "${aci_tmpdir}"/*.aci
+      local fetch_success=$?
+      rm -f "${aci_tmpdir}"/*.aci
+      rmdir "${aci_tmpdir}"
+      if [[ ${fetch_success} && ${aci_success} ]]; then
+        echo "rkt: Loaded ${img}"
+        break
+      fi
+      if [[ "${attempt}" == "${max_attempts}" ]]; then
+        echo "rkt: Failed to load image file ${img} after ${max_attempts} retries."
+        exit 1
+      fi
       sleep 5
-    fi
+    done
-  done
+  else
+    until timeout 30 docker load -i "${img}"; do
+      if [[ "${attempt_num}" == "${max_attempts}" ]]; then
+        echo "Fail to load docker image file ${img} after ${max_attempts} retries."
+        exit 1
+      else
+        attempt_num=$((attempt_num+1))
+        sleep 5
+      fi
+    done
+  fi
   # Re-enable errexit.
   set -e
 }
@@ -476,19 +424,6 @@ function start-kubelet {
   echo "Start kubelet"
   local kubelet_bin="${KUBE_HOME}/bin/kubelet"
   local -r version="$("${kubelet_bin}" --version=true | cut -f2 -d " ")"
-  local -r builtin_kubelet="/usr/bin/kubelet"
-  if [[ "${TEST_CLUSTER:-}" == "true" ]]; then
-    # Determine which binary to use on test clusters. We use the built-in
-    # version only if the downloaded version is the same as the built-in
-    # version. This allows GCI to run some of the e2e tests to qualify the
-    # built-in kubelet.
-    if [[ -x "${builtin_kubelet}" ]]; then
-      local -r builtin_version="$("${builtin_kubelet}"  --version=true | cut -f2 -d " ")"
-      if [[ "${builtin_version}" == "${version}" ]]; then
-        kubelet_bin="${builtin_kubelet}"
-      fi
-    fi
-  fi
   echo "Using kubelet binary at ${kubelet_bin}"
   local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}"
   flags+=" --allow-privileged=true"
@@ -498,7 +433,6 @@ function start-kubelet {
   flags+=" --cluster-dns=${DNS_SERVER_IP}"
   flags+=" --cluster-domain=${DNS_DOMAIN}"
   flags+=" --config=/etc/kubernetes/manifests"
-  flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter"
   flags+=" --experimental-check-node-capabilities-before-mount=true"
 
   if [[ -n "${KUBELET_PORT:-}" ]]; then
@@ -527,9 +461,9 @@ function start-kubelet {
   # Network plugin
   if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
     if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
-      flags+=" --cni-bin-dir=/home/kubernetes/bin"
+      flags+=" --cni-bin-dir=/opt/kubernetes/bin"
     else
-      flags+=" --network-plugin-dir=/home/kubernetes/bin"
+      flags+=" --network-plugin-dir=/opt/kubernetes/bin"
     fi
     flags+=" --network-plugin=${NETWORK_PROVIDER}"
   fi
@@ -552,8 +486,13 @@ function start-kubelet {
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     flags+=" --feature-gates=${FEATURE_GATES}"
   fi
+  if [[ -n "${CONTAINER_RUNTIME:-}" ]]; then
+    flags+=" --container-runtime=${CONTAINER_RUNTIME}"
+    flags+=" --rkt-path=${KUBE_HOME}/bin/rkt"
+    flags+=" --rkt-stage1-image=${RKT_STAGE1_IMAGE}"
+  fi
 
-  local -r kubelet_env_file="/etc/default/kubelet"
+  local -r kubelet_env_file="/etc/kubelet-env"
   echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
 
   # Write the systemd service file for kubelet.
@@ -600,7 +539,7 @@ function start-kube-proxy {
   if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
     kube_docker_registry=${KUBE_DOCKER_REGISTRY}
   fi
-  local -r kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag)
+  local -r kube_proxy_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-proxy.docker_tag)
   local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
   local params="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=2"}"
   if [[ -n "${FEATURE_GATES:-}" ]]; then
@@ -618,6 +557,18 @@ function start-kube-proxy {
   if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
     sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file}
   fi
+  if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
+    # Work arounds for https://github.com/coreos/rkt/issues/3245 and https://github.com/coreos/rkt/issues/3264
+    # This is an incredibly hacky workaround. It's fragile too. If the kube-proxy command changes too much, this breaks
+    # TODO, this could be done much better in many other places, such as an
+    # init script within the container, or even within kube-proxy's code.
+    local extra_workaround_cmd="ln -sf /proc/self/mounts /etc/mtab; \
+      mount -o remount,rw /proc; \
+      mount -o remount,rw /proc/sys; \
+      mount -o remount,rw /sys; "
+    sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}"
+  fi
+
   cp "${src_file}" /etc/kubernetes/manifests
 }
 
@@ -629,7 +580,7 @@ function start-kube-proxy {
 # $4: value for variable 'cpulimit'
 # $5: pod name, which should be either etcd or etcd-events
 function prepare-etcd-manifest {
-  local host_name=$(hostname)
+  local host_name=$(hostname -s)
   local etcd_cluster=""
   local cluster_state="new"
   local etcd_protocol="http"
@@ -671,6 +622,7 @@ function prepare-etcd-manifest {
   else
     sed -i -e "s@{{ *pillar\.get('etcd_docker_tag', '\(.*\)') *}}@\1@g" "${temp_file}"
   fi
+
   sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
   sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
   if [[ -n "${ETCD_VERSION:-}" ]]; then
@@ -862,7 +814,7 @@ function start-kube-apiserver {
   src_file="${src_dir}/kube-apiserver.manifest"
   remove-salt-config-comments "${src_file}"
   # Evaluate variables.
-  local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
+  local -r kube_apiserver_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
   sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
   sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
@@ -927,7 +879,7 @@ function start-kube-controller-manager {
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     params+=" --feature-gates=${FEATURE_GATES}"
   fi
-  local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
+  local -r kube_rc_docker_tag=$(cat /opt/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
 
   local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
   remove-salt-config-comments "${src_file}"
@@ -1182,63 +1134,74 @@ function start-rescheduler {
   fi
 }
 
-# Setup working directory for kubelet.
+# Install and setup rkt
-function setup-kubelet-dir {
+# TODO(euank): There should be a toggle to use the distro-provided rkt binary
-    echo "Making /var/lib/kubelet executable for kubelet"
+# Sets the following variables:
-    mount -B /var/lib/kubelet /var/lib/kubelet/
+#   RKT_BIN: the path to the rkt binary
-    mount -B -o remount,exec,suid,dev /var/lib/kubelet
+function setup-rkt {
-}
+    local rkt_bin="${KUBE_HOME}/bin/rkt"
+    if [[ -x "${rkt_bin}" ]]; then
+      # idempotency, skip downloading this time
+      # TODO(euank): this might get in the way of updates, but 'file busy'
+      # because of rkt-api would too
+      RKT_BIN="${rkt_bin}"
+      return
+    fi
+    mkdir -p /etc/rkt "${KUBE_HOME}/download/"
+    local rkt_tar="${KUBE_HOME}/download/rkt.tar.gz"
+    local rkt_tmpdir=$(mktemp -d "${KUBE_HOME}/rkt_download.XXXXX")
+    curl --retry 5 --retry-delay 3 --fail --silent --show-error \
+      --location --create-dirs --output "${rkt_tar}" \
+      https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
+    tar --strip-components=1 -xf "${rkt_tar}" -C "${rkt_tmpdir}" --overwrite
+    mv "${rkt_tmpdir}/rkt" "${rkt_bin}"
+    if [[ ! -x "${rkt_bin}" ]]; then
+      echo "Could not download requested rkt binary"
+      exit 1
+    fi
+    RKT_BIN="${rkt_bin}"
+    # Cache rkt stage1 images for speed
+    "${RKT_BIN}" fetch --insecure-options=image "${rkt_tmpdir}"/*.aci
+    rm -rf "${rkt_tmpdir}"
 
-function reset-motd {
+    cat > /etc/systemd/system/rkt-api.service <<EOF
-  # kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
+[Unit]
-  local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
+Description=rkt api service
-  # This logic grabs either a release tag (v1.2.1 or v1.2.1-alpha.1),
+Documentation=http://github.com/coreos/rkt
-  # or the git hash that's in the build info.
+After=network.target
-  local gitref="$(echo "${version}" | sed -r "s/(v[0-9]+\.[0-9]+\.[0-9]+)(-[a-z]+\.[0-9]+)?.*/\1\2/g")"
-  local devel=""
-  if [[ "${gitref}" != "${version}" ]]; then
-    devel="
-Note: This looks like a development version, which might not be present on GitHub.
-If it isn't, the closest tag is at:
-  https://github.com/kubernetes/kubernetes/tree/${gitref}
-"
-    gitref="${version//*+/}"
-  fi
-  cat > /etc/motd <<EOF
 
-Welcome to Kubernetes ${version}!
+[Service]
-
+ExecStart=${RKT_BIN} api-service --listen=127.0.0.1:15441
-You can find documentation for Kubernetes at:
-  http://docs.kubernetes.io/
-
-The source for this release can be found at:
-  /home/kubernetes/kubernetes-src.tar.gz
-Or you can download it at:
-  https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-src.tar.gz
-
-It is based on the Kubernetes source at:
-  https://github.com/kubernetes/kubernetes/tree/${gitref}
-${devel}
-For Kubernetes copyright and licensing information, see:
-  /home/kubernetes/LICENSES
 
+[Install]
+WantedBy=multi-user.target
 EOF
+    systemctl enable rkt-api.service
+    systemctl start rkt-api.service
 }
 
-function override-kubectl {
+# Install docker2aci, needed to load server images if using rkt runtime
-    echo "overriding kubectl"
+# This should be removed once rkt can fetch on-disk docker tarballs directly
-    echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
+# Sets the following variables:
-}
+#   DOCKER2ACI_BIN: the path to the docker2aci binary
-
+function install-docker2aci {
-function pre-warm-mounter {
+  local tar_path="${KUBE_HOME}/download/docker2aci.tar.gz"
-    echo "prewarming mounter"
+  local tmp_path="${KUBE_HOME}/docker2aci"
-    ${KUBE_HOME}/bin/mounter &> /dev/null
+  mkdir -p "${KUBE_HOME}/download/" "${tmp_path}"
+  curl --retry 5 --retry-delay 3 --fail --silent --show-error \
+    --location --create-dirs --output "${tar_path}" \
+    https://github.com/appc/docker2aci/releases/download/v0.14.0/docker2aci-v0.14.0.tar.gz
+  tar --strip-components=1 -xf "${tar_path}" -C "${tmp_path}" --overwrite
+  DOCKER2ACI_BIN="${KUBE_HOME}/bin/docker2aci"
+  mv "${tmp_path}/docker2aci" "${DOCKER2ACI_BIN}"
 }
 
 ########### Main Function ###########
 echo "Start to configure instance for kubernetes"
 
-KUBE_HOME="/home/kubernetes"
+# Note: this name doesn't make as much sense here as in gci where it's actually
+# /home/kubernetes, but for ease of diff-ing, retain the same variable name
+KUBE_HOME="/opt/kubernetes"
 if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
   echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
   exit 1
@@ -1253,12 +1216,13 @@ if [[ -n "${KUBE_USER:-}" ]]; then
   fi
 fi
 
-setup-os-params
+# KUBERNETES_CONTAINER_RUNTIME is set by the `kube-env` file, but it's a bit of a mouthful
-config-ip-firewall
+if [[ "${CONTAINER_RUNTIME:-}" == "" ]]; then
+  CONTAINER_RUNTIME="${KUBERNETES_CONTAINER_RUNTIME:-docker}"
+fi
+
 create-dirs
-setup-kubelet-dir
 ensure-local-ssds
-setup-logrotate
 if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
   mount-master-pd
   create-master-auth
@@ -1269,10 +1233,16 @@ else
   create-kubeproxy-kubeconfig
 fi
 
-override-kubectl
+if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
-# Run the containerized mounter once to pre-cache the container image.
+  systemctl stop docker
-pre-warm-mounter
+  systemctl disable docker
-assemble-docker-flags
+  setup-rkt
+  install-docker2aci
+  create-kube-controller-manager-dirs
+else
+  configure-docker-daemon
+fi
+
 load-docker-images
 start-kubelet
 
@@ -1298,5 +1268,5 @@ else
     start-image-puller
   fi
 fi
-reset-motd
+start-fluentd
 echo "Done for the configuration for kubernetes"

cluster/gce/coreos/configure.sh

@@ -14,30 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Due to the GCE custom metadata size limit, we split the entire script into two
-# files configure.sh and configure-helper.sh. The functionality of downloading
-# kubernetes configuration, manifests, docker images, and binary files are
-# put in configure.sh, which is uploaded via GCE custom metadata.
-
 set -o errexit
 set -o nounset
 set -o pipefail
 
-function set-broken-motd {
-  cat > /etc/motd <<EOF
-Broken (or in progress) Kubernetes node setup! Check the cluster initialization status
-using the following commands.
-
-Master instance:
-  - sudo systemctl status kube-master-installation
-  - sudo systemctl status kube-master-configuration
-
-Node instance:
-  - sudo systemctl status kube-node-installation
-  - sudo systemctl status kube-node-configuration
-EOF
-}
-
 function download-kube-env {
   # Fetch kube-env from GCE metadata server.
   local -r tmp_kube_env="/tmp/kube-env.yaml"
@@ -46,11 +26,7 @@ function download-kube-env {
     -o "${tmp_kube_env}" \
     http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
   # Convert the yaml format file into a shell-style file.
-  eval $(python -c '''
+  sed 's/: /=/' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env"
-import pipes,sys,yaml
-for k,v in yaml.load(sys.stdin).iteritems():
-  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
-''' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env")
   rm -f "${tmp_kube_env}"
 }
 
@@ -65,6 +41,7 @@ function validate-hash {
   fi
 }
 
+
 # Retry a download until we get it. Takes a hash and a set of URLs.
 #
 # $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
@@ -98,24 +75,8 @@ function split-commas {
   echo $1 | tr "," "\n"
 }
 
-function install-gci-mounter-tools {
-    local -r rkt_version="v1.18.0"
-    local -r gci_mounter_version="v2"
-    local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
-    local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091"
-    local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561"
-    download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt"
-    download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci"
-    download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci"
-    local -r rkt_dst="${KUBE_HOME}/bin/"
-    mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt"
-    mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci"
-    mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci"
-    chmod a+x "${rkt_dst}/rkt"
-}
-
 # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
-# and places them into suitable directories. Files are placed in /home/kubernetes.
+# and places them into suitable directories. Files are placed in /opt/kubernetes.
 function install-kube-binary-config {
   cd "${KUBE_HOME}"
   local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") )
@@ -186,14 +147,9 @@ function install-kube-binary-config {
     find "${dst_dir}" -name \*.manifest -or -name \*.json | \
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
-  cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
+  cp "${dst_dir}/kubernetes/gci-trusty/coreos-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
-  cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
-  cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
   chmod -R 755 "${kube_bin}"
 
-  # Install gci mounter related artifacts to allow mounting storage volumes in GCI
-  install-gci-mounter-tools
-  
   # Clean up.
   rm -rf "${KUBE_HOME}/kubernetes"
   rm -f "${KUBE_HOME}/${server_binary_tar}"
@@ -204,10 +160,17 @@ function install-kube-binary-config {
 
 ######### Main Function ##########
 echo "Start to install kubernetes files"
-set-broken-motd
+KUBE_HOME="/opt/kubernetes"
-KUBE_HOME="/home/kubernetes"
+mkdir -p "${KUBE_HOME}"
 download-kube-env
 source "${KUBE_HOME}/kube-env"
 install-kube-binary-config
 echo "Done for installing kubernetes files"
 
+# On CoreOS, the hosts is in /usr/share/baselayout/hosts
+# So we need to manually populdate the hosts file here on gce.
+echo "127.0.0.1 localhost" >> /etc/hosts
+echo "::1 localhost" >> /etc/hosts
+
+echo "Configuring hostname"
+hostnamectl set-hostname $(hostname | cut -f1 -d.)

cluster/gce/coreos/helper.sh

@@ -14,19 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# A library of helper functions and constant for GCI distro
+# A library of helper functions and constants for the CoreOS distro
 
-# Creates the GCI specific metadata files if they do not exit.
+# This file intentionally left blank
-# Assumed var
-#   KUBE_TEMP
-function ensure-gci-metadata-files {
-  if [[ ! -f "${KUBE_TEMP}/gci-update.txt" ]]; then
-    echo -n "update_disabled" > "${KUBE_TEMP}/gci-update.txt"
-  fi
-  if [[ ! -f "${KUBE_TEMP}/gci-ensure-gke-docker.txt" ]]; then
-    echo -n "true" > "${KUBE_TEMP}/gci-ensure-gke-docker.txt"
-  fi
-  if [[ ! -f "${KUBE_TEMP}/gci-docker-version.txt" ]]; then
-    echo -n "${GCI_DOCKER_VERSION:-}" > "${KUBE_TEMP}/gci-docker-version.txt"
-  fi
-}

cluster/gce/coreos/master-helper.sh

@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# A library of helper functions and constant for GCI distro
+# A library of helper functions and constant for coreos os distro
-source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
+source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh"
 
 # create-master-instance creates the master instance. If called with
 # an argument, the argument is used as the name to a reserved IP
@@ -35,7 +35,6 @@ function create-master-instance {
   [[ -n ${1:-} ]] && address_opt="--address ${1}"
 
   write-master-env
-  ensure-gci-metadata-files
   create-master-instance-internal "${MASTER_NAME}" "${address_opt}"
 }
 
@@ -60,9 +59,6 @@ function replicate-master-instance() {
 
   echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml
   get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt"
-  get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt"
-  get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt"
-  get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt"
 
   create-master-instance-internal "${REPLICA_NAME}"
 }
@@ -89,9 +85,9 @@ function create-master-instance-internal() {
     --scopes "storage-ro,compute-rw,monitoring,logging-write" \
     --can-ip-forward \
     --metadata-from-file \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" \
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
     --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
+    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \
     ${preemptible_master}
 }
 

cluster/gce/coreos/master.yaml

@@ -1,124 +1,56 @@
 #cloud-config
 
-write_files:
+coreos:
-  - path: /etc/systemd/system/kube-master-installation.service
+  update:
-    permissions: 0644
+    reboot-strategy: off
-    owner: root
+  units:
-    content: |
+    - name: locksmithd.service
-      [Unit]
+      mask: true
-      Description=Download and install k8s binaries and configurations
+    - name: kube-master-installation.service
-      After=network-online.target
+      command: start
+      content: |
+        [Unit]
+        Description=Download and install k8s binaries and configurations
+        After=network-online.target
 
-      [Service]
+        [Service]
-      Type=oneshot
+        Type=oneshot
-      RemainAfterExit=yes
+        RemainAfterExit=yes
-      ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
+        ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin
-      ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
+        ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
-      ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh
-      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+        ExecStart=/opt/kubernetes/bin/configure.sh
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
-      ExecStart=/home/kubernetes/bin/configure.sh
 
-      [Install]
+        [Install]
-      WantedBy=kubernetes.target
+        WantedBy=kubernetes.target
+    - name: kube-master-configuration.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure kubernetes master
+        After=kube-master-installation.service
 
-  - path: /etc/systemd/system/kube-master-configuration.service
+        [Service]
-    permissions: 0644
+        Type=oneshot
-    owner: root
+        RemainAfterExit=yes
-    content: |
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh
-      [Unit]
+        ExecStart=/opt/kubernetes/bin/configure-helper.sh
-      Description=Configure kubernetes master
-      After=kube-master-installation.service
 
-      [Service]
+        [Install]
-      Type=oneshot
+        WantedBy=kubernetes.target
-      RemainAfterExit=yes
+    - name: kubernetes.target
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      enable: true
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
+      command: start
-      ExecStart=/home/kubernetes/bin/configure-helper.sh
+      content: |
+        [Unit]
+        Description=Kubernetes
 
-      [Install]
+        [Install]
-      WantedBy=kubernetes.target
+        WantedBy=multi-user.target
-
+    - name: docker.service
-  - path: /etc/systemd/system/kube-docker-monitor.service
+      drop-ins:
-    permissions: 0644
+        - name: "use-cgroupfs-driver.conf"
-    owner: root
+          # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl
-    content: |
+          content: |
-      [Unit]
+            [Service]
-      Description=Kubernetes health monitoring for docker
+            Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="
-      After=kube-master-configuration.service
-
-      [Service]
-      Restart=always
-      RestartSec=10
-      RemainAfterExit=yes
-      RemainAfterExit=yes
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
-      ExecStart=/home/kubernetes/bin/health-monitor.sh docker
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kubelet-monitor.service
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Kubernetes health monitoring for kubelet
-      After=kube-master-configuration.service
-
-      [Service]
-      Restart=always
-      RestartSec=10
-      RemainAfterExit=yes
-      RemainAfterExit=yes
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
-      ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kube-logrotate.timer
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Hourly kube-logrotate invocation
-
-      [Timer]
-      OnCalendar=hourly
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kube-logrotate.service
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Kubernetes log rotation
-      After=kube-master-configuration.service
-
-      [Service]
-      Type=oneshot
-      ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kubernetes.target
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Kubernetes
-
-runcmd:
- - systemctl daemon-reload
- - systemctl enable kube-master-installation.service
- - systemctl enable kube-master-configuration.service
- - systemctl enable kube-docker-monitor.service
- - systemctl enable kubelet-monitor.service
- - systemctl enable kube-logrotate.timer
- - systemctl enable kube-logrotate.service
- - systemctl start kubernetes.target

cluster/gce/coreos/node-helper.sh

@@ -14,19 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# A library of helper functions and constant for GCI distro
+# A library of helper functions and constant for the CoreOS distro
-source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
+source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh"
 
 # $1: template name (required).
 function create-node-instance-template {
   local template_name="$1"
-  ensure-gci-metadata-files
+
   create-node-template "$template_name" "${scope_flags[*]}" \
     "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml" \
+    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \
-    "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \
+    "configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh" \
-    "cluster-name=${KUBE_TEMP}/cluster-name.txt" \
+    "cluster-name=${KUBE_TEMP}/cluster-name.txt"
-    "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \
+  # TODO(euank): We should include update-strategy here. We should also switch to ignition
-    "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \
-    "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
 }

cluster/gce/coreos/node.yaml

@@ -1,124 +1,56 @@
 #cloud-config
 
-write_files:
+coreos:
-  - path: /etc/systemd/system/kube-node-installation.service
+  update:
-    permissions: 0644
+    reboot-strategy: off
-    owner: root
+  units:
-    content: |
+    - name: locksmithd.service
-      [Unit]
+      mask: true
-      Description=Download and install k8s binaries and configurations
+    - name: kube-node-installation.service
-      After=network-online.target
+      command: start
+      content: |
+        [Unit]
+        Description=Download and install k8s binaries and configurations
+        After=network-online.target
 
-      [Service]
+        [Service]
-      Type=oneshot
+        Type=oneshot
-      RemainAfterExit=yes
+        RemainAfterExit=yes
-      ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
+        ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin
-      ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
+        ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
-      ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh
-      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+        ExecStart=/opt/kubernetes/bin/configure.sh
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
-      ExecStart=/home/kubernetes/bin/configure.sh
 
-      [Install]
+        [Install]
-      WantedBy=kubernetes.target
+        WantedBy=kubernetes.target
+    - name: kube-node-configuration.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure kubernetes master
+        After=kube-node-installation.service
 
-  - path: /etc/systemd/system/kube-node-configuration.service
+        [Service]
-    permissions: 0644
+        Type=oneshot
-    owner: root
+        RemainAfterExit=yes
-    content: |
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh
-      [Unit]
+        ExecStart=/opt/kubernetes/bin/configure-helper.sh
-      Description=Configure kubernetes node
-      After=kube-node-installation.service
 
-      [Service]
+        [Install]
-      Type=oneshot
+        WantedBy=kubernetes.target
-      RemainAfterExit=yes
+    - name: kubernetes.target
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      enable: true
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
+      command: start
-      ExecStart=/home/kubernetes/bin/configure-helper.sh
+      content: |
+        [Unit]
+        Description=Kubernetes
 
-      [Install]
+        [Install]
-      WantedBy=kubernetes.target
+        WantedBy=multi-user.target
-
+    - name: docker.service
-  - path: /etc/systemd/system/kube-docker-monitor.service
+      drop-ins:
-    permissions: 0644
+        - name: "use-cgroupfs-driver.conf"
-    owner: root
+          # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl
-    content: |
+          content: |
-      [Unit]
+            [Service]
-      Description=Kubernetes health monitoring for docker
+            Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="
-      After=kube-node-configuration.service
-
-      [Service]
-      Restart=always
-      RestartSec=10
-      RemainAfterExit=yes
-      RemainAfterExit=yes
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
-      ExecStart=/home/kubernetes/bin/health-monitor.sh docker
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kubelet-monitor.service
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Kubernetes health monitoring for kubelet
-      After=kube-node-configuration.service
-
-      [Service]
-      Restart=always
-      RestartSec=10
-      RemainAfterExit=yes
-      RemainAfterExit=yes
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
-      ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kube-logrotate.timer
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Hourly kube-logrotate invocation
-
-      [Timer]
-      OnCalendar=hourly
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kube-logrotate.service
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Kubernetes log rotation
-      After=kube-node-configuration.service
-
-      [Service]
-      Type=oneshot
-      ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
-
-      [Install]
-      WantedBy=kubernetes.target
-
-  - path: /etc/systemd/system/kubernetes.target
-    permissions: 0644
-    owner: root
-    content: |
-      [Unit]
-      Description=Kubernetes
-
-runcmd:
- - systemctl daemon-reload
- - systemctl enable kube-node-installation.service
- - systemctl enable kube-node-configuration.service
- - systemctl enable kube-docker-monitor.service
- - systemctl enable kubelet-monitor.service
- - systemctl enable kube-logrotate.timer
- - systemctl enable kube-logrotate.service
- - systemctl start kubernetes.target