cluster/gce/coreos: Update the cloud config and scripts to enable rkt runtime.

cluster/gce/coreos/configure-node.sh

@@ -119,7 +119,7 @@ function wait-url-up() {
   done
 }
 
-# Configure addon yamls, and run salt/kube-addons/kube-addon.sh
+# Configure addon yamls, and run salt/kube-addons/kube-addons.sh
 function configure-master-addons() {
   echo "Configuring master addons"
 

cluster/gce/coreos/helper.sh

@@ -23,7 +23,7 @@ function create-node-instance-template() {
   local template_name="$1"
   create-node-template "$template_name" "${scope_flags}" \
     "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \
+    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \
     "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \
     "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \
     "cluster-name=${KUBE_TEMP}/cluster-name.txt"
@@ -65,7 +65,7 @@ function create-master-instance() {
     --scopes "storage-ro,compute-rw,monitoring,logging-write" \
     --can-ip-forward \
     --metadata-from-file \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
     --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
     --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
       ${preemptible_master}

cluster/gce/coreos/kube-manifests/etcd-events.yaml

@@ -14,7 +14,7 @@ spec:
       --bind-addr=127.0.0.1:4002
       --data-dir=/var/etcd/data-events
       1>>/var/log/etcd-events.log 2>&1
-    image: gcr.io/google_containers/etcd:2.0.12
+    image: gcr.io/google_containers/etcd:2.2.1
     imagePullPolicy: IfNotPresent
     livenessProbe:
       httpGet:

cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml

@@ -12,7 +12,9 @@ spec:
     resources:
       requests:
         cpu: 5m
-        memory: 50Mi
+        # TODO(yifan): Figure out what's the memory usage should be here.
+        # See https://github.com/kubernetes/kubernetes/issues/23641.
+        memory: 100Mi
     volumeMounts:
     - mountPath: /etc/kubernetes/
       name: addons

cluster/gce/coreos/master-docker.yaml

@@ -38,24 +38,6 @@ coreos:
         ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
         ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
         
-    - name: kubernetes-install-rkt.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch rkt
-        Documentation=http://github.com/coreos/rkt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/
-        ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
-        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite
-
     - name: kubernetes-download-salt.service
       command: start
       content: |
@@ -139,7 +121,7 @@ coreos:
         --cluster-dns=${DNS_SERVER_IP} \
         --cluster-domain=${DNS_DOMAIN} \
         --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --container-runtime=docker \
         --rkt-path=/opt/rkt/rkt \
         --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
         --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
@@ -149,7 +131,7 @@ coreos:
         Restart=always
         RestartSec=10
         KillMode=process
-        
+
     - name: docker.service
       drop-ins:
         - name: 50-docker-opts.conf
@@ -171,8 +153,6 @@ coreos:
         Description=Configure Node For Kubernetes service
         Requires=kubernetes-install-node.service
         After=kubernetes-install-node.service
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
         Requires=kubernetes-download-salt.service
         After=kubernetes-download-salt.service
         Requires=kubernetes-download-manifests.service

cluster/gce/coreos/master-rkt.yaml

@@ -0,0 +1,209 @@
+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-cni.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download cni
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/cni
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/
+
+    - name: kubernetes-install-docker2aci.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download docker2aci
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz
+        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite
+
+    - name: kubernetes-install-rkt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch rkt
+        Documentation=http://github.com/coreos/rkt
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/
+        ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
+        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite
+
+    - name: kubernetes-download-salt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download salt
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: rkt-api-service.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start rkt API service as Daemon
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        [Service]
+        ExecStart=/opt/rkt/rkt api-service
+        Restart=always
+        RestartSec=10
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=false \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=rkt \
+        --rkt-path=/opt/rkt/rkt \
+        --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --pod-cidr=${MASTER_IP_RANGE} \
+        --register-schedulable=false \
+        --reconcile-cidr=false
+        Restart=always
+        RestartSec=10
+        KillMode=process
+
+    - name: docker.service
+      command: stop
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        Requires=kubernetes-download-salt.service
+        After=kubernetes-download-salt.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        Requires=kubernetes-install-docker2aci.service
+        After=kubernetes-install-docker2aci.service
+        # Need the kubelet/docker running because we will use docker load for docker images.
+        Requires=kubelet.service
+        After=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh

cluster/gce/coreos/node-docker.yaml

@@ -0,0 +1,161 @@
+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-cni.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download cni
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/cni
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=true \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=docker \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --network-plugin=${NETWORK_PROVIDER} \
+        --reconcile-cidr=true
+        Restart=always
+        RestartSec=10
+        KillMode=process
+
+    - name: kube-proxy.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start Kube-proxy service as Daemon
+        Requires=kubernetes-configure-node.service
+        After=kubernetes-configure-node.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStart=/opt/kubernetes/server/bin/kube-proxy \
+        --master=https://${KUBERNETES_MASTER_NAME} \
+        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
+        --v=2 \
+        --logtostderr=true
+        Restart=always
+        RestartSec=10
+
+    - name: docker.service
+      drop-ins:
+        - name: 50-docker-opts.conf
+          content: |
+            [Service]
+            Environment='DOCKER_OPTS= --iptables=false --ip-masq=false'
+            MountFlags=slave
+            LimitNOFILE=1048576
+            LimitNPROC=1048576
+            LimitCORE=infinity
+            Restart=always
+            RestartSec=2s
+            StartLimitInterval=0
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh

cluster/gce/coreos/node-rkt.yaml

@@ -131,10 +131,12 @@ coreos:
         --cluster-dns=${DNS_SERVER_IP} \
         --cluster-domain=${DNS_DOMAIN} \
         --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --container-runtime=rkt \
         --rkt-path=/opt/rkt/rkt \
         --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
-        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0}
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --network-plugin=kubenet \
+        --reconcile-cidr=true
         Restart=always
         RestartSec=10
         KillMode=process
@@ -157,18 +159,7 @@ coreos:
         RestartSec=10
 
     - name: docker.service
-      drop-ins:
-        - name: 50-docker-opts.conf
-          content: |
-            [Service]
-            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
-            MountFlags=slave
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-            LimitCORE=infinity
-            Restart=always
-            RestartSec=2s
-            StartLimitInterval=0
+      command: stop
 
     - name: kubernetes-configure-node.service
       command: start