Merge pull request #27220 from yifan-gu/kube-up-master

Automatic merge from submit-queue cluster/gce/coreos: Make kube-up works for both rkt and docker on coreos on gce With this PR, kube-up will be able to spin up a pure rkt cluster given the choice `KUBE_CONTAINER_RUNTIME=rkt` e.g. ``` export KUBE_GCE_ZONE=us-east1-b export KUBE_OS_DISTRIBUTION=coreos export KUBE_GCE_MASTER_PROJECT=coreos-cloud export KUBE_GCE_MASTER_IMAGE=coreos-alpha-1032-0-0-v20160428 export KUBE_GCE_NODE_PROJECT=coreos-cloud export KUBE_GCE_NODE_IMAGE=coreos-alpha-1032-0-0-v20160428 export KUBE_ENABLE_NODE_LOGGING=false export KUBE_ENABLE_CLUSTER_MONITORING=none export KUBE_CONTAINER_RUNTIME=rkt export KUBE_RKT_VERSION=v1.8.0 ``` ``` $ cluster/kube-up.sh ... $ kubectl cluster-info Kubernetes master is running at https://104.196.41.124 GLBCDefaultBackend is running at https://104.196.41.124/api/v1/proxy/namespaces/kube-system/services/default-http-backend Elasticsearch is running at https://104.196.41.124/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging Kibana is running at https://104.196.41.124/api/v1/proxy/namespaces/kube-system/services/kibana-logging KubeDNS is running at https://104.196.41.124/api/v1/proxy/namespaces/kube-system/services/kube-dns kubernetes-dashboard is running at https://104.196.41.124/api/v1/proxy/namespaces/kube-system/services/kubernetes-dashboard ``` ``` $ kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE kube-system elasticsearch-logging-v1-5zfrd 1/1 Running 0 2m kube-system elasticsearch-logging-v1-83u6w 1/1 Running 0 2m kube-system etcd-server-events-yifan-test-rkt-master 1/1 Running 0 2m kube-system etcd-server-yifan-test-rkt-master 1/1 Running 0 2m kube-system kibana-logging-v1-0g7yu 1/1 Running 2 2m kube-system kube-addon-manager-yifan-test-rkt-master 1/1 Running 0 3m kube-system kube-apiserver-yifan-test-rkt-master 1/1 Running 0 2m kube-system kube-controller-manager-yifan-test-rkt-master 1/1 Running 0 3m kube-system kube-dns-v14-1mqco 3/3 Running 0 2m kube-system kube-scheduler-yifan-test-rkt-master 1/1 Running 0 2m kube-system kubernetes-dashboard-v1.1.0-beta2-uwutn 1/1 Running 0 2m kube-system l7-lb-controller-v0.6.0-8pgbo 2/2 Running 0 2m kube-system node-problem-detector-v0.1-7iwb2 1/1 Running 0 2m kube-system node-problem-detector-v0.1-k4m8o 1/1 Running 0 2m kube-system node-problem-detector-v0.1-rxtp8 1/1 Running 0 2m kube-system node-problem-detector-v0.1-wsoqd 1/1 Running 0 2m ``` Fix #24103 cc @kubernetes/sig-node @kubernetes/rktnetes-maintainers
2025-08-28 13:03:43 +00:00 · 2016-06-17 19:39:22 -07:00 · 2016-06-17 19:39:22 -07:00 · 808f3ecbe6
commit 808f3ecbe6
parent 64f79df54e d96c7edc34
12 changed files with 525 additions and 150 deletions
--- a/cluster/gce/coreos/configure-node.sh
+++ b/cluster/gce/coreos/configure-node.sh
@ -119,7 +119,7 @@ function wait-url-up() {
  done
 }

-# Configure addon yamls, and run salt/kube-addons/kube-addon.sh
+# Configure addon yamls, and run salt/kube-addons/kube-addons.sh
 function configure-master-addons() {
  echo "Configuring master addons"

@ -160,6 +160,10 @@ function configure-master-addons() {
    CLUSTER_REGISTRY_DISK_SIZE=$(convert-bytes-gce-kube "${CLUSTER_REGISTRY_DISK_SIZE}")
    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/registry  ${addon_dir}/registry
  fi
+
+  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR}" == "true" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/node-problem-detector  ${addon_dir}/node-problem-detector
+  fi
 }

 function configure-master-components() {
@ -169,8 +173,8 @@ function configure-master-components() {
  configure-kube-apiserver
  configure-kube-scheduler
  configure-kube-controller-manager
-  configure-addon-manager
  configure-master-addons
+  configure-addon-manager
 }

 # TODO(yifan): Merge this with mount-master-pd() in configure-vm.sh
@ -297,9 +301,10 @@ function load-docker-images() {
 }


-# TODO(yifan): Making this function more generic for other runtimes.
 function load-master-components-images() {
  echo "Loading docker images for master components"
+  export RKT_BIN=/opt/rkt/rkt
+  export DOCKER2ACI_BIN=/opt/docker2aci/docker2aci
  ${SALT_DIR}/install.sh ${KUBE_BIN_TAR}
  ${SALT_DIR}/salt/kube-master-addons/kube-master-addons.sh

--- a/cluster/gce/coreos/helper.sh
+++ b/cluster/gce/coreos/helper.sh
@ -23,7 +23,7 @@ function create-node-instance-template() {
  local template_name="$1"
  create-node-template "$template_name" "${scope_flags}" \
    "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \
+    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \
    "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \
    "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \
    "cluster-name=${KUBE_TEMP}/cluster-name.txt"
@ -65,7 +65,7 @@ function create-master-instance() {
    --scopes "storage-ro,compute-rw,monitoring,logging-write" \
    --can-ip-forward \
    --metadata-from-file \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
      ${preemptible_master}
--- a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml
+++ b/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml
@ -1,27 +1,27 @@
 apiVersion: v1
 kind: ReplicationController
 metadata:
-  name: kube-dns-v11
+  name: kube-dns-v14
  namespace: kube-system
  labels:
    k8s-app: kube-dns
-    version: v11
+    version: v14
    kubernetes.io/cluster-service: "true"
 spec:
  replicas: ${DNS_REPLICAS}
  selector:
    k8s-app: kube-dns
-    version: v11
+    version: v14
  template:
    metadata:
      labels:
        k8s-app: kube-dns
-        version: v11
+        version: v14
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
-      - name: etcd
-        image: gcr.io/google_containers/etcd-amd64:2.2.1
+      - name: kubedns
+        image: gcr.io/google_containers/kubedns-amd64:1.3
        resources:
          # TODO: Set memory limits when we've profiled the container for large
          # clusters, then set request = limit to keep this container in
@ -29,33 +29,6 @@ spec:
          # "burstable" category so the kubelet doesn't backoff from restarting it.
          limits:
            cpu: 100m
-            memory: 500Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        command:
-        - /usr/local/bin/etcd
-        - -data-dir
-        - /var/etcd/data
-        - -listen-client-urls
-        - http://127.0.0.1:2379,http://127.0.0.1:4001
-        - -advertise-client-urls
-        - http://127.0.0.1:2379,http://127.0.0.1:4001
-        - -initial-cluster-token
-        - skydns-etcd
-        volumeMounts:
-        - name: etcd-storage
-          mountPath: /var/etcd/data
-      - name: kube2sky
-        image: gcr.io/google_containers/kube2sky:1.15
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            cpu: 100m
-            # Kube2sky watches all pods.
            memory: 200Mi
          requests:
            cpu: 100m
@ -78,32 +51,23 @@ spec:
          # only setup the /readiness HTTP server once that's available.
          initialDelaySeconds: 30
          timeoutSeconds: 5
-        command:
-        - /kube2sky
        args:
-        # command = "/kube2sky"
-        - --domain=${DNS_DOMAIN}
-      - name: skydns
-        image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            cpu: 100m
-            memory: 200Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        command:
-        - /skydns
+        # command = "/kube-dns"
+        - --domain=${DNS_DOMAIN}.
+        - --dns-port=10053
+        ports:
+        - containerPort: 10053
+          name: dns-local
+          protocol: UDP
+        - containerPort: 10053
+          name: dns-tcp-local
+          protocol: TCP
+      - name: dnsmasq
+        image: gcr.io/google_containers/dnsmasq:1.1
        args:
-        # command = "/skydns"
-        - -machines=http://127.0.0.1:4001
-        - -addr=0.0.0.0:53
-        - -ns-rotate=false
-        - -domain=${DNS_DOMAIN}.
+        - --cache-size=1000
+        - --no-resolv
+        - --server=127.0.0.1#10053
        ports:
        - containerPort: 53
          name: dns
@ -112,7 +76,7 @@ spec:
          name: dns-tcp
          protocol: TCP
      - name: healthz
-        image: gcr.io/google_containers/exechealthz:1.0
+        image: gcr.io/google_containers/exechealthz-amd64:1.0
        resources:
          # keep request = limit to keep this container in guaranteed class
          limits:
@ -121,15 +85,10 @@ spec:
          requests:
            cpu: 10m
            memory: 20Mi
-        command:
-        - /exechealthz
        args:
        - -cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1 >/dev/null
        - -port=8080
        ports:
        - containerPort: 8080
          protocol: TCP
-      volumes:
-      - name: etcd-storage
-        emptyDir: {}
-      dnsPolicy: Default  # Don't use cluster DNS.
+      dnsPolicy: Default  # Don't use cluster DNS.
--- a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml
+++ b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml
@ -20,7 +20,7 @@ spec:
        kubernetes.io/cluster-service: "true"
    spec:
      containers:
-      - image: gcr.io/google_containers/elasticsearch:1.8
+      - image: gcr.io/google_containers/elasticsearch:1.9
        name: elasticsearch-logging         
        resources:
          # keep request = limit to keep this container in guaranteed class
@ -40,4 +40,4 @@ spec:
          mountPath: /data
      volumes:
      - name: es-persistent-storage
-        emptyDir: {}
+        emptyDir: {}
--- a/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml
+++ b/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml
@ -0,0 +1,44 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: node-problem-detector-v0.1
+  namespace: kube-system
+  labels:
+    k8s-app: node-problem-detector
+    version: v0.1
+    kubernetes.io/cluster-service: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: node-problem-detector
+        version: v0.1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      hostNetwork: true
+      containers:
+      - name: node-problem-detector
+        image: gcr.io/google_containers/node-problem-detector:v0.1
+        env:
+        # Config the host ip and port of apiserver.
+        - name: "KUBERNETES_SERVICE_HOST"
+          value: ${INSTANCE_PREFIX}-master
+        - name: "KUBERNETES_SERVICE_PORT"
+          value: "443"
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: "200m"
+            memory: "100Mi"
+          requests:
+            cpu: "20m"
+            memory: "20Mi"
+        volumeMounts:
+        - name: log
+          mountPath: /log
+          readOnly: true
+      volumes:
+      - name: log
+        hostPath:
+          path: /var/log/
--- a/cluster/gce/coreos/kube-manifests/etcd-events.yaml
+++ b/cluster/gce/coreos/kube-manifests/etcd-events.yaml
@ -14,7 +14,7 @@ spec:
      --bind-addr=127.0.0.1:4002
      --data-dir=/var/etcd/data-events
      1>>/var/log/etcd-events.log 2>&1
-    image: gcr.io/google_containers/etcd:2.0.12
+    image: gcr.io/google_containers/etcd:2.2.1
    imagePullPolicy: IfNotPresent
    livenessProbe:
      httpGet:
--- a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml
+++ b/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml
@ -12,7 +12,9 @@ spec:
    resources:
      requests:
        cpu: 5m
-        memory: 50Mi
+        # TODO(yifan): Figure out what's the memory usage should be here.
+        # See https://github.com/kubernetes/kubernetes/issues/23641.
+        memory: 100Mi
    volumeMounts:
    - mountPath: /etc/kubernetes/
      name: addons
--- a/cluster/gce/coreos/master-docker.yaml
+++ b/cluster/gce/coreos/master-docker.yaml
@ -35,27 +35,9 @@ coreos:
        Type=oneshot
        RemainAfterExit=yes
        ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
        
-    - name: kubernetes-install-rkt.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch rkt
-        Documentation=http://github.com/coreos/rkt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/
-        ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
-        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite
-
    - name: kubernetes-download-salt.service
      command: start
      content: |
@ -139,7 +121,7 @@ coreos:
        --cluster-dns=${DNS_SERVER_IP} \
        --cluster-domain=${DNS_DOMAIN} \
        --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --container-runtime=docker \
        --rkt-path=/opt/rkt/rkt \
        --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
@ -149,7 +131,7 @@ coreos:
        Restart=always
        RestartSec=10
        KillMode=process
-        
+
    - name: docker.service
      drop-ins:
        - name: 50-docker-opts.conf
@ -171,8 +153,6 @@ coreos:
        Description=Configure Node For Kubernetes service
        Requires=kubernetes-install-node.service
        After=kubernetes-install-node.service
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
        Requires=kubernetes-download-salt.service
        After=kubernetes-download-salt.service
        Requires=kubernetes-download-manifests.service
--- a/cluster/gce/coreos/master-rkt.yaml
+++ b/cluster/gce/coreos/master-rkt.yaml
@ -0,0 +1,209 @@
+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-cni.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download cni
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/cni
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/
+
+    - name: kubernetes-install-docker2aci.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download docker2aci
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz
+        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite
+
+    - name: kubernetes-install-rkt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch rkt
+        Documentation=http://github.com/coreos/rkt
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/
+        ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
+        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite
+
+    - name: kubernetes-download-salt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download salt
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: rkt-api-service.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start rkt API service as Daemon
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        [Service]
+        ExecStart=/opt/rkt/rkt api-service
+        Restart=always
+        RestartSec=10
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=false \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=rkt \
+        --rkt-path=/opt/rkt/rkt \
+        --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --pod-cidr=${MASTER_IP_RANGE} \
+        --register-schedulable=false \
+        --reconcile-cidr=false
+        Restart=always
+        RestartSec=10
+        KillMode=process
+
+    - name: docker.service
+      command: stop
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        Requires=kubernetes-download-salt.service
+        After=kubernetes-download-salt.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        Requires=kubernetes-install-docker2aci.service
+        After=kubernetes-install-docker2aci.service
+        # Need the kubelet/docker running because we will use docker load for docker images.
+        Requires=kubelet.service
+        After=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh
--- a/cluster/gce/coreos/node-docker.yaml
+++ b/cluster/gce/coreos/node-docker.yaml
@ -0,0 +1,161 @@
+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-cni.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download cni
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/cni
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=true \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=docker \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --network-plugin=${NETWORK_PROVIDER} \
+        --reconcile-cidr=true
+        Restart=always
+        RestartSec=10
+        KillMode=process
+
+    - name: kube-proxy.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start Kube-proxy service as Daemon
+        Requires=kubernetes-configure-node.service
+        After=kubernetes-configure-node.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStart=/opt/kubernetes/server/bin/kube-proxy \
+        --master=https://${KUBERNETES_MASTER_NAME} \
+        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
+        --v=2 \
+        --logtostderr=true
+        Restart=always
+        RestartSec=10
+
+    - name: docker.service
+      drop-ins:
+        - name: 50-docker-opts.conf
+          content: |
+            [Service]
+            Environment='DOCKER_OPTS= --iptables=false --ip-masq=false'
+            MountFlags=slave
+            LimitNOFILE=1048576
+            LimitNPROC=1048576
+            LimitCORE=infinity
+            Restart=always
+            RestartSec=2s
+            StartLimitInterval=0
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh
--- a/cluster/gce/coreos/node-rkt.yaml
+++ b/cluster/gce/coreos/node-rkt.yaml
@ -35,7 +35,7 @@ coreos:
        Type=oneshot
        RemainAfterExit=yes
        ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        

    - name: kubernetes-install-rkt.service
@ -131,10 +131,12 @@ coreos:
        --cluster-dns=${DNS_SERVER_IP} \
        --cluster-domain=${DNS_DOMAIN} \
        --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --container-runtime=rkt \
        --rkt-path=/opt/rkt/rkt \
        --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
-        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0}
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --network-plugin=kubenet \
+        --reconcile-cidr=true
        Restart=always
        RestartSec=10
        KillMode=process
@ -157,18 +159,7 @@ coreos:
        RestartSec=10

    - name: docker.service
-      drop-ins:
-        - name: 50-docker-opts.conf
-          content: |
-            [Service]
-            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
-            MountFlags=slave
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-            LimitCORE=infinity
-            Restart=always
-            RestartSec=2s
-            StartLimitInterval=0
+      command: stop

    - name: kubernetes-configure-node.service
      command: start
--- a/cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh
+++ b/cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh
@ -15,52 +15,76 @@
 # limitations under the License.

 # loadedImageFlags is a bit-flag to track which docker images loaded successfully.
-let loadedImageFlags=0

-while true; do
-  restart_docker=false
-
-  if which docker 1>/dev/null 2>&1; then
-
-    timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
-    rc=$?
-    if [[ $rc == 0 ]]; then
-      let loadedImageFlags="$loadedImageFlags|1"
-    elif [[ $rc == 124 ]]; then
-      restart_docker=true
+function load-docker-images() {
+  let loadedImageFlags=0
+  
+  while true; do
+    restart_docker=false
+  
+    if which docker 1>/dev/null 2>&1; then
+  
+      timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
+      rc=$?
+      if [[ $rc == 0 ]]; then
+        let loadedImageFlags="$loadedImageFlags|1"
+      elif [[ $rc == 124 ]]; then
+        restart_docker=true
+      fi
+  
+      timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
+      rc=$?
+      if [[ $rc == 0 ]]; then
+        let loadedImageFlags="$loadedImageFlags|2"
+      elif [[ $rc == 124 ]]; then
+        restart_docker=true
+      fi
+  
+      timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
+      rc=$?
+      if [[ $rc == 0 ]]; then
+        let loadedImageFlags="$loadedImageFlags|4"
+      elif [[ $rc == 124 ]]; then
+        restart_docker=true
+      fi
    fi
-
-    timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
-    rc=$?
-    if [[ $rc == 0 ]]; then
-      let loadedImageFlags="$loadedImageFlags|2"
-    elif [[ $rc == 124 ]]; then
-      restart_docker=true
+  
+    # required docker images got installed. exit while loop.
+    if [[ $loadedImageFlags == 7 ]]; then break; fi
+  
+    # Sometimes docker load hang, restart docker daemon resolve the issue
+    if [[ $restart_docker ]]; then
+      if ! service docker restart; then # Try systemctl if there's no service command.
+        systemctl restart docker      
+      fi
    fi
+  
+    # sleep for 15 seconds before attempting to load docker images again
+    sleep 15
+  
+  done
+}

-    timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
-    rc=$?
-    if [[ $rc == 0 ]]; then
-      let loadedImageFlags="$loadedImageFlags|4"
-    elif [[ $rc == 124 ]]; then
-      restart_docker=true
-    fi
-  fi
+function convert-rkt-image() {
+  (cd /tmp; ${DOCKER2ACI_BIN} $1)
+}

-  # required docker images got installed. exit while loop.
-  if [[ $loadedImageFlags == 7 ]]; then break; fi
+function load-rkt-images() {
+  convert-rkt-image /srv/salt/kube-bins/kube-apiserver.tar
+  convert-rkt-image /srv/salt/kube-bins/kube-scheduler.tar
+  convert-rkt-image /srv/salt/kube-bins/kube-controller-manager.tar

-  # Sometimes docker load hang, restart docker daemon resolve the issue
-  if [[ $restart_docker ]]; then
-    if ! service docker restart; then # Try systemctl if there's no service command.
-      systemctl restart docker      
-    fi
-  fi
+  # Currently, we can't run docker image tarballs directly,
+  # So we use 'rkt fetch' to load the docker images into rkt image stores.
+  # see https://github.com/coreos/rkt/issues/2392.
+  ${RKT_BIN} fetch /tmp/*.aci --insecure-options=image
+}

-  # sleep for 15 seconds before attempting to load docker images again
-  sleep 15
-
-done
+if [[ "${KUBERNETES_CONTAINER_RUNTIME}" == "rkt" ]]; then
+  load-rkt-images
+else
+  load-docker-images
+fi

 # Now exit. After kube-push, salt will notice that the service is down and it
 # will start it and new docker images will be loaded.