cluster/gce/coreos: Update master and node cloud config.

cluster/gce/coreos/master.yaml

@@ -0,0 +1,188 @@
+#cloud-config
+
+coreos:
+  units:
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-rkt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch rkt
+        Documentation=http://github.com/coreos/rkt
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /etc/rkt
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/rkt.tar.gz -C /opt --overwrite
+
+    - name: kubernetes-download-salt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download salt
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=false \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --rkt-path=/opt/rkt-v${RKT_VERSON}/rkt \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --pod-cidr=${MASTER_IP_RANGE} \
+        --register-schedulable=false \
+        --reconcile-cidr=false
+        Restart=always
+        RestartSec=10
+        
+    - name: docker.service
+      command: start
+      drop-ins:
+        - name: 50-docker-opts.conf
+          content: |
+            [Service]
+            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
+            MountFlags=slave
+            LimitNOFILE=1048576
+            LimitNPROC=1048576
+            LimitCORE=infinity
+            Restart=always
+            RestartSec=2s
+            StartLimitInterval=0
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        Requires=kubernetes-download-salt.service
+        After=kubernetes-download-salt.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        # Need the kubelet/docker running because we will use docker load for docker images.
+        Requires=kubelet.service
+        After=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh
+
+    - name: kubernetes-addons.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start Kubernetes addons and watch for updates.
+        Requires=kubernetes-configure-node.service
+        After=kubernetes-configure-node.service
+        [Service]
+        Environment=KUBECTL_BIN=/opt/kubernetes/server/bin/kubectl
+        Environment=kubelet_kubeconfig_file=/var/lib/kubelet/kubeconfig
+        ExecStartPre=/usr/bin/chmod 0755 /opt/kubernetes/saltbase/salt/kube-addons/kube-addons.sh
+        ExecStart=/opt/kubernetes/saltbase/salt/kube-addons/kube-addons.sh
+        Restart=always
+        RestartSec=10

cluster/gce/coreos/node.yaml

@@ -1,68 +1,5 @@
 #cloud-config
 
-write_files:
-  - path: /run/configure-hostname.sh
-    permissions: "0755"
-    content: |
-      #!/bin/bash -e
-      set -x
-      source /etc/kube-env
-
-      hostnamectl set-hostname $(hostname | cut -f1 -d.)
-  - path: /run/setup-auth.sh
-    permissions: "0755"
-    content: |
-      #!/bin/bash -e
-      set -x
-      source /etc/kube-env
-
-      /usr/bin/mkdir -p /var/lib/kubelet
-      cat > /var/lib/kubelet/kubeconfig << EOF
-      apiVersion: v1
-      kind: Config
-      users:
-      - name: kubelet
-        user:
-          token: ${KUBELET_TOKEN}
-      clusters:
-      - name: local
-        cluster:
-          insecure-skip-tls-verify: true
-      contexts:
-      - context:
-          cluster: local
-          user: kubelet
-        name: service-account-context
-      current-context: service-account-context
-      EOF
-
-  - path: /run/config-kube-proxy.sh
-    permissions: "0755"
-    content: |
-      #!/bin/bash -e
-      set -x
-      source /etc/kube-env
-
-      /usr/bin/mkdir -p /var/lib/kube-proxy
-      cat > /var/lib/kube-proxy/kubeconfig << EOF
-      apiVersion: v1
-      kind: Config
-      users:
-      - name: kube-proxy
-        user:
-          token: ${KUBE_PROXY_TOKEN}
-      clusters:
-      - name: local
-        cluster:
-          insecure-skip-tls-verify: true
-      contexts:
-      - context:
-          cluster: local
-          user: kube-proxy
-        name: service-account-context
-      current-context: service-account-context
-      EOF
-
 coreos:
   units:
     - name: kube-env.service
@@ -80,30 +17,46 @@ coreos:
         -o /etc/kube-env.yaml \
         http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
         # Transform the yaml to env file.
-        ExecStartPre=/usr/bin/cp /etc/kube-env.yaml /etc/kube-env
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
         ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
 
     - name: kubernetes-install-rkt.service
       command: start
       content: |
         [Unit]
-        Description=Fetch Rocket
+        Description=Fetch rkt
         Documentation=http://github.com/coreos/rkt
         Requires=network-online.target
         After=network-online.target
         [Service]
         Type=oneshot
+        RemainAfterExit=yes
         EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/rm -rf /opt/rkt
-        ExecStartPre=/usr/bin/mkdir -p /opt/rkt
         ExecStartPre=/usr/bin/mkdir -p /etc/rkt
-        ExecStartPre=/usr/bin/wget \
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        -O /opt/rkt/rkt-v${RKT_VERSION}.tar.gz \
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
-        https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/rkt.tar.gz -C /opt --overwrite
-        ExecStartPre=/usr/bin/tar xzvf /opt/rkt/rkt-v${RKT_VERSION}.tar.gz -C /opt --overwrite
-        ExecStart=/usr/bin/mv /opt/rkt-v${RKT_VERSION} /opt/rkt/rkt
 
-    - name: kubernetes-install-minion.service
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
       command: start
       content: |
         [Unit]
@@ -120,33 +73,25 @@ coreos:
         ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
         ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
 
-    - name: kubernetes-preparation.service
-      command: start
-      content: |
-        [Unit]
-        Description=Configure Node For Kubernetes service
-        Requires=kubernetes-install-minion.service
-        After=kubernetes-install-minion.service
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        # TODO(dawnchen): Push this to separate write-files
-        ExecStart=/run/configure-hostname.sh
-
     - name: kubelet.service
       command: start
       content: |
         [Unit]
         Description=Run Kubelet service
-        Requires=kubernetes-preparation.service
+        Requires=network-online.target
-        After=kubernetes-preparation.service
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
         [Service]
         EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        ExecStartPre=/run/setup-auth.sh
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
         ExecStart=/opt/kubernetes/server/bin/kubelet \
         --api-servers=https://${INSTANCE_PREFIX}-master \
         --enable-debugging-handlers=true \
@@ -158,9 +103,8 @@ coreos:
         --cluster-domain=${DNS_DOMAIN} \
         --logtostderr=true \
         --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
-        --rkt-path=/opt/rkt/rkt/rkt \
+        --rkt-path=/opt/rkt-v${RKT_VERSON}/rkt \
-        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0}
-        --pod-cidr=${MASTER_IP_RANGE} \
         Restart=always
         RestartSec=10
 
@@ -169,15 +113,12 @@ coreos:
       content: |
         [Unit]
         Description=Start Kube-proxy service as Daemon
-        Requires=kubernetes-install-minion.service
+        Requires=kubernetes-configure-node.service
-        After=kubernetes-install-minion.service
+        After=kubernetes-configure-node.service
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
         [Service]
         EnvironmentFile=/etc/kube-env
-        ExecStartPre=/run/config-kube-proxy.sh
         ExecStart=/opt/kubernetes/server/bin/kube-proxy \
-        --master=https://${KUBERNETES_MASTER_NAME}.c.${PROJECT_ID}.internal \
+        --master=https://${KUBERNETES_MASTER_NAME} \
         --kubeconfig=/var/lib/kube-proxy/kubeconfig \
         --v=2 \
         --logtostderr=true
@@ -191,3 +132,32 @@ coreos:
           content: |
             [Service]
             Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
+            MountFlags=slave
+            LimitNOFILE=1048576
+            LimitNPROC=1048576
+            LimitCORE=infinity
+            Restart=always
+            RestartSec=2s
+            StartLimitInterval=0
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh