Merge pull request #7667 from antoineco/master

update AWS CloudFormation template and cloud-configs

docs/getting-started-guides/README.md

@@ -21,7 +21,7 @@ libvirt        | custom       | Fedora | flannel   | [docs](../../docs/getting-s
 KVM            | custom       | Fedora | flannel   | [docs](../../docs/getting-started-guides/fedora/flannel_multi_node_cluster.md) | Community ([@aveshagarwal](https://github.com/aveshagarwal))| Tested with 0.15.0
 Bare-metal     | custom       | Ubuntu Cluster | flannel | [docs](../../docs/getting-started-guides/ubuntu_multinodes_cluster.md) | Community (@resouer @WIZARD-CXY) | use k8s version 0.12.0
 Mesos/GCE      |              |        |             | [docs](../../docs/getting-started-guides/mesos.md)     | [Community](https://github.com/mesosphere/kubernetes-mesos) ([@jdef](https://github.com/jdef)) | Uses K8s v0.11.2
-AWS            | CoreOS       | CoreOS | flannel     | [docs](../../docs/getting-started-guides/coreos.md)    | Community                    | Uses K8s version 0.15.0
+AWS            | CoreOS       | CoreOS | flannel     | [docs](../../docs/getting-started-guides/coreos.md)    | Community                    | Uses K8s version 0.16.1
 GCE            | CoreOS       | CoreOS | flannel     | [docs](../../docs/getting-started-guides/coreos.md)    | Community (@kelseyhightower) | Uses K8s version 0.15.0
 Vagrant        | CoreOS       | CoreOS | flannel     | [docs](../../docs/getting-started-guides/coreos.md)    | Community ( [@pires](https://github.com/pires), [@AntonioMeireles](https://github.com/AntonioMeireles) )           | Uses K8s version 0.15.0
 Bare-metal (Offline) | CoreOS       | CoreOS | flannel      | [docs](../../docs/getting-started-guides/coreos/bare_metal_offline.md) | Community([@jeffbean](https://github.com/jeffbean))    | Uses K8s version 0.15.0

docs/getting-started-guides/aws/cloud-configs/master.yaml

@@ -1,47 +1,56 @@
 #cloud-config
 
----
+write_files:
-hostname: master
+  - path: /opt/bin/waiter.sh
+    owner: root
+    permissions: 0755
+    content: |
+      #! /usr/bin/bash
+      until curl http://127.0.0.1:2379/v2/machines; do sleep 2; done
+
 coreos:
   etcd2:
     name: master
-    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
-    advertise-client-urls: http://<master-private-ip>:2379,http://<master-private-ip>:4001
     initial-cluster-token: k8s_etcd
-    listen-peer-urls: http://<master-private-ip>:2380,http://<master-private-ip>:7001
+    initial-cluster: master=http://$private_ipv4:2380
-    initial-advertise-peer-urls: http://<master-private-ip>:2380
+    listen-peer-urls: http://$private_ipv4:2380,http://localhost:2380
-    initial-cluster: master=http://<master-private-ip>:2380
+    initial-advertise-peer-urls: http://$private_ipv4:2380
-    initial-cluster-state: new
+    listen-client-urls: http://$private_ipv4:2379,http://localhost:2379
+    advertise-client-urls: http://$private_ipv4:2379
   fleet:
-    metadata: "role=master"
+    etcd_servers: http://localhost:2379
+    metadata: k8srole=master
+  flannel:
+    etcd_endpoints: http://localhost:2379
+  locksmithd:
+    endpoint: http://localhost:2379
   units:
-    - name: setup-network-environment.service
+    - name: etcd2.service
+      command: start
+    - name: fleet.service
+      command: start
+    - name: etcd2-waiter.service
       command: start
       content: |
         [Unit]
-        Description=Setup Network Environment
+        Description=etcd waiter
-        Documentation=https://github.com/kelseyhightower/setup-network-environment
+        Wants=network-online.target
-        Requires=network-online.target
+        Wants=etcd2.service
+        After=etcd2.service
         After=network-online.target
+        Before=flanneld.service fleet.service locksmithd.service
 
         [Service]
-        ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+        ExecStart=/usr/bin/bash /opt/bin/waiter.sh
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/setup-network-environment 
+        RemainAfterExit=true
-        ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment
-        ExecStart=/opt/bin/setup-network-environment
-        RemainAfterExit=yes
         Type=oneshot
-    - name: fleet.service
-      command: start
     - name: flanneld.service
       command: start
       drop-ins:
         - name: 50-network-config.conf
           content: |
-            [Unit]
-            Requires=etcd2.service
             [Service]
-            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"10.244.0.0/16", "Backend": {"Type": "vxlan"}}'
+            ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config '{"Network": "10.244.0.0/16", "Backend": {"Type": "vxlan"}}'
     - name: docker-cache.service
       command: start
       content: |
@@ -55,8 +64,8 @@ coreos:
         Restart=always
         TimeoutStartSec=0
         RestartSec=5
-        Environment="TMPDIR=/var/tmp/"
+        Environment=TMPDIR=/var/tmp/
-        Environment="DOCKER_HOST=unix:///var/run/early-docker.sock"
+        Environment=DOCKER_HOST=unix:///var/run/early-docker.sock
         ExecStartPre=-/usr/bin/docker kill docker-registry
         ExecStartPre=-/usr/bin/docker rm docker-registry
         ExecStartPre=/usr/bin/docker pull quay.io/devops/docker-registry:latest
@@ -76,33 +85,41 @@ coreos:
             [Unit]
             # making sure that docker-cache is up and that flanneld finished
             # startup, otherwise containers won't land in flannel's network...
-            Requires=docker-cache.service flanneld.service
+            Requires=docker-cache.service
-            After=docker-cache.service flanneld.service
+            After=docker-cache.service
+
             [Service]
             Environment=DOCKER_OPTS='--registry-mirror=http://$private_ipv4:5000'
+    - name: get-kubectl.service
+      command: start
+      content: |
+        [Unit]
+        Description=Get kubectl client tool
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+        Requires=network-online.target
+        After=network-online.target
+
+        [Service]
+        ExecStart=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kubectl
+        ExecStart=/usr/bin/chmod +x /opt/bin/kubectl
+        Type=oneshot
+        RemainAfterExit=true
     - name: kube-apiserver.service
       command: start
       content: |
         [Unit]
         Description=Kubernetes API Server
         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-        Requires=etcd2.service
+        Requires=etcd2-waiter.service
-        After=etcd2.service
+        After=etcd2-waiter.service
 
         [Service]
-        ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-apiserver
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-apiserver
         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-apiserver
         ExecStart=/opt/bin/kube-apiserver \
-        --allow_privileged=true \
+        --insecure-bind-address=0.0.0.0 \
-        --insecure_bind_address=0.0.0.0 \
+        --portal-net=10.100.0.0/16 \
-        --insecure_port=8080 \
+        --etcd-servers=http://localhost:2379
-        --kubelet_https=true \
-        --secure_port=6443 \
-        --portal_net=10.100.0.0/16 \
-        --etcd_servers=http://127.0.0.1:4001 \
-        --public_address_override=$private_ipv4 \
-        --logtostderr=true
         Restart=always
         RestartSec=10
     - name: kube-controller-manager.service
@@ -115,11 +132,10 @@ coreos:
         After=kube-apiserver.service
 
         [Service]
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-controller-manager
+        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-controller-manager
         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-controller-manager
         ExecStart=/opt/bin/kube-controller-manager \
-        --master=127.0.0.1:8080 \
+        --master=127.0.0.1:8080
-        --logtostderr=true
         Restart=always
         RestartSec=10
     - name: kube-scheduler.service
@@ -132,9 +148,10 @@ coreos:
         After=kube-apiserver.service
 
         [Service]
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-scheduler
+        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-scheduler
         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-scheduler
-        ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080
+        ExecStart=/opt/bin/kube-scheduler \
+        --master=127.0.0.1:8080
         Restart=always
         RestartSec=10
     - name: kube-register.service
@@ -143,20 +160,16 @@ coreos:
         [Unit]
         Description=Kubernetes Registration Service
         Documentation=https://github.com/kelseyhightower/kube-register
-        Requires=kube-apiserver.service
+        Requires=kube-apiserver.service fleet.service
-        After=kube-apiserver.service
+        After=kube-apiserver.service fleet.service
-        Requires=fleet.service
-        After=fleet.service
 
         [Service]
-        # ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/kube-register
+        ExecStartPre=-/usr/bin/wget -nc -O /opt/bin/kube-register https://github.com/kelseyhightower/kube-register/releases/download/v0.0.3/kube-register-0.0.3-linux-amd64
-        ExecStartPre=/usr/bin/wget -N -O /opt/bin/kube-register https://github.com/kelseyhightower/kube-register/releases/download/v0.0.3/kube-register-0.0.3-linux-amd64
         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register
         ExecStart=/opt/bin/kube-register \
-        --metadata=role=node \
+        --metadata=k8srole=node \
         --fleet-endpoint=unix:///var/run/fleet.sock \
-        --api-endpoint=http://127.0.0.1:8080 \
+        --api-endpoint=http://127.0.0.1:8080
-        --healthz-port=10248
         Restart=always
         RestartSec=10
   update:

docs/getting-started-guides/aws/cloud-configs/node.yaml

@@ -1,7 +1,9 @@
 #cloud-config
-write-files:
+
+write_files:
   - path: /opt/bin/wupiao
-    permissions: '0755'
+    owner: root
+    permissions: 0755
     content: |
       #!/bin/bash
       # [w]ait [u]ntil [p]ort [i]s [a]ctually [o]pen
@@ -9,94 +11,68 @@ write-files:
         --silent --head --fail \
         http://${1}:${2}; do sleep 1 && echo -n .; done;
       exit $?
+
 coreos:
   etcd2:
-    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001
+    listen-client-urls: http://localhost:2379
     initial-cluster: master=http://<master-private-ip>:2380
     proxy: on
   fleet:
-    metadata: "role=node"
+    etcd_servers: http://localhost:2379
+    metadata: k8srole=node
+  flannel:
+    etcd_endpoints: http://localhost:2379
+  locksmithd:
+    endpoint: http://localhost:2379
   units:
+    - name: etcd2.service
+      command: start
     - name: fleet.service
       command: start
     - name: flanneld.service
       command: start
-      drop-ins:
-        - name: 50-network-config.conf
-          content: |
-            [Unit]
-            Requires=etcd2.service
-            [Service]
-            ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{"Network":"10.244.0.0/16", "Backend": {"Type": "vxlan"}}'
     - name: docker.service
       command: start
       drop-ins:
-        - name: 51-docker-mirror.conf
+        - name: 50-docker-mirror.conf
           content: |
-            [Unit]
-            Requires=flanneld.service
-            After=flanneld.service
             [Service]
             Environment=DOCKER_OPTS='--registry-mirror=http://<master-private-ip>:5000'
-    - name: setup-network-environment.service
+    - name: kubelet.service
       command: start
       content: |
         [Unit]
-        Description=Setup Network Environment
+        Description=Kubernetes Kubelet
-        Documentation=https://github.com/kelseyhightower/setup-network-environment
+        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
         Requires=network-online.target
         After=network-online.target
 
         [Service]
-        ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kubelet
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/setup-network-environment 
+        ExecStartPre=/usr/bin/chmod +x /opt/bin/kubelet
-        ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment
+        # wait for kubernetes master to be up and ready
-        ExecStart=/opt/bin/setup-network-environment
+        ExecStartPre=/opt/bin/wupiao <master-private-ip> 8080
-        RemainAfterExit=yes
+        ExecStart=/opt/bin/kubelet \
-        Type=oneshot
+        --api-servers=<master-private-ip>:8080 \
+        --hostname-override=$private_ipv4
+        Restart=always
+        RestartSec=10
     - name: kube-proxy.service
       command: start
       content: |
         [Unit]
         Description=Kubernetes Proxy
         Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-        Requires=setup-network-environment.service
+        Requires=network-online.target
-        After=setup-network-environment.service
+        After=network-online.target
 
         [Service]
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-proxy
+        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-proxy
         ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-proxy
         # wait for kubernetes master to be up and ready
         ExecStartPre=/opt/bin/wupiao <master-private-ip> 8080
         ExecStart=/opt/bin/kube-proxy \
-        --master=<master-private-ip>:8080 \
+        --master=http://<master-private-ip>:8080
-        --logtostderr=true
-        Restart=always
-        RestartSec=10
-    - name: kube-kubelet.service
-      command: start
-      content: |
-        [Unit]
-        Description=Kubernetes Kubelet
-        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-        Requires=setup-network-environment.service
-        After=setup-network-environment.service
-
-        [Service]
-        EnvironmentFile=/etc/network-environment
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kubelet
-        ExecStartPre=/usr/bin/chmod +x /opt/bin/kubelet
-        # wait for kubernetes master to be up and ready
-        ExecStartPre=/opt/bin/wupiao <master-private-ip> 8080
-        ExecStart=/opt/bin/kubelet \
-        --address=0.0.0.0 \
-        --port=10250 \
-        --hostname_override=$private_ipv4 \
-        --api_servers=<master-private-ip>:8080 \
-        --allow_privileged=true \
-        --logtostderr=true \
-        --healthz_bind_address=0.0.0.0 \
-        --healthz_port=10248
         Restart=always
         RestartSec=10
   update:

docs/getting-started-guides/aws/cloudformation-template.json

@@ -1,17 +1,38 @@
 {
   "AWSTemplateFormatVersion": "2010-09-09",
-  "Description": "Kubernetes 0.11.0 on EC2",
+  "Description": "Kubernetes 0.16.1 on EC2 powered by CoreOS 668.2.0 (alpha)",
   "Mappings": {
       "RegionMap": {
-          "eu-central-1": {"AMI": "ami-488ebe55"},
+          "eu-central-1" : {
-          "ap-northeast-1": {"AMI": "ami-14273415"},
+              "AMI" : "ami-5a330c47"
-          "sa-east-1": {"AMI": "ami-e7b30efa"},
+          },
-          "ap-southeast-2": {"AMI": "ami-4d2b5e77"},
+          "ap-northeast-1" : {
-          "ap-southeast-1": {"AMI": "ami-8c032bde"},
+              "AMI" : "ami-905c9f90"
-          "us-east-1": {"AMI": "ami-4c651824"},
+          },
-          "us-west-2": {"AMI": "ami-ff2679cf"},
+          "us-gov-west-1" : {
-          "us-west-1": {"AMI": "ami-5d4f5118"},
+              "AMI" : "ami-d1debff2"
-          "eu-west-1": {"AMI": "ami-5d71f02a"}
+          },
+          "sa-east-1" : {
+              "AMI" : "ami-d365e1ce"
+          },
+          "ap-southeast-2" : {
+              "AMI" : "ami-7db5c947"
+          },
+          "ap-southeast-1" : {
+              "AMI" : "ami-06c8f454"
+          },
+          "us-east-1" : {
+              "AMI" : "ami-2cd3dc44"
+          },
+          "us-west-2" : {
+              "AMI" : "ami-69447259"
+          },
+          "us-west-1" : {
+              "AMI" : "ami-d524c891"
+          },
+          "eu-west-1" : {
+              "AMI" : "ami-c5b7d8b2"
+          }
       }
   },
   "Parameters": {
@@ -49,9 +70,9 @@
       "ConstraintDescription": "Must be a valid EC2 HVM instance type."
     },
     "ClusterSize": {
-      "Description": "Number of nodes in cluster (3-12).",
+      "Description": "Number of nodes in cluster (2-12).",
-      "Default": "3",
+      "Default": "2",
-      "MinValue": "3",
+      "MinValue": "2",
       "MaxValue": "12",
       "Type": "Number"
     },
@@ -66,18 +87,15 @@
     },
     "VpcId": {
       "Description": "The ID of the VPC to launch into.",
-      "Type": "AWS::EC2::VPC::Id",
+      "Type": "AWS::EC2::VPC::Id"
-      "Default": ""
     },
     "SubnetId": {
       "Description": "The ID of the subnet to launch into (that must be within the supplied VPC)",
-      "Type": "AWS::EC2::Subnet::Id",
+      "Type": "AWS::EC2::Subnet::Id"
-      "Default": ""
     },
     "SubnetAZ": {
       "Description": "The availability zone of the subnet supplied (for example eu-west-1a)",
-      "Type": "String",
+      "Type": "String"
-      "Default": ""
     }
   },
   "Conditions": {
@@ -126,124 +144,124 @@
     "KubernetesMasterInstance": {
       "Type": "AWS::EC2::Instance",
       "Properties": {
-        "SubnetId": {"Fn::If": ["UseEC2Classic", {"Ref": "AWS::NoValue"}, {"Ref": "SubnetId"}]},
+        "NetworkInterfaces" : [{
+          "GroupSet"                 : [{"Fn::GetAtt": ["KubernetesSecurityGroup", "GroupId"]}],
+          "AssociatePublicIpAddress" : "true",
+          "DeviceIndex"              : "0",
+          "DeleteOnTermination"      : "true",
+          "SubnetId"                 : {"Fn::If": ["UseEC2Classic", {"Ref": "AWS::NoValue"}, {"Ref": "SubnetId"}]}
+        }],
         "ImageId": {"Fn::FindInMap" : ["RegionMap", {"Ref": "AWS::Region" }, "AMI"]},
         "InstanceType": {"Ref": "InstanceType"},
         "KeyName": {"Ref": "KeyPair"},
-        "SecurityGroups": [{"Fn::If": [
+        "Tags" : [
-          "UseEC2Classic",
+          {"Key" : "Name", "Value" : {"Fn::Join" : [ "-", [ {"Ref" : "AWS::StackName"}, "k8s-master" ] ]}},
-          {"Ref": "KubernetesSecurityGroup"},
+          {"Key" : "KubernetesRole", "Value" : "node"}
-          {"Ref": "AWS::NoValue"}]
+        ],
-        }],
-        "SecurityGroupIds": [{"Fn::If": [
-          "UseEC2Classic",
-          {"Ref": "AWS::NoValue"},
-          {"Fn::GetAtt": ["KubernetesSecurityGroup", "GroupId"]}]
-        }],
         "UserData": { "Fn::Base64": {"Fn::Join" : ["", [
           "#cloud-config\n\n",
-          "---\n",
           "write_files:\n",
           "- path: /opt/bin/waiter.sh\n",
           "  owner: root\n",
           "  content: |\n",
           "    #! /usr/bin/bash\n",
-          "    until curl http://127.0.0.1:4001/v2/machines; do sleep 2; done\n",
+          "    until curl http://127.0.0.1:2379/v2/machines; do sleep 2; done\n",
           "coreos:\n",
+          "  etcd2:\n",
+          "    name: master\n",
+          "    initial-cluster-token: k8s_etcd\n",
+          "    initial-cluster: master=http://$private_ipv4:2380\n",
+          "    listen-peer-urls: http://$private_ipv4:2380,http://localhost:2380\n",
+          "    initial-advertise-peer-urls: http://$private_ipv4:2380\n",
+          "    listen-client-urls: http://$private_ipv4:2379,http://localhost:2379\n",
+          "    advertise-client-urls: http://$private_ipv4:2379\n",
           "  fleet:\n",
-          "    etcd-servers: http://localhost:4001\n",
+          "    etcd_servers: http://localhost:2379\n",
-          "    metadata: \"role=master\"\n",
+          "    metadata: k8srole=master\n",
+          "  flannel:\n",
+          "    etcd_endpoints: http://localhost:2379\n",
+          "  locksmithd:\n",
+          "    endpoint: http://localhost:2379\n",
           "  units:\n",
-          "    - name: setup-network-environment.service\n",
+          "    - name: etcd2.service\n",
           "      command: start\n",
-          "      content: |\n",
-          "        [Unit]\n",
-          "        Description=Setup Network Environment\n",
-          "        Documentation=https://github.com/kelseyhightower/setup-network-environment\n",
-          "        Requires=network-online.target\n",
-          "        After=network-online.target\n\n",
-          "        [Service]\n",
-          "        ExecStartPre=/usr/bin/mkdir -p /opt/bin\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/setup-network-environment\n",
-          "        ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment\n",
-          "        ExecStart=/opt/bin/setup-network-environment\n",
-          "        RemainAfterExit=yes\n",
-          "        Type=oneshot\n",
-          "    - name: etcd.service\n",
-          "      command: start\n",
-          "      content: |\n",
-          "        [Unit]\n",
-          "        Description=etcd\n",
-          "        Requires=setup-network-environment.service\n",
-          "        After=setup-network-environment.service\n\n",
-          "        [Service]\n",
-          "        EnvironmentFile=/etc/network-environment\n",
-          "        User=etcd\n",
-          "        PermissionsStartOnly=true\n",
-          "        ExecStart=/usr/bin/etcd \\\n",
-          "        --name $private_ipv4 \\\n",
-          "        --addr $private_ipv4:4001 \\\n",
-          "        --bind-addr 0.0.0.0 \\\n",
-          "        --cluster-active-size 1 \\\n",
-          "        --data-dir /var/lib/etcd \\\n",
-          "        --http-read-timeout 86400 \\\n",
-          "        --peer-addr $private_ipv4:7001 \\\n",
-          "        --snapshot true\n",
-          "        Restart=always\n",
-          "        RestartSec=10s\n",
           "    - name: fleet.service\n",
           "      command: start\n",
-          "    - name: etcd-waiter.service\n",
+          "    - name: etcd2-waiter.service\n",
           "      command: start\n",
           "      content: |\n",
           "        [Unit]\n",
           "        Description=etcd waiter\n",
           "        Wants=network-online.target\n",
-          "        Wants=etcd.service\n",
+          "        Wants=etcd2.service\n",
-          "        After=etcd.service\n",
+          "        After=etcd2.service\n",
           "        After=network-online.target\n",
-          "        Before=flannel.service\n\n",
+          "        Before=flanneld.service fleet.service locksmithd.service\n\n",
           "        [Service]\n",
-          "        ExecStartPre=/usr/bin/chmod +x /opt/bin/waiter.sh\n",
           "        ExecStart=/usr/bin/bash /opt/bin/waiter.sh\n",
           "        RemainAfterExit=true\n",
           "        Type=oneshot\n",
-          "    - name: flannel.service\n",
+          "    - name: flanneld.service\n",
+          "      command: start\n",
+          "      drop-ins:\n",
+          "        - name: 50-network-config.conf\n",
+          "          content: |\n",
+          "            [Service]\n",
+          "            ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config '{\"Network\": \"10.244.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}'\n",
+          "    - name: docker-cache.service\n",
           "      command: start\n",
           "      content: |\n",
           "        [Unit]\n",
-          "        Requires=etcd-waiter.service\n",
+          "        Description=Docker cache proxy\n",
-          "        After=etcd-waiter.service\n",
+          "        Requires=early-docker.service\n",
-          "        Requires=etcd.service\n",
+          "        After=early-docker.service\n",
-          "        After=etcd.service\n",
+          "        Before=early-docker.target\n\n",
-          "        After=network-online.target\n",
-          "        Wants=network-online.target\n",
-          "        Description=flannel is an etcd backed overlay network for containers\n\n",
           "        [Service]\n",
-          "        Type=notify\n",
+          "        Restart=always\n",
-          "        ExecStartPre=-/usr/bin/mkdir -p /opt/bin\n",
+          "        TimeoutStartSec=0\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/flanneld\n",
+          "        RestartSec=5\n",
-          "        ExecStartPre=/usr/bin/chmod +x /opt/bin/flanneld\n",
+          "        Environment=TMPDIR=/var/tmp/\n",
-          "        ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config '{\"Network\":\"10.244.0.0/16\"}'\n",
+          "        Environment=DOCKER_HOST=unix:///var/run/early-docker.sock\n",
-          "        ExecStart=/opt/bin/flanneld\n",
+          "        ExecStartPre=-/usr/bin/docker kill docker-registry\n",
+          "        ExecStartPre=-/usr/bin/docker rm docker-registry\n",
+          "        ExecStartPre=/usr/bin/docker pull quay.io/devops/docker-registry:latest\n",
+          "        # GUNICORN_OPTS is an workaround for\n",
+          "        # https://github.com/docker/docker-registry/issues/892\n",
+          "        ExecStart=/usr/bin/docker run --rm --net host --name docker-registry \\\n",
+          "            -e STANDALONE=false \\\n",
+          "            -e GUNICORN_OPTS=[--preload] \\\n",
+          "            -e MIRROR_SOURCE=https://registry-1.docker.io \\\n",
+          "            -e MIRROR_SOURCE_INDEX=https://index.docker.io \\\n",
+          "            -e MIRROR_TAGS_CACHE_TTL=1800 \\\n",
+          "            quay.io/devops/docker-registry:latest\n",
+          "    - name: get-kubectl.service\n",
+          "      command: start\n",
+          "      content: |\n",
+          "        [Unit]\n",
+          "        Description=Get kubectl client tool\n",
+          "        Documentation=https://github.com/GoogleCloudPlatform/kubernetes\n",
+          "        Requires=network-online.target\n",
+          "        After=network-online.target\n\n",
+          "        [Service]\n",
+          "        ExecStart=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kubectl\n",
+          "        ExecStart=/usr/bin/chmod +x /opt/bin/kubectl\n",
+          "        Type=oneshot\n",
+          "        RemainAfterExit=true\n",
           "    - name: kube-apiserver.service\n",
           "      command: start\n",
           "      content: |\n",
           "        [Unit]\n",
           "        Description=Kubernetes API Server\n",
           "        Documentation=https://github.com/GoogleCloudPlatform/kubernetes\n",
-          "        Requires=etcd.service\n",
+          "        Requires=etcd2-waiter.service\n",
-          "        After=etcd.service\n\n",
+          "        After=etcd2-waiter.service\n\n",
           "        [Service]\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.11.0/bin/linux/amd64/kube-apiserver\n",
+          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-apiserver\n",
           "        ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-apiserver\n",
           "        ExecStart=/opt/bin/kube-apiserver \\\n",
-          "        --address=0.0.0.0 \\\n",
+          "        --insecure-bind-address=0.0.0.0 \\\n",
-          "        --port=8080 \\\n",
+          "        --portal-net=10.100.0.0/16 \\\n",
-          "        --portal_net=10.100.0.0/16 \\\n",
+          "        --etcd-servers=http://localhost:2379\n",
-          "        --etcd_servers=http://127.0.0.1:4001 \\\n",
-          "        --public_address_override=$private_ipv4 \\\n",
-          "        --logtostderr=true\n",
           "        Restart=always\n",
           "        RestartSec=10\n",
           "    - name: kube-controller-manager.service\n",
@@ -255,11 +273,10 @@
           "        Requires=kube-apiserver.service\n",
           "        After=kube-apiserver.service\n\n",
           "        [Service]\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.11.0/bin/linux/amd64/kube-controller-manager\n",
+          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-controller-manager\n",
           "        ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-controller-manager\n",
           "        ExecStart=/opt/bin/kube-controller-manager \\\n",
-          "        --master=127.0.0.1:8080 \\\n",
+          "        --master=127.0.0.1:8080\n",
-          "        --logtostderr=true\n",
           "        Restart=always\n",
           "        RestartSec=10\n",
           "    - name: kube-scheduler.service\n",
@@ -271,9 +288,10 @@
           "        Requires=kube-apiserver.service\n",
           "        After=kube-apiserver.service\n\n",
           "        [Service]\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.11.0/bin/linux/amd64/kube-scheduler\n",
+          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-scheduler\n",
           "        ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-scheduler\n",
-          "        ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080\n",
+          "        ExecStart=/opt/bin/kube-scheduler \\\n",
+          "        --master=127.0.0.1:8080\n",
           "        Restart=always\n",
           "        RestartSec=10\n",
           "    - name: kube-register.service\n",
@@ -282,15 +300,13 @@
           "        [Unit]\n",
           "        Description=Kubernetes Registration Service\n",
           "        Documentation=https://github.com/kelseyhightower/kube-register\n",
-          "        Requires=kube-apiserver.service\n",
+          "        Requires=kube-apiserver.service fleet.service\n",
-          "        After=kube-apiserver.service\n\n",
+          "        After=kube-apiserver.service fleet.service\n\n",
-          "        Requires=fleet.service\n",
-          "        After=fleet.service\n\n",
           "        [Service]\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/kube-register\n",
+          "        ExecStartPre=-/usr/bin/wget -nc -O /opt/bin/kube-register https://github.com/kelseyhightower/kube-register/releases/download/v0.0.3/kube-register-0.0.3-linux-amd64\n",
           "        ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register\n",
           "        ExecStart=/opt/bin/kube-register \\\n",
-          "        --metadata=role=knode \\\n",
+          "        --metadata=k8srole=node \\\n",
           "        --fleet-endpoint=unix:///var/run/fleet.sock \\\n",
           "        --api-endpoint=http://127.0.0.1:8080\n",
           "        Restart=always\n",
@@ -308,6 +324,7 @@
         "ImageId": {"Fn::FindInMap" : ["RegionMap", {"Ref": "AWS::Region" }, "AMI" ]},
         "InstanceType": {"Ref": "InstanceType"},
         "KeyName": {"Ref": "KeyPair"},
+        "AssociatePublicIpAddress" : "true",
         "SecurityGroups": [{"Fn::If": [
           "UseEC2Classic",
           {"Ref": "KubernetesSecurityGroup"},
@@ -316,74 +333,45 @@
         "UserData": { "Fn::Base64": {"Fn::Join" : ["", [
           "#cloud-config\n\n",
           "coreos:\n",
+          "  etcd2:\n",
+          "    listen-client-urls: http://localhost:2379\n",
+          "    initial-cluster: master=http://", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":2380\n",
+          "    proxy: on\n",
           "  fleet:\n",
-          "    etcd_servers: http://", {"Fn::GetAtt":["KubernetesMasterInstance" , "PrivateIp"]}, ":4001\n",
+          "    etcd_servers: http://localhost:2379\n",
-          "    metadata: role=knode\n",
+          "    metadata: k8srole=node\n",
+          "  flannel:\n",
+          "    etcd_endpoints: http://localhost:2379\n",
+          "  locksmithd:\n",
+          "    endpoint: http://localhost:2379\n",
           "  units:\n",
-          "    - name: etcd.service\n",
+          "    - name: etcd2.service\n",
-          "      mask: true\n",
+          "      command: start\n",
           "    - name: fleet.service\n",
           "      command: start\n",
-          "    - name: flannel.service\n",
+          "    - name: flanneld.service\n",
           "      command: start\n",
-          "      content: |\n",
-          "        [Unit]\n",
-          "        After=network-online.target\n",
-          "        Wants=network-online.target\n",
-          "        Description=flannel is an etcd backed overlay network for containers\n\n",
-          "        [Service]\n",
-          "        Type=notify\n",
-          "        ExecStartPre=-/usr/bin/mkdir -p /opt/bin\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/flanneld\n",
-          "        ExecStartPre=/usr/bin/chmod +x /opt/bin/flanneld\n",
-          "        ExecStart=/opt/bin/flanneld -etcd-endpoints http://", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":4001\n",
           "    - name: docker.service\n",
           "      command: start\n",
+          "      drop-ins:\n",
+          "        - name: 50-docker-mirror.conf\n",
           "          content: |\n",
-          "        [Unit]\n",
-          "        After=flannel.service\n",
-          "        Wants=flannel.service\n",
-          "        Description=Docker Application Container Engine\n",
-          "        Documentation=http://docs.docker.io\n\n",
           "            [Service]\n",
-          "        EnvironmentFile=/run/flannel/subnet.env\n",
+          "            Environment=DOCKER_OPTS='--registry-mirror=http://", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":5000'\n",
-          "        ExecStartPre=/bin/mount --make-rprivate /\n",
+          "    - name: kubelet.service\n",
-          "        ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} -s=overlay -H fd://\n\n",
-          "        [Install]\n",
-          "        WantedBy=multi-user.target\n",
-          "    - name: setup-network-environment.service\n",
-          "      command: start\n",
-          "      content: |\n",
-          "        [Unit]\n",
-          "        Description=Setup Network Environment\n",
-          "        Documentation=https://github.com/kelseyhightower/setup-network-environment\n",
-          "        Requires=network-online.target\n",
-          "        After=network-online.target\n\n",
-          "        [Service]\n",
-          "        ExecStartPre=-/usr/bin/mkdir -p /opt/bin\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/k8s/setup-network-environment\n",
-          "        ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment\n",
-          "        ExecStart=/opt/bin/setup-network-environment\n",
-          "        RemainAfterExit=yes\n",
-          "        Type=oneshot\n",
-          "    - name: kube-kubelet.service\n",
           "      command: start\n",
           "      content: |\n",
           "        [Unit]\n",
           "        Description=Kubernetes Kubelet\n",
           "        Documentation=https://github.com/GoogleCloudPlatform/kubernetes\n",
-          "        Requires=setup-network-environment.service\n",
+          "        Requires=network-online.target\n",
-          "        After=setup-network-environment.service\n\n",
+          "        After=network-online.target\n\n",
           "        [Service]\n",
-          "        EnvironmentFile=/etc/network-environment\n",
+          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kubelet\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.11.0/bin/linux/amd64/kubelet\n",
           "        ExecStartPre=/usr/bin/chmod +x /opt/bin/kubelet\n",
           "        ExecStart=/opt/bin/kubelet \\\n",
-          "        --address=0.0.0.0 \\\n",
+          "        --api-servers=", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":8080 \\\n",
-          "        --port=10250 \\\n",
+          "        --hostname-override=$private_ipv4\n",
-          "        --hostname_override=${DEFAULT_IPV4} \\\n",
-          "        --api_servers=", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":8080\\\n",
-          "        --logtostderr=true\n",
           "        Restart=always\n",
           "        RestartSec=10\n",
           "    - name: kube-proxy.service\n",
@@ -392,14 +380,13 @@
           "        [Unit]\n",
           "        Description=Kubernetes Proxy\n",
           "        Documentation=https://github.com/GoogleCloudPlatform/kubernetes\n",
-          "        Requires=setup-network-environment.service\n",
+          "        Requires=network-online.target\n",
-          "        After=setup-network-environment.service\n\n",
+          "        After=network-online.target\n\n",
           "        [Service]\n",
-          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.11.0/bin/linux/amd64/kube-proxy\n",
+          "        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes-release/release/v0.16.1/bin/linux/amd64/kube-proxy\n",
           "        ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-proxy\n",
           "        ExecStart=/opt/bin/kube-proxy \\\n",
-          "        --master=http://", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":8080\\\n",
+          "        --master=http://", {"Fn::GetAtt" :["KubernetesMasterInstance" , "PrivateIp"]}, ":8080\n",
-          "        --logtostderr=true\n",
           "        Restart=always\n",
           "        RestartSec=10\n",
           "  update:\n",
@@ -415,9 +402,13 @@
         "AvailabilityZones": {"Fn::If": ["UseEC2Classic", {"Fn::GetAZs": ""}, [{"Ref": "SubnetAZ"}]]},
         "VPCZoneIdentifier": {"Fn::If": ["UseEC2Classic", {"Ref": "AWS::NoValue"}, [{"Ref": "SubnetId"}]]},
         "LaunchConfigurationName": {"Ref": "KubernetesNodeLaunchConfig"},
-        "MinSize": "3",
+        "MinSize": "2",
         "MaxSize": "12",
-        "DesiredCapacity": {"Ref": "ClusterSize"}
+        "DesiredCapacity": {"Ref": "ClusterSize"},
+        "Tags" : [
+          {"Key" : "Name", "Value" : {"Fn::Join" : [ "-", [ {"Ref" : "AWS::StackName"}, "k8s-node" ] ]}, "PropagateAtLaunch" : true},
+          {"Key" : "KubernetesRole", "Value" : "node", "PropagateAtLaunch" : true}
+        ]
       }
     }
   },