From b6725a1b119016e5cf281d659ae4c0626fa0f792 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 12 Sep 2017 13:07:48 +0100
Subject: [PATCH 01/16] kubernetes: give more resources to worker nodes.

The sock-shop demo[0] requires around 5G of images on a worker node and 3G of
RAM (if there is only one worker node and therefore everything runs on that
node).

Since the master is more than happy with the 4G disk and 1G RAM it is given
today split the settings into master and node specific and bump only the
latter.

KUBE_PORT_BASE is unused and was already removed in 54ddde0d4344 but
accidentally reintroduced (by me) in 62aa9248a406, whack it again.

[0] https://microservices-demo.github.io/microservices-demo

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/boot.sh | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh
index 97f15aa89..218a6c512 100755
--- a/projects/kubernetes/boot.sh
+++ b/projects/kubernetes/boot.sh
@@ -2,10 +2,14 @@
 
 set -e
 
-: ${KUBE_PORT_BASE:=2222}
-: ${KUBE_VCPUS:=2}
-: ${KUBE_MEM:=1024}
-: ${KUBE_DISK:=4G}
+: ${KUBE_MASTER_VCPUS:=2}
+: ${KUBE_MASTER_MEM:=1024}
+: ${KUBE_MASTER_DISK:=4G}
+
+: ${KUBE_NODE_VCPUS:=2}
+: ${KUBE_NODE_MEM:=4096}
+: ${KUBE_NODE_DISK:=8G}
+
 : ${KUBE_NETWORKING:=default}
 : ${KUBE_RUN_ARGS:=}
 : ${KUBE_EFI:=}
@@ -19,6 +23,10 @@ if [ $# -eq 0 ] ; then
     img="kube-master"
     data=""
     state="kube-master-state"
+
+    : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS}
+    : ${KUBE_MEM:=$KUBE_MASTER_MEM}
+    : ${KUBE_DISK:=$KUBE_MASTER_DISK}
 elif [ $# -gt 1 ] ; then
     case $1 in
 	''|*[!0-9]*)
@@ -36,6 +44,10 @@ elif [ $# -gt 1 ] ; then
     shift
     data="${*}"
     state="kube-${name}-state"
+
+    : ${KUBE_VCPUS:=$KUBE_NODE_VCPUS}
+    : ${KUBE_MEM:=$KUBE_NODE_MEM}
+    : ${KUBE_DISK:=$KUBE_NODE_DISK}
 else
     echo "Usage:"
     echo " - Boot master:"

From 32e53cf76971abd492bd83a8a2c6facc1c8f8a1c Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 12 Sep 2017 13:18:48 +0100
Subject: [PATCH 02/16] kubernetes: Add boot.sh options to preserve state dir
 and to set MAC

Not having to redo the kubeadm-init.sh step massively speeds up the test/dev
cycle. Having the same MAC (and hence same IP) is useful there too since you
don't need to figure out the mac on each boot.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/boot.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh
index 218a6c512..3b041aef4 100755
--- a/projects/kubernetes/boot.sh
+++ b/projects/kubernetes/boot.sh
@@ -13,6 +13,8 @@ set -e
 : ${KUBE_NETWORKING:=default}
 : ${KUBE_RUN_ARGS:=}
 : ${KUBE_EFI:=}
+: ${KUBE_MAC:=}
+: ${KUBE_PRESERVE_STATE:=}
 
 [ "$(uname -s)" = "Darwin" ] && KUBE_EFI=1
 
@@ -27,7 +29,7 @@ if [ $# -eq 0 ] ; then
     : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS}
     : ${KUBE_MEM:=$KUBE_MASTER_MEM}
     : ${KUBE_DISK:=$KUBE_MASTER_DISK}
-elif [ $# -gt 1 ] ; then
+elif [ $# -gt 1 ] || [ $# -eq 1 -a -n "${KUBE_PRESERVE_STATE}" ] ; then
     case $1 in
 	''|*[!0-9]*)
 	    echo "Node number must be a number"
@@ -57,5 +59,11 @@ else
     exit 1
 fi
 set -x
-rm -rf "${state}"
+if [ -z "${KUBE_PRESERVE_STATE}" ] ; then
+    rm -rf "${state}"
+    mkdir "${state}"
+    if [ -n "${KUBE_MAC}" ] ; then
+	echo -n "${KUBE_MAC}" > "${state}"/mac-addr
+    fi
+fi
 linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" ${uefi} "${img}${suffix}"

From 67fa0ad66291c1eda4d25487d49725f0071a13b2 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 29 Aug 2017 22:09:06 +0100
Subject: [PATCH 03/16] kubernetes: allow users to add kubelet options by
 overriding the command

For example to tell kubelet to use cri-containerd:

    command: ["/usr/bin/kubelet.sh", "--container-runtime=remote", "--container-runtime-endpoint=unix:///var/run/cri-containerd.sock"]

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kubernetes/kubelet.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh
index fb8ef2e81..ed28bc374 100755
--- a/projects/kubernetes/kubernetes/kubelet.sh
+++ b/projects/kubernetes/kubernetes/kubelet.sh
@@ -11,7 +11,8 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --enforce-node-allocatable= \
 	      --network-plugin=cni \
 	      --cni-conf-dir=/etc/cni/net.d \
-	      --cni-bin-dir=/opt/cni/bin ; do
+	      --cni-bin-dir=/opt/cni/bin \
+	      $@; do
     if [ ! -f /var/config/userdata ] ; then
 	sleep 1
     else

From 35542e493cd5d21893cdb118d557b8b0e420a3ea Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Thu, 31 Aug 2017 00:39:55 +0100
Subject: [PATCH 04/16] kubernetes: drop /rootfs from kubelet container

Since 424203cd9cf1 we can use /var/lib/cni/{etc,opt} directly.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kubernetes/Dockerfile | 2 +-
 projects/kubernetes/kubernetes/kubelet.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile
index 21f13eccc..032bfcea6 100644
--- a/projects/kubernetes/kubernetes/Dockerfile
+++ b/projects/kubernetes/kubernetes/Dockerfile
@@ -47,4 +47,4 @@ WORKDIR /
 ENTRYPOINT ["/usr/bin/kubelet.sh"]
 COPY --from=build /out /
 ENV KUBECONFIG "/etc/kubernetes/admin.conf"
-LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/rootfs/etc/cni:rshared,rbind", "/var/lib/cni/opt:/rootfs/opt/cni:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]}}'
+LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]}}'
diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh
index ed28bc374..de9a63535 100755
--- a/projects/kubernetes/kubernetes/kubelet.sh
+++ b/projects/kubernetes/kubernetes/kubelet.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
-mount --bind /opt/cni /rootfs/opt/cni
-mount --bind /etc/cni /rootfs/etc/cni
+mount --bind /opt/cni /var/lib/cni/opt
+mount --bind /etc/cni /var/lib/cni/etc
 until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --require-kubeconfig=true \
 	      --pod-manifest-path=/var/lib/kubeadm/manifests \

From 1074b4a8d4a235987a3b4c1cb89f301086fb6c67 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Wed, 13 Sep 2017 17:01:31 +0100
Subject: [PATCH 05/16] kubernetes: point kubelet at /var/lib/cni directly

Still need the /opt bind to host for weave.yaml but no need for etc any longer.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kubernetes/kubelet.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh
index de9a63535..59e80b35c 100755
--- a/projects/kubernetes/kubernetes/kubelet.sh
+++ b/projects/kubernetes/kubernetes/kubelet.sh
@@ -1,6 +1,5 @@
 #!/bin/sh
 mount --bind /opt/cni /var/lib/cni/opt
-mount --bind /etc/cni /var/lib/cni/etc
 until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --require-kubeconfig=true \
 	      --pod-manifest-path=/var/lib/kubeadm/manifests \
@@ -10,8 +9,8 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --cgroups-per-qos=false \
 	      --enforce-node-allocatable= \
 	      --network-plugin=cni \
-	      --cni-conf-dir=/etc/cni/net.d \
-	      --cni-bin-dir=/opt/cni/bin \
+	      --cni-conf-dir=/var/lib/cni/etc/net.d \
+	      --cni-bin-dir=/var/lib/cni/opt/bin \
 	      $@; do
     if [ ! -f /var/config/userdata ] ; then
 	sleep 1

From 07e4f515a4d0eb332c6ab81b3cc93b7d55e6a5d7 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Thu, 14 Sep 2017 15:55:25 +0100
Subject: [PATCH 06/16] kubernetes: add basic cni plugins dynamically

This avoids a slightly tricky sequence of nested bind mounts by just unpacking
a tarball on boot (with a stamp so it only happens once).

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kubernetes/Dockerfile | 4 +---
 projects/kubernetes/kubernetes/kubelet.sh | 6 +++++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile
index 032bfcea6..689b7b632 100644
--- a/projects/kubernetes/kubernetes/Dockerfile
+++ b/projects/kubernetes/kubernetes/Dockerfile
@@ -30,9 +30,7 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
 
 RUN rmdir /out/var/run && ln -nfs /run /out/var/run
 
-RUN curl -fSL -o /tmp/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz && \
-    mkdir -p /out/opt/cni/bin /out/etc/cni/net.d && \
-    tar -xzf /tmp/cni.tgz -C /out/opt/cni/bin
+RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz
 RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} 
 RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet
 RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm
diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh
index 59e80b35c..d42033968 100755
--- a/projects/kubernetes/kubernetes/kubelet.sh
+++ b/projects/kubernetes/kubernetes/kubelet.sh
@@ -1,5 +1,9 @@
 #!/bin/sh
-mount --bind /opt/cni /var/lib/cni/opt
+if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then
+    mkdir -p /var/lib/cni/opt/bin
+    tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin
+    touch /var/lib/cni/.opt.defaults-extracted
+fi
 until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --require-kubeconfig=true \
 	      --pod-manifest-path=/var/lib/kubeadm/manifests \

From 9397b9480a7698164f3efe7e7d0ddc2afa0a5834 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Wed, 13 Sep 2017 09:44:47 +0100
Subject: [PATCH 07/16] kubernetes: populate host CNI paths by binding from
 kubernetes service.

Kubernetes assumes (for now) that various paths are valid at the host level to
be mounted into containers, including /opt/cni and /etc/cni.

We cannot (easily) use symlinks here because the weave.yml mounts /opt and /etc
rather than /opt/cni and /etc/cni (this seems likely to be common pattern). So
if /etc/cni were a symlink to the persistent disk (under /var/lib) then it will
be dangling link within the weave container.

So add bind mounts to the runtime configuration of the kubernetes image. This
also means we must create the target mount points in the yml.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kube-master.yml       | 4 ++++
 projects/kubernetes/kube-node.yml         | 4 ++++
 projects/kubernetes/kubernetes/Dockerfile | 2 +-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml
index 3745a89c1..d1e007f9b 100644
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@@ -60,6 +60,10 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
 files:
+  - path: /opt/cni
+    directory: true
+  - path: /etc/cni
+    directory: true
   - path: root/.ssh/authorized_keys
     source: ~/.ssh/id_rsa.pub
     mode: "0600"
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
index 655314d8a..9ae19f239 100644
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@@ -58,6 +58,10 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
 files:
+  - path: /opt/cni
+    directory: true
+  - path: /etc/cni
+    directory: true
   - path: root/.ssh/authorized_keys
     source: ~/.ssh/id_rsa.pub
     mode: "0600"
diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile
index 689b7b632..2fff3e89b 100644
--- a/projects/kubernetes/kubernetes/Dockerfile
+++ b/projects/kubernetes/kubernetes/Dockerfile
@@ -45,4 +45,4 @@ WORKDIR /
 ENTRYPOINT ["/usr/bin/kubelet.sh"]
 COPY --from=build /out /
 ENV KUBECONFIG "/etc/kubernetes/admin.conf"
-LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]}}'
+LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'

From 825e43f7c416b6af70a68e69c63ed5b6e7f8a775 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 12 Sep 2017 13:21:27 +0100
Subject: [PATCH 08/16] kubernetes: bind /run with rshared,rbind

Depending on the configuration/components used the system can expect to be able
to share `/var/run/netns` (=`/run/netns` via symlink) bind mounts with other
system level containers, which requires exposing those to the host.

This doesn't appear to be needed when using Docker engine but it is with
cri-containerd.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kubernetes/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile
index 2fff3e89b..aaa9e1bd0 100644
--- a/projects/kubernetes/kubernetes/Dockerfile
+++ b/projects/kubernetes/kubernetes/Dockerfile
@@ -45,4 +45,4 @@ WORKDIR /
 ENTRYPOINT ["/usr/bin/kubelet.sh"]
 COPY --from=build /out /
 ENV KUBECONFIG "/etc/kubernetes/admin.conf"
-LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'
+LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'

From 1e15243b11182fe6bd843075952beca22b5ccc47 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 12 Sep 2017 14:27:07 +0100
Subject: [PATCH 09/16] kubernetes: Enable net.ipv4.ip_forward

This doesn't seem to be necessary when using Docker Engine as the CRI backend,
but in general it is.

The sysctl container must be writeable to allow the
/etc/sysctl.d/01-kubernetes.conf mount point to be created. See #2503.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kube-master.yml | 5 +++++
 projects/kubernetes/kube-node.yml   | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml
index d1e007f9b..73802f8d0 100644
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@@ -9,6 +9,9 @@ init:
 onboot:
   - name: sysctl
     image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051
+    binds:
+     - /etc/sysctl.d/01-kubernetes.conf:/etc/sysctl.d/01-kubernetes.conf
+    readonly: false
   - name: sysfs
     image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9
   - name: dhcpcd
@@ -60,6 +63,8 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
 files:
+  - path: /etc/sysctl.d/01-kubernetes.conf
+    contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni
     directory: true
   - path: /etc/cni
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
index 9ae19f239..d815e3e7b 100644
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@@ -9,6 +9,9 @@ init:
 onboot:
   - name: sysctl
     image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051
+    binds:
+     - /etc/sysctl.d/01-kubernetes.conf:/etc/sysctl.d/01-kubernetes.conf
+    readonly: false
   - name: sysfs
     image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9
   - name: dhcpcd
@@ -58,6 +61,8 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
 files:
+  - path: /etc/sysctl.d/01-kubernetes.conf
+    contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni
     directory: true
   - path: /etc/cni

From b6d7f769bebf68f0358253872ba18303d44319c5 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Tue, 12 Sep 2017 16:50:01 +0100
Subject: [PATCH 10/16] kubernetes: Link /etc/kubernetes to /var/lib/kubeadm

In some configurations /etc/kubernetes needs to be a valid host path.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kube-master.yml | 2 ++
 projects/kubernetes/kube-node.yml   | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml
index 73802f8d0..0f574badd 100644
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@@ -63,6 +63,8 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
 files:
+  - path: /etc/kubernetes
+    symlink: "/var/lib/kubeadm"
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
index d815e3e7b..811e97e54 100644
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@@ -61,6 +61,8 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
 files:
+  - path: /etc/kubernetes
+    symlink: "/var/lib/kubeadm"
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni

From b6fbc82e418966556f4a49e8aaeb906689c1050f Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Mon, 18 Sep 2017 10:23:33 +0100
Subject: [PATCH 11/16] kubernetes: Configuration of kubelet via file

This allows cri-containerd and docker based systems to pass the correct options
via composition of yml files, while keeping the kubelet service stanza common.

Since bind mounts are not conditional on the presence of the source we need to
create an empty file in the docker case.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kube-master.yml       | 2 ++
 projects/kubernetes/kube-node.yml         | 2 ++
 projects/kubernetes/kubernetes/Dockerfile | 2 +-
 projects/kubernetes/kubernetes/kubelet.sh | 5 ++++-
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml
index 0f574badd..1f944580d 100644
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@@ -65,6 +65,8 @@ services:
 files:
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
+  - path: /etc/kubelet.conf
+    contents: ""
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
index 811e97e54..c0410ba54 100644
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@@ -63,6 +63,8 @@ services:
 files:
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
+  - path: /etc/kubelet.conf
+    contents: ""
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni
diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile
index aaa9e1bd0..a6cafa178 100644
--- a/projects/kubernetes/kubernetes/Dockerfile
+++ b/projects/kubernetes/kubernetes/Dockerfile
@@ -45,4 +45,4 @@ WORKDIR /
 ENTRYPOINT ["/usr/bin/kubelet.sh"]
 COPY --from=build /out /
 ENV KUBECONFIG "/etc/kubernetes/admin.conf"
-LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'
+LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'
diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh
index d42033968..f58067802 100755
--- a/projects/kubernetes/kubernetes/kubelet.sh
+++ b/projects/kubernetes/kubernetes/kubelet.sh
@@ -4,6 +4,9 @@ if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then
     tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin
     touch /var/lib/cni/.opt.defaults-extracted
 fi
+if [ -e /etc/kubelet.conf ] ; then
+    . /etc/kubelet.conf
+fi
 until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --require-kubeconfig=true \
 	      --pod-manifest-path=/var/lib/kubeadm/manifests \
@@ -15,7 +18,7 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --network-plugin=cni \
 	      --cni-conf-dir=/var/lib/cni/etc/net.d \
 	      --cni-bin-dir=/var/lib/cni/opt/bin \
-	      $@; do
+	      $KUBELET_ARGS $@; do
     if [ ! -f /var/config/userdata ] ; then
 	sleep 1
     else

From 80dff9fe99c9f7fd0639833c1e316dcabd1443bd Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Wed, 13 Sep 2017 14:28:56 +0100
Subject: [PATCH 12/16] kubernetes: update yml

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kube-master.yml | 2 +-
 projects/kubernetes/kube-node.yml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml
index 1f944580d..d8e1b2f15 100644
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@@ -61,7 +61,7 @@ services:
   - name: kubernetes-image-cache-control-plane
     image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
   - name: kubelet
-    image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
+    image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
 files:
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
index c0410ba54..efc866fb4 100644
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@@ -59,7 +59,7 @@ services:
   - name: kubernetes-image-cache-common
     image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
   - name: kubelet
-    image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
+    image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
 files:
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"

From 445dcc0ac2a11b35d9d50a96d614158ff01225a0 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Fri, 11 Aug 2017 11:06:32 +0100
Subject: [PATCH 13/16] kubernetes: Add cri-containerd package

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/cri-containerd/Dockerfile | 49 +++++++++++++++++++
 projects/kubernetes/cri-containerd/Makefile   |  7 +++
 2 files changed, 56 insertions(+)
 create mode 100644 projects/kubernetes/cri-containerd/Dockerfile
 create mode 100644 projects/kubernetes/cri-containerd/Makefile

diff --git a/projects/kubernetes/cri-containerd/Dockerfile b/projects/kubernetes/cri-containerd/Dockerfile
new file mode 100644
index 000000000..707642370
--- /dev/null
+++ b/projects/kubernetes/cri-containerd/Dockerfile
@@ -0,0 +1,49 @@
+FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build
+
+RUN \
+  apk add \
+  bash \
+  gcc \
+  git \
+  go \
+  libc-dev \
+  make \
+  && true
+ENV GOPATH=/go PATH=$PATH:/go/bin
+
+ENV CRI_CONTAINERD_URL https://github.com/kubernetes-incubator/cri-containerd.git
+#ENV CRI_CONTAINERD_BRANCH pull/NNN/head
+ENV CRI_CONTAINERD_COMMIT a8d49402859167a232b094d971e70c2f4b71b8ea
+RUN mkdir -p $GOPATH/src/github.com/kubernetes-incubator && \
+    cd $GOPATH/src/github.com/kubernetes-incubator && \
+    git clone $CRI_CONTAINERD_URL cri-containerd
+WORKDIR $GOPATH/src/github.com/kubernetes-incubator/cri-containerd
+RUN set -e; \
+    if [ -n "$CRI_CONTAINERD_BRANCH" ] ; then \
+        git fetch origin "$CRI_CONTAINERD_BRANCH"; \
+    fi; \
+    git checkout $CRI_CONTAINERD_COMMIT
+RUN make static-binaries
+
+RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
+# util-linux because a full ns-enter is required.
+# example commands: /usr/bin/nsenter --net= -F -- <ip commandline>
+#                   /usr/bin/nsenter --net=/var/run/netns/cni-5e8acebe-810d-c1b9-ced0-47be2f312fa8 -F -- <ip commandline>
+# NB the first ("--net=") is actually not valid -- see https://github.com/kubernetes-incubator/cri-containerd/issues/245
+RUN apk add --no-cache --initdb -p /out \
+    alpine-baselayout \
+    busybox \
+    ca-certificates \
+    iptables \
+    util-linux \
+    && true
+# Remove apk residuals. We have a read-only rootfs, so apk is of no use.
+RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
+
+RUN make DESTDIR=/out install
+
+FROM scratch
+WORKDIR /
+ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/var/lib/cni/opt/bin", "--network-conf-dir", "/var/lib/cni/etc/net.d"]
+COPY --from=build /out /
+LABEL org.mobyproject.config='{"binds": ["/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/tmp:/tmp", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/etc/cni:rshared,rbind", "/var/lib/cni/opt:/opt/cni:rshared,rbind", "/run/containerd/containerd.sock:/run/containerd/containerd.sock"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc/net.d", "/var/lib/cni/opt"]}}'
diff --git a/projects/kubernetes/cri-containerd/Makefile b/projects/kubernetes/cri-containerd/Makefile
new file mode 100644
index 000000000..fe0253576
--- /dev/null
+++ b/projects/kubernetes/cri-containerd/Makefile
@@ -0,0 +1,7 @@
+ORG?=linuxkitprojects
+IMAGE=cri-containerd
+NETWORK=1
+NOTRUST=1
+ARCHES=x86_64
+
+include ../../../pkg/package.mk

From 1fa059bbd2c407d164c2b0ad577ebac786136d28 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Thu, 14 Sep 2017 15:48:18 +0100
Subject: [PATCH 14/16] kubernetes: allow selection of runtime engine (docker
 vs cri-containerd)

Currently this is at build time

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/Makefile                  | 10 ++++---
 projects/kubernetes/README.md                 |  5 ++++
 projects/kubernetes/cri-containerd-master.yml |  0
 projects/kubernetes/cri-containerd.yml        |  7 +++++
 projects/kubernetes/docker-master.yml         |  3 +++
 projects/kubernetes/docker.yml                | 27 +++++++++++++++++++
 projects/kubernetes/kube-master.yml           | 27 -------------------
 projects/kubernetes/kube-node.yml             | 25 -----------------
 8 files changed, 48 insertions(+), 56 deletions(-)
 create mode 100644 projects/kubernetes/cri-containerd-master.yml
 create mode 100644 projects/kubernetes/cri-containerd.yml
 create mode 100644 projects/kubernetes/docker-master.yml
 create mode 100644 projects/kubernetes/docker.yml

diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile
index c9e5da86d..8862f2868 100644
--- a/projects/kubernetes/Makefile
+++ b/projects/kubernetes/Makefile
@@ -1,3 +1,5 @@
+KUBE_RUNTIME ?= docker
+
 all: tag-container-images build-vm-images
 
 tag-container-images:
@@ -12,11 +14,11 @@ push-container-images:
 
 build-vm-images: kube-master.iso kube-node.iso
 
-kube-master.iso: kube-master.yml
-	moby build -name kube-master -format iso-efi -format iso-bios kube-master.yml
+kube-master.iso: kube-master.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
+	moby build -name kube-master -format iso-efi -format iso-bios kube-master.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
 
-kube-node.iso: kube-node.yml
-	moby build -name kube-node -format iso-efi -format iso-bios kube-node.yml
+kube-node.iso: kube-node.yml $(KUBE_RUNTIME).yml
+	moby build -name kube-node -format iso-efi -format iso-bios kube-node.yml $(KUBE_RUNTIME).yml
 
 clean:
 	rm -f -r \
diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md
index 536b65402..3a1c6300d 100644
--- a/projects/kubernetes/README.md
+++ b/projects/kubernetes/README.md
@@ -11,6 +11,11 @@ Build OS images:
 make build-vm-images
 ```
 
+By default this will build images using Docker Engine for execution. To instead use cri-containerd use:
+```
+make build-vm-images KUBE_RUNTIME=cri-containerd
+```
+
 Boot Kubernetes master OS image using `hyperkit` on macOS: or `qemu` on Linux:
 ```
 ./boot.sh
diff --git a/projects/kubernetes/cri-containerd-master.yml b/projects/kubernetes/cri-containerd-master.yml
new file mode 100644
index 000000000..e69de29bb
diff --git a/projects/kubernetes/cri-containerd.yml b/projects/kubernetes/cri-containerd.yml
new file mode 100644
index 000000000..a3520ef5d
--- /dev/null
+++ b/projects/kubernetes/cri-containerd.yml
@@ -0,0 +1,7 @@
+services:
+  - name: cri-containerd
+    image: linuxkitprojects/cri-containerd:b8b6a48426c2165055534b06fb0119f07e24506a
+files:
+  - path: /etc/kubelet.conf
+    contents: |
+      KUBELET_ARGS="--container-runtime=remote --container-runtime-endpoint=unix:///var/run/cri-containerd.sock"
diff --git a/projects/kubernetes/docker-master.yml b/projects/kubernetes/docker-master.yml
new file mode 100644
index 000000000..ec6298647
--- /dev/null
+++ b/projects/kubernetes/docker-master.yml
@@ -0,0 +1,3 @@
+services:
+  - name: kubernetes-image-cache-control-plane
+    image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
diff --git a/projects/kubernetes/docker.yml b/projects/kubernetes/docker.yml
new file mode 100644
index 000000000..03388d91b
--- /dev/null
+++ b/projects/kubernetes/docker.yml
@@ -0,0 +1,27 @@
+services:
+  - name: docker
+    image: docker:17.07.0-ce-dind
+    capabilities:
+     - all
+    pid: host
+    mounts:
+     - type: cgroup
+       options: ["rw","nosuid","noexec","nodev","relatime"]
+    binds:
+     - /dev:/dev
+     - /etc/resolv.conf:/etc/resolv.conf
+     - /lib/modules:/lib/modules
+     - /run:/run
+     - /var:/var:rshared,rbind
+     - /var/lib/kubeadm:/etc/kubernetes
+     - /var/lib/cni/etc:/etc/cni:rshared,rbind
+     - /var/lib/cni/opt:/opt/cni:rshared,rbind
+    rootfsPropagation: shared
+    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
+    runtime:
+      mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]
+  - name: kubernetes-image-cache-common
+    image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
+files:
+  - path: /etc/kubelet.conf
+    contents: ""
diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml
index d8e1b2f15..a561be679 100644
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@@ -35,38 +35,11 @@ services:
     image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67
   - name: sshd
     image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0
-  - name: docker
-    image: docker:17.07.0-ce-dind
-    capabilities:
-     - all
-    pid: host
-    mounts:
-     - type: cgroup
-       options: ["rw","nosuid","noexec","nodev","relatime"]
-    binds:
-     - /dev:/dev
-     - /etc/resolv.conf:/etc/resolv.conf
-     - /lib/modules:/lib/modules
-     - /run:/run
-     - /var:/var:rshared,rbind
-     - /var/lib/kubeadm:/etc/kubernetes
-     - /var/lib/cni/etc:/etc/cni:rshared,rbind
-     - /var/lib/cni/opt:/opt/cni:rshared,rbind
-    rootfsPropagation: shared
-    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
-    runtime:
-      mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]
-  - name: kubernetes-image-cache-common
-    image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
-  - name: kubernetes-image-cache-control-plane
-    image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
   - name: kubelet
     image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
 files:
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
-  - path: /etc/kubelet.conf
-    contents: ""
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
index efc866fb4..a561be679 100644
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@@ -35,36 +35,11 @@ services:
     image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67
   - name: sshd
     image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0
-  - name: docker
-    image: docker:17.07.0-ce-dind
-    capabilities:
-     - all
-    pid: host
-    mounts:
-     - type: cgroup
-       options: ["rw","nosuid","noexec","nodev","relatime"]
-    binds:
-     - /dev:/dev
-     - /etc/resolv.conf:/etc/resolv.conf
-     - /lib/modules:/lib/modules
-     - /run:/run
-     - /var:/var:rshared,rbind
-     - /var/lib/kubeadm:/etc/kubernetes
-     - /var/lib/cni/etc:/etc/cni:rshared,rbind
-     - /var/lib/cni/opt:/opt/cni:rshared,rbind
-    rootfsPropagation: shared
-    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
-    runtime:
-      mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]
-  - name: kubernetes-image-cache-common
-    image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
   - name: kubelet
     image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
 files:
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
-  - path: /etc/kubelet.conf
-    contents: ""
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni

From 7e19d388212bcd27a7712b811e8c6bf164afa0fc Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Thu, 14 Sep 2017 16:45:33 +0100
Subject: [PATCH 15/16] kubernetes: use a common base kube yml file

With the master tailoring for docker now being in docker-master.yml,
kube-master and kube-node are identical, so just use a single kube.yml.

The reference to kube-master.yml in README.md is obsolete, so just drop it.

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/Makefile                  |  8 +--
 projects/kubernetes/README.md                 |  2 -
 projects/kubernetes/kube-node.yml             | 52 -------------------
 .../kubernetes/{kube-master.yml => kube.yml}  |  0
 4 files changed, 4 insertions(+), 58 deletions(-)
 delete mode 100644 projects/kubernetes/kube-node.yml
 rename projects/kubernetes/{kube-master.yml => kube.yml} (100%)

diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile
index 8862f2868..034e519fd 100644
--- a/projects/kubernetes/Makefile
+++ b/projects/kubernetes/Makefile
@@ -14,11 +14,11 @@ push-container-images:
 
 build-vm-images: kube-master.iso kube-node.iso
 
-kube-master.iso: kube-master.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
-	moby build -name kube-master -format iso-efi -format iso-bios kube-master.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
+kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
+	moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
 
-kube-node.iso: kube-node.yml $(KUBE_RUNTIME).yml
-	moby build -name kube-node -format iso-efi -format iso-bios kube-node.yml $(KUBE_RUNTIME).yml
+kube-node.iso: kube.yml $(KUBE_RUNTIME).yml
+	moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml
 
 clean:
 	rm -f -r \
diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md
index 3a1c6300d..d5e3a13a3 100644
--- a/projects/kubernetes/README.md
+++ b/projects/kubernetes/README.md
@@ -4,8 +4,6 @@ This project aims to demonstrate how one can create minimal and immutable Kubern
 
 Make sure to `cd projects/kubernetes` first.
 
-Edit `kube-master.yml` and add your public SSH key to `files` section.
-
 Build OS images:
 ```
 make build-vm-images
diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml
deleted file mode 100644
index a561be679..000000000
--- a/projects/kubernetes/kube-node.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-kernel:
-  image: linuxkit/kernel:4.9.50
-  cmdline: "console=tty0 console=ttyS0"
-init:
-  - linuxkit/init:851e9c3ad0574d640b733b92fdb26c368d2f7f8f
-  - linuxkit/runc:a1b564248a0d0b118c11e61db9f84ecf41dd2d2a
-  - linuxkit/containerd:06876ceef325e49e9ba119659357768d5df89075
-  - linuxkit/ca-certificates:e44b0a66df5a102c0e220f0066b0d904710dcb10
-onboot:
-  - name: sysctl
-    image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051
-    binds:
-     - /etc/sysctl.d/01-kubernetes.conf:/etc/sysctl.d/01-kubernetes.conf
-    readonly: false
-  - name: sysfs
-    image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9
-  - name: dhcpcd
-    image: linuxkit/dhcpcd:f3f5413abb78fae9020e35bd4788fa93df4530b7
-    command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
-  - name: metadata
-    image: linuxkit/metadata:da3138079c168e0c5608d8f3853366c113ed91d2
-  - name: format
-    image: linuxkit/format:158d992b7bf7ab984100c697d7e72161ea7d7382
-  - name: mounts
-    image: linuxkit/mount:4fe245efb01384e42622c36302e13e386bbaeb08
-    command: ["/usr/bin/mountie", "/var/lib/"]
-services:
-  - name: getty
-    image: linuxkit/getty:797cb79e0a229fcd16ebf44a0da74bcec03968ec
-    env:
-     - INSECURE=true
-  - name: rngd
-    image: linuxkit/rngd:558e86a36242bb74353bc9287b715ddb8567357e
-  - name: ntpd
-    image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67
-  - name: sshd
-    image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0
-  - name: kubelet
-    image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
-files:
-  - path: /etc/kubernetes
-    symlink: "/var/lib/kubeadm"
-  - path: /etc/sysctl.d/01-kubernetes.conf
-    contents: 'net.ipv4.ip_forward = 1'
-  - path: /opt/cni
-    directory: true
-  - path: /etc/cni
-    directory: true
-  - path: root/.ssh/authorized_keys
-    source: ~/.ssh/id_rsa.pub
-    mode: "0600"
-    optional: true
diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube.yml
similarity index 100%
rename from projects/kubernetes/kube-master.yml
rename to projects/kubernetes/kube.yml

From 5ab6c03a08bde70fbb61323601a21d82c4a62c77 Mon Sep 17 00:00:00 2001
From: Ian Campbell <ijc@docker.com>
Date: Thu, 14 Sep 2017 17:19:33 +0100
Subject: [PATCH 16/16] kubernetes: Add yaml metadata

Signed-off-by: Ian Campbell <ijc@docker.com>
---
 projects/kubernetes/kube.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml
index a561be679..924648234 100644
--- a/projects/kubernetes/kube.yml
+++ b/projects/kubernetes/kube.yml
@@ -38,6 +38,8 @@ services:
   - name: kubelet
     image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
 files:
+  - path: etc/linuxkit.yml
+    metadata: yaml
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
   - path: /etc/sysctl.d/01-kubernetes.conf