kubernetes: rework kubelet and kubeadm start of day interations

Rework the kubelet.sh script by adding an explicit step which waits for the
configuration to be valid, either by finding appropriate metadata or by waiting
explicitly for kubelet.conf to be created (e.g. by kubeadm) before launching
kubelet. The previous construct was implicitly waiting for kubelet.conf to be
created since kubelet fails if that file is not present.

Pull the set of start of day yaml files to be applied (currently just weave)
out of the kubelet image and into the LinuxKit yaml by providing a directory
which is searched for *.yaml after init.

Signed-off-by: Ian Campbell <ijc@docker.com>

projects/kubernetes/Makefile

@@ -1,5 +1,10 @@
 KUBE_RUNTIME ?= docker
 
+WEAVE_VERSION := v2.0.4
+
+INIT_YAML ?=
+INIT_YAML += weave.yaml
+
 all: tag-container-images build-vm-images
 
 tag-container-images:
@@ -14,11 +19,15 @@ push-container-images:
 
 build-vm-images: kube-master.iso kube-node.iso
 
-kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
+# NB cannot use $^ because $(INIT_YAML) is not for consumption by "moby build"
+kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml $(INIT_YAML)
 	moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
 
 kube-node.iso: kube.yml $(KUBE_RUNTIME).yml
-	moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml
+	moby build -name kube-node -format iso-efi -format iso-bios $^
+
+weave.yaml:
+	curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$(WEAVE_VERSION)
 
 clean:
 	rm -f -r \

projects/kubernetes/kube.yml

@@ -42,6 +42,8 @@ files:
     metadata: yaml
   - path: /etc/kubernetes
     symlink: "/var/lib/kubeadm"
+  - path: /etc/kubeadm/kube-system.init/01-weave.yaml
+    source: weave.yaml
   - path: /etc/sysctl.d/01-kubernetes.conf
     contents: 'net.ipv4.ip_forward = 1'
   - path: /opt/cni

projects/kubernetes/kubernetes/Dockerfile

@@ -1,7 +1,6 @@
 FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build
 
 ENV kubernetes_version v1.7.6
-ENV weave_version      v2.0.4
 ENV cni_version        v0.6.0
 
 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64"
@@ -31,7 +30,6 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
 RUN rmdir /out/var/run && ln -nfs /run /out/var/run
 
 RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/plugins/releases/download/${cni_version}/cni-plugins-amd64-${cni_version}.tgz
-RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} 
 RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet
 RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm
 RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl
@@ -45,4 +43,4 @@ WORKDIR /
 ENTRYPOINT ["/usr/bin/kubelet.sh"]
 COPY --from=build /out /
 ENV KUBECONFIG "/etc/kubernetes/admin.conf"
-LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'
+LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf", "/etc/kubeadm:/etc/kubeadm"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'

projects/kubernetes/kubernetes/kubeadm-init.sh

@@ -1,4 +1,9 @@
 #!/bin/sh
 set -e
 kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@
-kubectl create -n kube-system -f /etc/weave.yaml
+for i in /etc/kubeadm/kube-system.init/*.yaml ; do
+    if [ -e "$i" ] ; then
+	echo "Applying "$(basename "$i")
+	kubectl create -n kube-system -f "$i"
+    fi
+done

projects/kubernetes/kubernetes/kubelet.sh

@@ -1,4 +1,7 @@
 #!/bin/sh
+# Kubelet outputs only to stderr, so arrange for everything we do to go there too
+exec 1>&2
+
 if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then
     mkdir -p /var/lib/cni/opt/bin
     tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin
@@ -7,7 +10,27 @@ fi
 if [ -e /etc/kubelet.conf ] ; then
     . /etc/kubelet.conf
 fi
-until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
+if [ -e /var/config/userdata ] ; then
+    echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\""
+    kubeadm join --skip-preflight-checks $(cat /var/config/userdata)
+fi
+
+conf=/var/lib/kubeadm/kubelet.conf
+
+echo "kubelet.sh: waiting for ${conf}"
+# TODO(ijc) is there a race between kubeadm creating this file and
+# finishing the write where we might be able to fall through and
+# start kubelet with an incomplete configuration file? I've tried
+# to provoke such a race without success. An explicit
+# synchronisation barrier or changing kubeadm to write
+# kubelet.conf atomically might be good in any case.
+until [ -f "${conf}" ] ; do
+    sleep 1
+done
+
+echo "kubelet.sh: ${conf} has arrived" 2>&1
+
+exec kubelet --kubeconfig=${conf} \
 	      --require-kubeconfig=true \
 	      --pod-manifest-path=/var/lib/kubeadm/manifests \
 	      --allow-privileged=true \
@@ -18,10 +41,4 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --network-plugin=cni \
 	      --cni-conf-dir=/var/lib/cni/etc/net.d \
 	      --cni-bin-dir=/var/lib/cni/opt/bin \
-	      $KUBELET_ARGS $@; do
-    if [ ! -f /var/config/userdata ] ; then
-	sleep 1
-    else
-	kubeadm join --skip-preflight-checks $(cat /var/config/userdata)
-    fi
-done
+	      $KUBELET_ARGS $@