diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 034e519fd..8b8c6d19e 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,5 +1,10 @@ KUBE_RUNTIME ?= docker +WEAVE_VERSION := v2.0.4 + +INIT_YAML ?= +INIT_YAML += weave.yaml + all: tag-container-images build-vm-images tag-container-images: @@ -14,11 +19,15 @@ push-container-images: build-vm-images: kube-master.iso kube-node.iso -kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml +# NB cannot use $^ because $(INIT_YAML) is not for consumption by "moby build" +kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml $(INIT_YAML) moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml kube-node.iso: kube.yml $(KUBE_RUNTIME).yml - moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml + moby build -name kube-node -format iso-efi -format iso-bios $^ + +weave.yaml: + curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$(WEAVE_VERSION) clean: rm -f -r \ diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 27cbbb305..a4850c15c 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -42,6 +42,8 @@ files: metadata: yaml - path: /etc/kubernetes symlink: "/var/lib/kubeadm" + - path: /etc/kubeadm/kube-system.init/01-weave.yaml + source: weave.yaml - path: /etc/sysctl.d/01-kubernetes.conf contents: 'net.ipv4.ip_forward = 1' - path: /opt/cni diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index fb0d293b0..9d3be0909 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -1,7 +1,6 @@ FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build ENV kubernetes_version v1.7.6 -ENV weave_version v2.0.4 ENV cni_version v0.6.0 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" @@ -31,7 +30,6 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache RUN rmdir /out/var/run && ln -nfs /run /out/var/run RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/plugins/releases/download/${cni_version}/cni-plugins-amd64-${cni_version}.tgz -RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl @@ -45,4 +43,4 @@ WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] COPY --from=build /out / ENV KUBECONFIG "/etc/kubernetes/admin.conf" -LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf", "/etc/kubeadm:/etc/kubeadm"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 1229dae64..8c66f85d1 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,4 +1,9 @@ #!/bin/sh set -e kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ -kubectl create -n kube-system -f /etc/weave.yaml +for i in /etc/kubeadm/kube-system.init/*.yaml ; do + if [ -e "$i" ] ; then + echo "Applying "$(basename "$i") + kubectl create -n kube-system -f "$i" + fi +done diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index f58067802..fd044b266 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Kubelet outputs only to stderr, so arrange for everything we do to go there too +exec 1>&2 + if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then mkdir -p /var/lib/cni/opt/bin tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin @@ -7,7 +10,27 @@ fi if [ -e /etc/kubelet.conf ] ; then . /etc/kubelet.conf fi -until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ +if [ -e /var/config/userdata ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" + kubeadm join --skip-preflight-checks $(cat /var/config/userdata) +fi + +conf=/var/lib/kubeadm/kubelet.conf + +echo "kubelet.sh: waiting for ${conf}" +# TODO(ijc) is there a race between kubeadm creating this file and +# finishing the write where we might be able to fall through and +# start kubelet with an incomplete configuration file? I've tried +# to provoke such a race without success. An explicit +# synchronisation barrier or changing kubeadm to write +# kubelet.conf atomically might be good in any case. +until [ -f "${conf}" ] ; do + sleep 1 +done + +echo "kubelet.sh: ${conf} has arrived" 2>&1 + +exec kubelet --kubeconfig=${conf} \ --require-kubeconfig=true \ --pod-manifest-path=/var/lib/kubeadm/manifests \ --allow-privileged=true \ @@ -18,10 +41,4 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ --network-plugin=cni \ --cni-conf-dir=/var/lib/cni/etc/net.d \ --cni-bin-dir=/var/lib/cni/opt/bin \ - $KUBELET_ARGS $@; do - if [ ! -f /var/config/userdata ] ; then - sleep 1 - else - kubeadm join --skip-preflight-checks $(cat /var/config/userdata) - fi -done + $KUBELET_ARGS $@