diff --git a/projects/kubernetes/.gitignore b/projects/kubernetes/.gitignore index 87935610d..6af0a22b7 100644 --- a/projects/kubernetes/.gitignore +++ b/projects/kubernetes/.gitignore @@ -2,3 +2,5 @@ image-cache/common/*.tar image-cache/common/Dockerfile image-cache/control-plane/*.tar image-cache/control-plane/Dockerfile +weave.yaml +network.yaml diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 034e519fd..a17d706a2 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,4 +1,8 @@ KUBE_RUNTIME ?= docker +NETWORK ?= weave-v2.0.4 + +INIT_YAML ?= +INIT_YAML += network.yaml all: tag-container-images build-vm-images @@ -14,13 +18,21 @@ push-container-images: build-vm-images: kube-master.iso kube-node.iso -kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml +# NB cannot use $^ because $(INIT_YAML) is not for consumption by "moby build" +kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml $(INIT_YAML) moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml kube-node.iso: kube.yml $(KUBE_RUNTIME).yml - moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml + moby build -name kube-node -format iso-efi -format iso-bios $^ + +network.yaml: $(NETWORK).yaml + ln -nf $< $@ + +weave-%.yaml: + curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$* clean: rm -f -r \ - kube-*-kernel kube-*-cmdline kube-*-state kube-*-initrd.img *.iso + kube-*-kernel kube-*-cmdline kube-*-state kube-*-initrd.img *.iso \ + weave-*.yaml network.yaml $(MAKE) -C image-cache clean diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 3b041aef4..5be099635 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -23,7 +23,14 @@ suffix=".iso" if [ $# -eq 0 ] ; then img="kube-master" - data="" + # If $KUBE_MASTER_AUTOINIT is set, including if it is set to "" + # then we configure for auto init. If it is completely unset then + # we do not. + if [ -n "${KUBE_MASTER_AUTOINIT+x}" ] ; then + data="{\"kubeadm\": {\"init\": \"${KUBE_MASTER_AUTOINIT}\"} }" + else + data="" + fi state="kube-master-state" : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS} @@ -44,7 +51,7 @@ elif [ $# -gt 1 ] || [ $# -eq 1 -a -n "${KUBE_PRESERVE_STATE}" ] ; then img="kube-node" name="node-${1}" shift - data="${*}" + data="{\"kubeadm\": {\"join\": \"${*}\"} }" state="kube-${name}-state" : ${KUBE_VCPUS:=$KUBE_NODE_VCPUS} diff --git a/projects/kubernetes/cri-containerd.yml b/projects/kubernetes/cri-containerd.yml index 4e98b58e8..637857e1b 100644 --- a/projects/kubernetes/cri-containerd.yml +++ b/projects/kubernetes/cri-containerd.yml @@ -1,6 +1,6 @@ services: - name: cri-containerd - image: linuxkitprojects/cri-containerd:4d159d37996954dfdc78a6d07a324fd872a019d5 + image: linuxkitprojects/cri-containerd:2ff7dce33400a4d184976ca439725d8306295f1a files: - path: /etc/kubelet.conf contents: | diff --git a/projects/kubernetes/cri-containerd/Dockerfile b/projects/kubernetes/cri-containerd/Dockerfile index 0387f3a3e..38ba34636 100644 --- a/projects/kubernetes/cri-containerd/Dockerfile +++ b/projects/kubernetes/cri-containerd/Dockerfile @@ -13,7 +13,7 @@ ENV GOPATH=/go PATH=$PATH:/go/bin ENV CRI_CONTAINERD_URL https://github.com/kubernetes-incubator/cri-containerd.git #ENV CRI_CONTAINERD_BRANCH pull/NNN/head -ENV CRI_CONTAINERD_COMMIT 437131299b805de9d1431300d8054af61fa54a26 +ENV CRI_CONTAINERD_COMMIT a2dbc6ec1ce63fe8c54543c04df0a1a45abdd989 RUN mkdir -p $GOPATH/src/github.com/kubernetes-incubator && \ cd $GOPATH/src/github.com/kubernetes-incubator && \ git clone $CRI_CONTAINERD_URL cri-containerd diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 27cbbb305..fd0e1f450 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -36,12 +36,14 @@ services: - name: sshd image: linuxkit/sshd:d313eea3d9d7fbcbc927d06a6700325725db2a82 - name: kubelet - image: linuxkitprojects/kubernetes:8f3f75005ce56b8588b669ce5d7b02c5d705bbd4 + image: linuxkitprojects/kubernetes:4eba50ea1fae6f881c65429b9c21afadcdeec853 files: - path: etc/linuxkit.yml metadata: yaml - path: /etc/kubernetes symlink: "/var/lib/kubeadm" + - path: /etc/kubeadm/kube-system.init/50-network.yaml + source: network.yaml - path: /etc/sysctl.d/01-kubernetes.conf contents: 'net.ipv4.ip_forward = 1' - path: /opt/cni diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index fb0d293b0..09e499ee6 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -1,7 +1,6 @@ -FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build +FROM linuxkit/alpine:79987c65c66700171c073151c1d3f0372597bec2 AS build ENV kubernetes_version v1.7.6 -ENV weave_version v2.0.4 ENV cni_version v0.6.0 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" @@ -11,10 +10,14 @@ RUN apk add -U --no-cache \ && true RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +#coreutils needed for du -B for disk image checks made by kubelet +# example: $ du -s -B 1 /var/lib/kubelet/pods/... +# du: unrecognized option: B RUN apk add --no-cache --initdb -p /out \ alpine-baselayout \ busybox \ ca-certificates \ + coreutils \ curl \ ebtables \ ethtool \ @@ -31,7 +34,6 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache RUN rmdir /out/var/run && ln -nfs /run /out/var/run RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/plugins/releases/download/${cni_version}/cni-plugins-amd64-${cni_version}.tgz -RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl @@ -45,4 +47,4 @@ WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] COPY --from=build /out / ENV KUBECONFIG "/etc/kubernetes/admin.conf" -LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf", "/etc/kubeadm:/etc/kubeadm"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 1229dae64..89403fee1 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,4 +1,9 @@ #!/bin/sh set -e -kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ -kubectl create -n kube-system -f /etc/weave.yaml +kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ $@ +for i in /etc/kubeadm/kube-system.init/*.yaml ; do + if [ -e "$i" ] ; then + echo "Applying "$(basename "$i") + kubectl create -n kube-system -f "$i" + fi +done diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index f58067802..65df80a84 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Kubelet outputs only to stderr, so arrange for everything we do to go there too +exec 1>&2 + if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then mkdir -p /var/lib/cni/opt/bin tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin @@ -7,7 +10,35 @@ fi if [ -e /etc/kubelet.conf ] ; then . /etc/kubelet.conf fi -until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ +if [ -e /var/config/kubeadm/init ] ; then + echo "kubelet.sh: init cluster with metadata \"$(cat /var/config/kubeadm/init)\"" + # This needs to be in the background since it waits for kubelet to start. + # We skip printing the token so it is not persisted in the log. + kubeadm-init.sh --skip-token-print $(cat /var/config/kubeadm/init) & +elif [ -e /var/config/kubeadm/join ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/kubeadm/join)\"" + kubeadm join --skip-preflight-checks $(cat /var/config/kubeadm/join) +elif [ -e /var/config/userdata ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" + kubeadm join --skip-preflight-checks $(cat /var/config/userdata) +fi + +conf=/var/lib/kubeadm/kubelet.conf + +echo "kubelet.sh: waiting for ${conf}" +# TODO(ijc) is there a race between kubeadm creating this file and +# finishing the write where we might be able to fall through and +# start kubelet with an incomplete configuration file? I've tried +# to provoke such a race without success. An explicit +# synchronisation barrier or changing kubeadm to write +# kubelet.conf atomically might be good in any case. +until [ -f "${conf}" ] ; do + sleep 1 +done + +echo "kubelet.sh: ${conf} has arrived" 2>&1 + +exec kubelet --kubeconfig=${conf} \ --require-kubeconfig=true \ --pod-manifest-path=/var/lib/kubeadm/manifests \ --allow-privileged=true \ @@ -18,10 +49,4 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ --network-plugin=cni \ --cni-conf-dir=/var/lib/cni/etc/net.d \ --cni-bin-dir=/var/lib/cni/opt/bin \ - $KUBELET_ARGS $@; do - if [ ! -f /var/config/userdata ] ; then - sleep 1 - else - kubeadm join --skip-preflight-checks $(cat /var/config/userdata) - fi -done + $KUBELET_ARGS $@ diff --git a/tools/alpine/packages b/tools/alpine/packages index 93ca88a4b..8b99d3998 100644 --- a/tools/alpine/packages +++ b/tools/alpine/packages @@ -19,6 +19,7 @@ busybox-initscripts ca-certificates cdrkit cmake +coreutils cryptsetup curl dhcpcd diff --git a/tools/alpine/versions.aarch64 b/tools/alpine/versions.aarch64 index 73268a0a6..163d440c1 100644 --- a/tools/alpine/versions.aarch64 +++ b/tools/alpine/versions.aarch64 @@ -1,4 +1,4 @@ -# linuxkit/alpine:02b8eb30ca2e3ef16e756b017258fb90e34fb7db-arm64 +# linuxkit/alpine:415e9417004bbd8940a4c2303195b20a5dbd8c9a-arm64 # automatically generated list of installed packages abuild-3.0.0_rc2-r8 alpine-baselayout-3.0.4-r0 @@ -28,6 +28,7 @@ ca-certificates-20161130-r2 cdrkit-1.1.11-r2 celt051-0.5.1.3-r0 cmake-3.8.1-r0 +coreutils-8.27-r0 cryptsetup-1.7.5-r0 cryptsetup-libs-1.7.5-r0 curl-7.55.0-r0 diff --git a/tools/alpine/versions.x86_64 b/tools/alpine/versions.x86_64 index f4ff7c1b0..b202a4abb 100644 --- a/tools/alpine/versions.x86_64 +++ b/tools/alpine/versions.x86_64 @@ -1,4 +1,4 @@ -# linuxkit/alpine:f2f4db272c910d136380781a97e475013fabda8b-amd64 +# linuxkit/alpine:79987c65c66700171c073151c1d3f0372597bec2-amd64 # automatically generated list of installed packages abuild-3.0.0_rc2-r8 alpine-baselayout-3.0.4-r0 @@ -29,6 +29,7 @@ ca-certificates-20161130-r2 cdrkit-1.1.11-r2 celt051-0.5.1.3-r0 cmake-3.8.1-r0 +coreutils-8.27-r0 cryptsetup-1.7.5-r0 cryptsetup-libs-1.7.5-r0 curl-7.55.0-r0