From dea2e1df668398810c73234437f2392b959d562f Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Sep 2017 13:34:46 +0100 Subject: [PATCH 1/7] alpine: add coreutils Signed-off-by: Ian Campbell --- tools/alpine/packages | 1 + tools/alpine/versions.aarch64 | 3 ++- tools/alpine/versions.x86_64 | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/alpine/packages b/tools/alpine/packages index 93ca88a4b..8b99d3998 100644 --- a/tools/alpine/packages +++ b/tools/alpine/packages @@ -19,6 +19,7 @@ busybox-initscripts ca-certificates cdrkit cmake +coreutils cryptsetup curl dhcpcd diff --git a/tools/alpine/versions.aarch64 b/tools/alpine/versions.aarch64 index 73268a0a6..163d440c1 100644 --- a/tools/alpine/versions.aarch64 +++ b/tools/alpine/versions.aarch64 @@ -1,4 +1,4 @@ -# linuxkit/alpine:02b8eb30ca2e3ef16e756b017258fb90e34fb7db-arm64 +# linuxkit/alpine:415e9417004bbd8940a4c2303195b20a5dbd8c9a-arm64 # automatically generated list of installed packages abuild-3.0.0_rc2-r8 alpine-baselayout-3.0.4-r0 @@ -28,6 +28,7 @@ ca-certificates-20161130-r2 cdrkit-1.1.11-r2 celt051-0.5.1.3-r0 cmake-3.8.1-r0 +coreutils-8.27-r0 cryptsetup-1.7.5-r0 cryptsetup-libs-1.7.5-r0 curl-7.55.0-r0 diff --git a/tools/alpine/versions.x86_64 b/tools/alpine/versions.x86_64 index f4ff7c1b0..b202a4abb 100644 --- a/tools/alpine/versions.x86_64 +++ b/tools/alpine/versions.x86_64 @@ -1,4 +1,4 @@ -# linuxkit/alpine:f2f4db272c910d136380781a97e475013fabda8b-amd64 +# linuxkit/alpine:79987c65c66700171c073151c1d3f0372597bec2-amd64 # automatically generated list of installed packages abuild-3.0.0_rc2-r8 alpine-baselayout-3.0.4-r0 @@ -29,6 +29,7 @@ ca-certificates-20161130-r2 cdrkit-1.1.11-r2 celt051-0.5.1.3-r0 cmake-3.8.1-r0 +coreutils-8.27-r0 cryptsetup-1.7.5-r0 cryptsetup-libs-1.7.5-r0 curl-7.55.0-r0 From 7145b40947bc2a00da149a4985c396d76c5a9b91 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Sep 2017 10:40:45 +0100 Subject: [PATCH 2/7] kubernetes: bump to cri-containerd a2dbc6ec1ce6 Adds some bug fixes and support for copy up of anonymous volume contents from the underlying image. Signed-off-by: Ian Campbell --- projects/kubernetes/cri-containerd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kubernetes/cri-containerd/Dockerfile b/projects/kubernetes/cri-containerd/Dockerfile index 0387f3a3e..38ba34636 100644 --- a/projects/kubernetes/cri-containerd/Dockerfile +++ b/projects/kubernetes/cri-containerd/Dockerfile @@ -13,7 +13,7 @@ ENV GOPATH=/go PATH=$PATH:/go/bin ENV CRI_CONTAINERD_URL https://github.com/kubernetes-incubator/cri-containerd.git #ENV CRI_CONTAINERD_BRANCH pull/NNN/head -ENV CRI_CONTAINERD_COMMIT 437131299b805de9d1431300d8054af61fa54a26 +ENV CRI_CONTAINERD_COMMIT a2dbc6ec1ce63fe8c54543c04df0a1a45abdd989 RUN mkdir -p $GOPATH/src/github.com/kubernetes-incubator && \ cd $GOPATH/src/github.com/kubernetes-incubator && \ git clone $CRI_CONTAINERD_URL cri-containerd From 3ce260cb9a364bb135126217f1cbc60c5b2ed237 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Sep 2017 10:38:56 +0100 Subject: [PATCH 3/7] kubernetes: rework kubelet and kubeadm start of day interations Rework the kubelet.sh script by adding an explicit step which waits for the configuration to be valid, either by finding appropriate metadata or by waiting explicitly for kubelet.conf to be created (e.g. by kubeadm) before launching kubelet. The previous construct was implicitly waiting for kubelet.conf to be created since kubelet fails if that file is not present. Pull the set of start of day yaml files to be applied (currently just weave) out of the kubelet image and into the LinuxKit yaml by providing a directory which is searched for *.yaml after init. Signed-off-by: Ian Campbell --- projects/kubernetes/Makefile | 13 ++++++-- projects/kubernetes/kube.yml | 2 ++ projects/kubernetes/kubernetes/Dockerfile | 4 +-- .../kubernetes/kubernetes/kubeadm-init.sh | 7 +++- projects/kubernetes/kubernetes/kubelet.sh | 33 ++++++++++++++----- 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 034e519fd..8b8c6d19e 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,5 +1,10 @@ KUBE_RUNTIME ?= docker +WEAVE_VERSION := v2.0.4 + +INIT_YAML ?= +INIT_YAML += weave.yaml + all: tag-container-images build-vm-images tag-container-images: @@ -14,11 +19,15 @@ push-container-images: build-vm-images: kube-master.iso kube-node.iso -kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml +# NB cannot use $^ because $(INIT_YAML) is not for consumption by "moby build" +kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml $(INIT_YAML) moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml kube-node.iso: kube.yml $(KUBE_RUNTIME).yml - moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml + moby build -name kube-node -format iso-efi -format iso-bios $^ + +weave.yaml: + curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$(WEAVE_VERSION) clean: rm -f -r \ diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 27cbbb305..a4850c15c 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -42,6 +42,8 @@ files: metadata: yaml - path: /etc/kubernetes symlink: "/var/lib/kubeadm" + - path: /etc/kubeadm/kube-system.init/01-weave.yaml + source: weave.yaml - path: /etc/sysctl.d/01-kubernetes.conf contents: 'net.ipv4.ip_forward = 1' - path: /opt/cni diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index fb0d293b0..9d3be0909 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -1,7 +1,6 @@ FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build ENV kubernetes_version v1.7.6 -ENV weave_version v2.0.4 ENV cni_version v0.6.0 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" @@ -31,7 +30,6 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache RUN rmdir /out/var/run && ln -nfs /run /out/var/run RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/plugins/releases/download/${cni_version}/cni-plugins-amd64-${cni_version}.tgz -RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl @@ -45,4 +43,4 @@ WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] COPY --from=build /out / ENV KUBECONFIG "/etc/kubernetes/admin.conf" -LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf", "/etc/kubeadm:/etc/kubeadm"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 1229dae64..8c66f85d1 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,4 +1,9 @@ #!/bin/sh set -e kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ -kubectl create -n kube-system -f /etc/weave.yaml +for i in /etc/kubeadm/kube-system.init/*.yaml ; do + if [ -e "$i" ] ; then + echo "Applying "$(basename "$i") + kubectl create -n kube-system -f "$i" + fi +done diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index f58067802..fd044b266 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Kubelet outputs only to stderr, so arrange for everything we do to go there too +exec 1>&2 + if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then mkdir -p /var/lib/cni/opt/bin tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin @@ -7,7 +10,27 @@ fi if [ -e /etc/kubelet.conf ] ; then . /etc/kubelet.conf fi -until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ +if [ -e /var/config/userdata ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" + kubeadm join --skip-preflight-checks $(cat /var/config/userdata) +fi + +conf=/var/lib/kubeadm/kubelet.conf + +echo "kubelet.sh: waiting for ${conf}" +# TODO(ijc) is there a race between kubeadm creating this file and +# finishing the write where we might be able to fall through and +# start kubelet with an incomplete configuration file? I've tried +# to provoke such a race without success. An explicit +# synchronisation barrier or changing kubeadm to write +# kubelet.conf atomically might be good in any case. +until [ -f "${conf}" ] ; do + sleep 1 +done + +echo "kubelet.sh: ${conf} has arrived" 2>&1 + +exec kubelet --kubeconfig=${conf} \ --require-kubeconfig=true \ --pod-manifest-path=/var/lib/kubeadm/manifests \ --allow-privileged=true \ @@ -18,10 +41,4 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ --network-plugin=cni \ --cni-conf-dir=/var/lib/cni/etc/net.d \ --cni-bin-dir=/var/lib/cni/opt/bin \ - $KUBELET_ARGS $@; do - if [ ! -f /var/config/userdata ] ; then - sleep 1 - else - kubeadm join --skip-preflight-checks $(cat /var/config/userdata) - fi -done + $KUBELET_ARGS $@ From 5c4c25cac6e817631a6fd9c265a617939ec78d43 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Sep 2017 12:03:42 +0100 Subject: [PATCH 4/7] kubernetes: support auto-init via metadata Set KUBE_MASTER_AUTOINIT when using boot.sh to enable. User will need to pick up the token for other nodes using `kubeadm token list`. Signed-off-by: Ian Campbell --- projects/kubernetes/boot.sh | 11 +++++++++-- projects/kubernetes/kubernetes/kubeadm-init.sh | 2 +- projects/kubernetes/kubernetes/kubelet.sh | 10 +++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 3b041aef4..5be099635 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -23,7 +23,14 @@ suffix=".iso" if [ $# -eq 0 ] ; then img="kube-master" - data="" + # If $KUBE_MASTER_AUTOINIT is set, including if it is set to "" + # then we configure for auto init. If it is completely unset then + # we do not. + if [ -n "${KUBE_MASTER_AUTOINIT+x}" ] ; then + data="{\"kubeadm\": {\"init\": \"${KUBE_MASTER_AUTOINIT}\"} }" + else + data="" + fi state="kube-master-state" : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS} @@ -44,7 +51,7 @@ elif [ $# -gt 1 ] || [ $# -eq 1 -a -n "${KUBE_PRESERVE_STATE}" ] ; then img="kube-node" name="node-${1}" shift - data="${*}" + data="{\"kubeadm\": {\"join\": \"${*}\"} }" state="kube-${name}-state" : ${KUBE_VCPUS:=$KUBE_NODE_VCPUS} diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 8c66f85d1..89403fee1 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ +kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ $@ for i in /etc/kubeadm/kube-system.init/*.yaml ; do if [ -e "$i" ] ; then echo "Applying "$(basename "$i") diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index fd044b266..65df80a84 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -10,7 +10,15 @@ fi if [ -e /etc/kubelet.conf ] ; then . /etc/kubelet.conf fi -if [ -e /var/config/userdata ] ; then +if [ -e /var/config/kubeadm/init ] ; then + echo "kubelet.sh: init cluster with metadata \"$(cat /var/config/kubeadm/init)\"" + # This needs to be in the background since it waits for kubelet to start. + # We skip printing the token so it is not persisted in the log. + kubeadm-init.sh --skip-token-print $(cat /var/config/kubeadm/init) & +elif [ -e /var/config/kubeadm/join ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/kubeadm/join)\"" + kubeadm join --skip-preflight-checks $(cat /var/config/kubeadm/join) +elif [ -e /var/config/userdata ] ; then echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" kubeadm join --skip-preflight-checks $(cat /var/config/userdata) fi From 615f135d61ce37d00a21c520bfa0efef683ab4e3 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 14 Sep 2017 12:18:46 +0100 Subject: [PATCH 5/7] kubernetes: use coreutils in kubelet container for full fat `du` Signed-off-by: Ian Campbell --- projects/kubernetes/kubernetes/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 9d3be0909..09e499ee6 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -1,4 +1,4 @@ -FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build +FROM linuxkit/alpine:79987c65c66700171c073151c1d3f0372597bec2 AS build ENV kubernetes_version v1.7.6 ENV cni_version v0.6.0 @@ -10,10 +10,14 @@ RUN apk add -U --no-cache \ && true RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +#coreutils needed for du -B for disk image checks made by kubelet +# example: $ du -s -B 1 /var/lib/kubelet/pods/... +# du: unrecognized option: B RUN apk add --no-cache --initdb -p /out \ alpine-baselayout \ busybox \ ca-certificates \ + coreutils \ curl \ ebtables \ ethtool \ From 451fe22c820da1af9d03db43a53d93ba70768820 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Sep 2017 10:43:22 +0100 Subject: [PATCH 6/7] kubernetes: update yml Signed-off-by: Ian Campbell --- projects/kubernetes/cri-containerd.yml | 2 +- projects/kubernetes/kube.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/cri-containerd.yml b/projects/kubernetes/cri-containerd.yml index 4e98b58e8..637857e1b 100644 --- a/projects/kubernetes/cri-containerd.yml +++ b/projects/kubernetes/cri-containerd.yml @@ -1,6 +1,6 @@ services: - name: cri-containerd - image: linuxkitprojects/cri-containerd:4d159d37996954dfdc78a6d07a324fd872a019d5 + image: linuxkitprojects/cri-containerd:2ff7dce33400a4d184976ca439725d8306295f1a files: - path: /etc/kubelet.conf contents: | diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index a4850c15c..7aba213b5 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -36,7 +36,7 @@ services: - name: sshd image: linuxkit/sshd:d313eea3d9d7fbcbc927d06a6700325725db2a82 - name: kubelet - image: linuxkitprojects/kubernetes:8f3f75005ce56b8588b669ce5d7b02c5d705bbd4 + image: linuxkitprojects/kubernetes:4eba50ea1fae6f881c65429b9c21afadcdeec853 files: - path: etc/linuxkit.yml metadata: yaml From ac6b8dcf6a18f549c710f38342bafc0495498bf4 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Sep 2017 16:43:17 +0100 Subject: [PATCH 7/7] kubernetes: more flexible selection of network stack Also move to 50 in the ordering so things can be added before as well as after. Signed-off-by: Ian Campbell --- projects/kubernetes/.gitignore | 2 ++ projects/kubernetes/Makefile | 15 +++++++++------ projects/kubernetes/kube.yml | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/projects/kubernetes/.gitignore b/projects/kubernetes/.gitignore index 87935610d..6af0a22b7 100644 --- a/projects/kubernetes/.gitignore +++ b/projects/kubernetes/.gitignore @@ -2,3 +2,5 @@ image-cache/common/*.tar image-cache/common/Dockerfile image-cache/control-plane/*.tar image-cache/control-plane/Dockerfile +weave.yaml +network.yaml diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 8b8c6d19e..a17d706a2 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,9 +1,8 @@ KUBE_RUNTIME ?= docker - -WEAVE_VERSION := v2.0.4 +NETWORK ?= weave-v2.0.4 INIT_YAML ?= -INIT_YAML += weave.yaml +INIT_YAML += network.yaml all: tag-container-images build-vm-images @@ -26,10 +25,14 @@ kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml $(INIT_ kube-node.iso: kube.yml $(KUBE_RUNTIME).yml moby build -name kube-node -format iso-efi -format iso-bios $^ -weave.yaml: - curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$(WEAVE_VERSION) +network.yaml: $(NETWORK).yaml + ln -nf $< $@ + +weave-%.yaml: + curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$* clean: rm -f -r \ - kube-*-kernel kube-*-cmdline kube-*-state kube-*-initrd.img *.iso + kube-*-kernel kube-*-cmdline kube-*-state kube-*-initrd.img *.iso \ + weave-*.yaml network.yaml $(MAKE) -C image-cache clean diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 7aba213b5..fd0e1f450 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -42,8 +42,8 @@ files: metadata: yaml - path: /etc/kubernetes symlink: "/var/lib/kubeadm" - - path: /etc/kubeadm/kube-system.init/01-weave.yaml - source: weave.yaml + - path: /etc/kubeadm/kube-system.init/50-network.yaml + source: network.yaml - path: /etc/sysctl.d/01-kubernetes.conf contents: 'net.ipv4.ip_forward = 1' - path: /opt/cni