From 9a99164cf769a2a1db79d1ddde448a3bee81248c Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 2 Oct 2017 12:29:16 +0100 Subject: [PATCH 1/6] kubernetes: Use /etc/kubernetes rather than /var/lib/kubelet This is a bind mount, but /etc/kubernetes is the path formally expected/create by kubeadm. Signed-off-by: Ian Campbell --- projects/kubernetes/kubernetes/kubelet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index 0f6e9aee5..17bc46dd7 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -11,7 +11,7 @@ if [ -e /etc/kubelet.sh.conf ] ; then . /etc/kubelet.sh.conf fi -conf=/var/lib/kubeadm/kubelet.conf +conf=/etc/kubernetes/kubelet.conf if [ -f "${conf}" ] ; then echo "kubelet.sh: kubelet already configured" @@ -43,7 +43,7 @@ echo "kubelet.sh: ${conf} has arrived" 2>&1 exec kubelet --kubeconfig=${conf} \ --require-kubeconfig=true \ - --pod-manifest-path=/var/lib/kubeadm/manifests \ + --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged=true \ --cluster-dns=10.96.0.10 \ --cluster-domain=cluster.local \ From f85208c74a8b6547b17cfaa3f21718b42b8dfd1a Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 2 Oct 2017 12:28:49 +0100 Subject: [PATCH 2/6] kubernetes: Bump to v1.8.0 Signed-off-by: Ian Campbell --- projects/kubernetes/Makefile | 2 +- projects/kubernetes/image-cache/mkversions | 2 +- projects/kubernetes/image-cache/versions.mk | 8 ++++---- projects/kubernetes/kubernetes/Dockerfile | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index a17d706a2..70dff6d93 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -29,7 +29,7 @@ network.yaml: $(NETWORK).yaml ln -nf $< $@ weave-%.yaml: - curl -L -o $@ https://cloud.weave.works/k8s/v1.7/net?v=$* + curl -L -o $@ https://cloud.weave.works/k8s/v1.8/net?v=$* clean: rm -f -r \ diff --git a/projects/kubernetes/image-cache/mkversions b/projects/kubernetes/image-cache/mkversions index 62b2a47e5..5361dd3ec 100755 --- a/projects/kubernetes/image-cache/mkversions +++ b/projects/kubernetes/image-cache/mkversions @@ -1,6 +1,6 @@ #!/bin/sh repo=gcr.io/google_containers -kube_version=v1.7.6 +kube_version=v1.8.0 kube_dns_version=1.14.4 pause_version=3.0 etcd_version=3.0.17 diff --git a/projects/kubernetes/image-cache/versions.mk b/projects/kubernetes/image-cache/versions.mk index 02c6b0ea7..929d93b4d 100644 --- a/projects/kubernetes/image-cache/versions.mk +++ b/projects/kubernetes/image-cache/versions.mk @@ -1,13 +1,13 @@ # autogenerated by mkversions COMMON_IMAGES := \ - kube-proxy-amd64\:v1.7.6@sha256\:1509f2fc8a60501d604d21d983ed6f5d0ea40ccdd7cc6ba6c994389ef7db16d8 \ + kube-proxy-amd64\:v1.8.0@sha256\:3da0e03a49404119fbf104037b7ac24d871842bf565a4364390a7d6ea60f48a0 \ k8s-dns-sidecar-amd64\:1.14.4@sha256\:97074c951046e37d3cbb98b82ae85ed15704a290cce66a8314e7f846404edde9 \ k8s-dns-kube-dns-amd64\:1.14.4@sha256\:40790881bbe9ef4ae4ff7fe8b892498eecb7fe6dcc22661402f271e03f7de344 \ k8s-dns-dnsmasq-nanny-amd64\:1.14.4@sha256\:aeeb994acbc505eabc7415187cd9edb38cbb5364dc1c2fc748154576464b3dc2 \ pause-amd64\:3.0@sha256\:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516 CONTROL_PLANE_IMAGES := \ - kube-apiserver-amd64\:v1.7.6@sha256\:f3a208d30314a89952cf613e5ee671f9d2ed7b197cd6c5d91bebfe02571d7e1b \ - kube-controller-manager-amd64\:v1.7.6@sha256\:42a42e8d39fd68de7c1db6844f909bfa6bff89019ecef86e6c542354cf8ab9fb \ - kube-scheduler-amd64\:v1.7.6@sha256\:334a38ac844be07599f74876f6c923271bbd0aab48a43e7ca1ad4942e9ebdabd \ + kube-apiserver-amd64\:v1.8.0@sha256\:e7377096f0b88b0fcc5dce1c56aed002f999f095a30676c68b8f686a6bb1e943 \ + kube-controller-manager-amd64\:v1.8.0@sha256\:9b1ae1007d3d0f4272e7ad5e8a6d34a1f49c2e3a33339773864aed525a667d26 \ + kube-scheduler-amd64\:v1.8.0@sha256\:79b5b697b1bc1023b5f18a792ea7020c4ad8403d861169b1b420d3fe673b9fe4 \ etcd-amd64\:3.0.17@sha256\:d83d3545e06fb035db8512e33bd44afb55dea007a3abd7b17742d3ac6d235940 diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index af4da42b9..770cf12cc 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -1,6 +1,6 @@ FROM linuxkit/alpine:28254e4530703db4caa6b0199a025c30a987dfa1 AS build -ENV kubernetes_version v1.7.6 +ENV kubernetes_version v1.8.0 ENV cni_version v0.6.0 RUN apk add -U --no-cache \ From 57a2ae43183da5c4327841d8c36a82406476c703 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 3 Oct 2017 14:53:03 +0100 Subject: [PATCH 3/6] kubernetes: adjust for v1.8 bootstrapping arrangements. With kube 1.8 kubeadm initially configures worker nodes with a bootstrap-kubelet.conf. Adjust our start of day scripting to DTRT. Signed-off-by: Ian Campbell --- projects/kubernetes/kubernetes/kubelet.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index 17bc46dd7..cab7d1f00 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -11,9 +11,9 @@ if [ -e /etc/kubelet.sh.conf ] ; then . /etc/kubelet.sh.conf fi -conf=/etc/kubernetes/kubelet.conf +await=/etc/kubernetes/kubelet.conf -if [ -f "${conf}" ] ; then +if [ -f "/etc/kubernetes/kubelet.conf" ] ; then echo "kubelet.sh: kubelet already configured" elif [ -e /var/config/kubeadm/init ] ; then echo "kubelet.sh: init cluster with metadata \"$(cat /var/config/kubeadm/init)\"" @@ -23,26 +23,30 @@ elif [ -e /var/config/kubeadm/init ] ; then elif [ -e /var/config/kubeadm/join ] ; then echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/kubeadm/join)\"" kubeadm join --skip-preflight-checks $(cat /var/config/kubeadm/join) + await=/etc/kubernetes/bootstrap-kubelet.conf elif [ -e /var/config/userdata ] ; then echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" kubeadm join --skip-preflight-checks $(cat /var/config/userdata) + await=/etc/kubernetes/bootstrap-kubelet.conf fi -echo "kubelet.sh: waiting for ${conf}" +echo "kubelet.sh: waiting for ${await}" # TODO(ijc) is there a race between kubeadm creating this file and # finishing the write where we might be able to fall through and # start kubelet with an incomplete configuration file? I've tried # to provoke such a race without success. An explicit # synchronisation barrier or changing kubeadm to write # kubelet.conf atomically might be good in any case. -until [ -f "${conf}" ] ; do +until [ -f "${await}" ] ; do sleep 1 done -echo "kubelet.sh: ${conf} has arrived" 2>&1 +echo "kubelet.sh: ${await} has arrived" 2>&1 -exec kubelet --kubeconfig=${conf} \ - --require-kubeconfig=true \ +mkdir -p /etc/kubernetes/manifests + +exec kubelet --kubeconfig=/etc/kubernetes/kubelet.conf \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged=true \ --cluster-dns=10.96.0.10 \ From 72dff9059de806303f2fc0c43cfc6ce4ef931455 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 3 Oct 2017 14:56:12 +0100 Subject: [PATCH 4/6] kubernetes: arrange for kubelet-plugins directory to be persistent /usr/libexec/kubernetes/kubelet-plugins is a new path in Kube 1.8 (related to flexvolumes) which should be persisted. Like /etc/cni and /opt/cni we also need to arrange for this path to be valid in the host environment (since various system containers will try and mount bind mount it). Signed-off-by: Ian Campbell --- projects/kubernetes/cri-containerd/Dockerfile | 2 +- projects/kubernetes/docker.yml | 3 ++- projects/kubernetes/kube.yml | 2 ++ projects/kubernetes/kubernetes/Dockerfile | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/kubernetes/cri-containerd/Dockerfile b/projects/kubernetes/cri-containerd/Dockerfile index f524dcdae..cb4205ec5 100644 --- a/projects/kubernetes/cri-containerd/Dockerfile +++ b/projects/kubernetes/cri-containerd/Dockerfile @@ -48,4 +48,4 @@ FROM scratch WORKDIR / ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/var/lib/cni/opt/bin", "--network-conf-dir", "/var/lib/cni/etc/net.d"] COPY --from=build /out / -LABEL org.mobyproject.config='{"binds": ["/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/dev:/dev", "/tmp:/tmp", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/etc/cni:rshared,rbind", "/var/lib/cni/opt:/opt/cni:rshared,rbind", "/run/containerd/containerd.sock:/run/containerd/containerd.sock"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc/net.d", "/var/lib/cni/opt"]}}' +LABEL org.mobyproject.config='{"binds": ["/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/dev:/dev", "/tmp:/tmp", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/etc/cni:rshared,rbind", "/var/lib/cni/opt:/opt/cni:rshared,rbind", "/run/containerd/containerd.sock:/run/containerd/containerd.sock", "/var/lib/kubelet-plugins:/usr/libexec/kubernetes/kubelet-plugins:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc/net.d", "/var/lib/cni/opt", "/var/lib/kubelet-plugins"]}}' diff --git a/projects/kubernetes/docker.yml b/projects/kubernetes/docker.yml index a34477eac..b1d865d80 100644 --- a/projects/kubernetes/docker.yml +++ b/projects/kubernetes/docker.yml @@ -16,10 +16,11 @@ services: - /var/lib/kubeadm:/etc/kubernetes - /var/lib/cni/etc:/etc/cni:rshared,rbind - /var/lib/cni/opt:/opt/cni:rshared,rbind + - /var/lib/kubelet-plugins:/usr/libexec/kubernetes/kubelet-plugins:rshared,rbind rootfsPropagation: shared command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] runtime: - mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"] + mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt", "/var/lib/kubelet-plugins"] - name: kubernetes-image-cache-common image: linuxkitprojects/kubernetes-image-cache-common:465c8781efd6664c9a744eb78bf5b90d6ce5db3e files: diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 370aa2b20..1d7394999 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -42,6 +42,8 @@ files: metadata: yaml - path: /etc/kubernetes symlink: "/var/lib/kubeadm" + - path: /usr/libexec/kubernetes/kubelet-plugins + symlink: "/var/lib/kubelet-plugins" - path: /etc/kubeadm/kube-system.init/50-network.yaml source: network.yaml - path: /etc/sysctl.d/01-kubernetes.conf diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 770cf12cc..26f676a41 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -75,4 +75,4 @@ WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] COPY --from=build /out / ENV KUBECONFIG "/etc/kubernetes/admin.conf" -LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.sh.conf:/etc/kubelet.sh.conf", "/etc/kubeadm:/etc/kubeadm"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.sh.conf:/etc/kubelet.sh.conf", "/etc/kubeadm:/etc/kubeadm", "/var/lib/kubelet-plugins:/usr/libexec/kubernetes/kubelet-plugins:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt", "/var/lib/kubelet-plugins"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' From c0a3656eb3c92717d09cd3e7ba607a77426655e9 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 3 Oct 2017 15:06:26 +0100 Subject: [PATCH 5/6] kubernetes: update to kub dns 1.14.5 Signed-off-by: Ian Campbell --- projects/kubernetes/image-cache/mkversions | 2 +- projects/kubernetes/image-cache/versions.mk | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/kubernetes/image-cache/mkversions b/projects/kubernetes/image-cache/mkversions index 5361dd3ec..a9e559f86 100755 --- a/projects/kubernetes/image-cache/mkversions +++ b/projects/kubernetes/image-cache/mkversions @@ -1,7 +1,7 @@ #!/bin/sh repo=gcr.io/google_containers kube_version=v1.8.0 -kube_dns_version=1.14.4 +kube_dns_version=1.14.5 pause_version=3.0 etcd_version=3.0.17 diff --git a/projects/kubernetes/image-cache/versions.mk b/projects/kubernetes/image-cache/versions.mk index 929d93b4d..57c13147b 100644 --- a/projects/kubernetes/image-cache/versions.mk +++ b/projects/kubernetes/image-cache/versions.mk @@ -1,9 +1,9 @@ # autogenerated by mkversions COMMON_IMAGES := \ kube-proxy-amd64\:v1.8.0@sha256\:3da0e03a49404119fbf104037b7ac24d871842bf565a4364390a7d6ea60f48a0 \ - k8s-dns-sidecar-amd64\:1.14.4@sha256\:97074c951046e37d3cbb98b82ae85ed15704a290cce66a8314e7f846404edde9 \ - k8s-dns-kube-dns-amd64\:1.14.4@sha256\:40790881bbe9ef4ae4ff7fe8b892498eecb7fe6dcc22661402f271e03f7de344 \ - k8s-dns-dnsmasq-nanny-amd64\:1.14.4@sha256\:aeeb994acbc505eabc7415187cd9edb38cbb5364dc1c2fc748154576464b3dc2 \ + k8s-dns-sidecar-amd64\:1.14.5@sha256\:9aab42bf6a2a068b797fe7d91a5d8d915b10dbbc3d6f2b10492848debfba6044 \ + k8s-dns-kube-dns-amd64\:1.14.5@sha256\:1a3fc069de481ae690188f6f1ba4664b5cc7760af37120f70c86505c79eea61d \ + k8s-dns-dnsmasq-nanny-amd64\:1.14.5@sha256\:46b933bb70270c8a02fa6b6f87d440f6f1fce1a5a2a719e164f83f7b109f7544 \ pause-amd64\:3.0@sha256\:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516 CONTROL_PLANE_IMAGES := \ From eb0fde3a10581527fbd000a891c8c28ece4303be Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 3 Oct 2017 15:18:21 +0100 Subject: [PATCH 6/6] kubernetes: update yml Signed-off-by: Ian Campbell --- projects/kubernetes/cri-containerd.yml | 2 +- projects/kubernetes/docker-master.yml | 2 +- projects/kubernetes/docker.yml | 2 +- projects/kubernetes/kube.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/kubernetes/cri-containerd.yml b/projects/kubernetes/cri-containerd.yml index ca4f05efe..bd7aeb6f8 100644 --- a/projects/kubernetes/cri-containerd.yml +++ b/projects/kubernetes/cri-containerd.yml @@ -1,6 +1,6 @@ services: - name: cri-containerd - image: linuxkitprojects/cri-containerd:3ccc80f3257647a02c4a78fb0f0557805601c958 + image: linuxkitprojects/cri-containerd:2e3f5fcb87f92e1176637943b496910b9d979fb5 files: - path: /etc/kubelet.sh.conf contents: | diff --git a/projects/kubernetes/docker-master.yml b/projects/kubernetes/docker-master.yml index 16acb2fce..9c1854ace 100644 --- a/projects/kubernetes/docker-master.yml +++ b/projects/kubernetes/docker-master.yml @@ -1,3 +1,3 @@ services: - name: kubernetes-image-cache-control-plane - image: linuxkitprojects/kubernetes-image-cache-control-plane:465c8781efd6664c9a744eb78bf5b90d6ce5db3e + image: linuxkitprojects/kubernetes-image-cache-control-plane:2c8a47f1dfc9f4b00771b27b5f8b74065c132a3a diff --git a/projects/kubernetes/docker.yml b/projects/kubernetes/docker.yml index b1d865d80..28f682d44 100644 --- a/projects/kubernetes/docker.yml +++ b/projects/kubernetes/docker.yml @@ -22,7 +22,7 @@ services: runtime: mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt", "/var/lib/kubelet-plugins"] - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:465c8781efd6664c9a744eb78bf5b90d6ce5db3e + image: linuxkitprojects/kubernetes-image-cache-common:2c8a47f1dfc9f4b00771b27b5f8b74065c132a3a files: - path: /etc/kubelet.sh.conf contents: "" diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 1d7394999..88723876a 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -36,7 +36,7 @@ services: - name: sshd image: linuxkit/sshd:d313eea3d9d7fbcbc927d06a6700325725db2a82 - name: kubelet - image: linuxkitprojects/kubernetes:807de925610de344864cadd21b9860342a5c2bda + image: linuxkitprojects/kubernetes:98d03686d3665b935dcd68da192f79c4cb618ec7 files: - path: etc/linuxkit.yml metadata: yaml