From 2cfac0e8bbdfb487129f544a8142aa5d93b3e49e Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 7 Jul 2017 15:44:20 +0100 Subject: [PATCH 01/12] kubernetes: Expose each node's ssh port on the host starting from 2222 Port base is configurable (via $KUBE_PORT_BASE envvar). Master uses this and nodes use subsequent ports. Check that the node number is numeric so we can add them to things, but avoid worker node 0 since the port will clash with master. Signed-off-by: Ian Campbell --- projects/kubernetes/boot.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index c11643a6b..2f69fc990 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -1,11 +1,25 @@ #!/bin/bash -eu +: ${KUBE_PORT_BASE:=2222} if [ $# -eq 0 ] ; then img="kube-master" + port=${KUBE_PORT_BASE} data="" state="kube-master-state" elif [ $# -gt 1 ] ; then + case $1 in + ''|*[!0-9]*) + echo "Node number must be a number" + exit 1 + ;; + 0) + echo "Node number must be greater than 0" + exit 1 + ;; + *) ;; + esac img="kube-node" name="node-${1}" + port=$((${KUBE_PORT_BASE} + $1)) shift data="${*}" state="kube-${name}-state" @@ -19,4 +33,4 @@ else fi set -x rm -rf "${state}" -../../bin/linuxkit run -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" +../../bin/linuxkit run -publish $port:22 -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" From 32506bd7f739a835a85a2dcef66db16b06e62cbb Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 14 Jul 2017 14:22:40 +0100 Subject: [PATCH 02/12] kubernetes: Drop custom mounts image and use standard one Apart from the /var/lib mount itself the custom package: - Made host /etc/cni and /opt/cni rshared. This has been handled by init make / rshared since 3c326bebdfdc ("Make / rshared"). - Make /var/lib/kubeadm after mount. For now handle this with a dedicated start of day container instead. Signed-off-by: Ian Campbell --- projects/kubernetes/Makefile | 3 +-- projects/kubernetes/common.rb | 16 ---------------- projects/kubernetes/kube-master.yml | 13 ++++++------- projects/kubernetes/kube-node.yml | 13 ++++++------- projects/kubernetes/mounts.rb | 15 --------------- 5 files changed, 13 insertions(+), 47 deletions(-) delete mode 100644 projects/kubernetes/mounts.rb diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 052802810..71198c8c1 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,6 +1,6 @@ all: build-container-images build-vm-images -BOX_PLANS = kubernetes.rb mounts.rb +BOX_PLANS = kubernetes.rb build-container-images: $(BOX_PLANS) for plan in $(BOX_PLANS) ; do \ @@ -13,7 +13,6 @@ build-container-images: $(BOX_PLANS) push-container-images: build-container-images cache-images docker image push linuxkit/kubernetes:latest - docker image push linuxkit/kubernetes:latest-mounts docker image push linuxkit/kubernetes:latest-image-cache-common docker image push linuxkit/kubernetes:latest-image-cache-control-plane diff --git a/projects/kubernetes/common.rb b/projects/kubernetes/common.rb index b902cd0c1..d1a666e44 100644 --- a/projects/kubernetes/common.rb +++ b/projects/kubernetes/common.rb @@ -22,19 +22,3 @@ end def mount_bind src, dst "mount --bind #{src} #{dst}" end - -def mount_bind_hostns_self mnt - "nsenter --mount=/proc/1/ns/mnt mount -- --bind #{mnt} #{mnt}" -end - -def mount_make_hostns_rshared mnt - "nsenter --mount=/proc/1/ns/mnt mount -- --make-rshared #{mnt}" -end - -def mount_persistent_disk mnt - "/mount.sh #{mnt}" -end - -def mkdir_p dir - "mkdir -p #{dir}" -end diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 0445b5d70..f06ab0dd9 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -16,14 +16,13 @@ onboot: - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts - image: linuxkit/kubernetes:latest-mounts - capabilities: - - all - pid: host - rootfsPropagation: shared + image: linuxkit/mount:b24bd97ae43397b469dbaadd80f17f291c817bdf + command: ["/mount.sh", "/var/lib/"] + - name: var + image: library/alpine:3.6 + command: ["mkdir", "/var/lib/kubeadm"] binds: - - /dev:/dev - - /var:/var:rshared,rbind + - /var/lib:/var/lib services: - name: getty image: linuxkit/getty:deb9332e786e72591bd9be200bcc9c7a534eb754 diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index cb757a7c2..165e52feb 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -16,14 +16,13 @@ onboot: - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts - image: linuxkit/kubernetes:latest-mounts - capabilities: - - all - pid: host - rootfsPropagation: shared + image: linuxkit/mount:b24bd97ae43397b469dbaadd80f17f291c817bdf + command: ["/mount.sh", "/var/lib/"] + - name: var + image: library/alpine:3.6 + command: ["mkdir", "/var/lib/kubeadm"] binds: - - /dev:/dev - - /var:/var:rshared,rbind + - /var/lib:/var/lib services: - name: getty image: linuxkit/getty:deb9332e786e72591bd9be200bcc9c7a534eb754 diff --git a/projects/kubernetes/mounts.rb b/projects/kubernetes/mounts.rb deleted file mode 100644 index a0cddb491..000000000 --- a/projects/kubernetes/mounts.rb +++ /dev/null @@ -1,15 +0,0 @@ -import 'common.rb' - -from "linuxkit/mount:ac8939c4102f97c084d9ddfd445c1908fce6d768" - -script = [ - mount_bind_hostns_self("/etc/cni"), mount_make_hostns_rshared("/etc/cni"), - mount_bind_hostns_self("/opt/cni"), mount_make_hostns_rshared("/opt/cni"), - mount_persistent_disk("/var/lib"), - mkdir_p("/var/lib/kubeadm"), -] - -create_shell_wrapper script.join(' && '), '/usr/bin/kube-mounts.sh' -set_exec cmd: [ '/usr/bin/kube-mounts.sh' ] - -tag "#{@image_name}:latest-mounts" From 4a270083976bf3792c8c2b4e60a3b6936aa79121 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 14 Jul 2017 14:54:42 +0100 Subject: [PATCH 03/12] kubernetes: Update to latest pkg/mount. I somehow managed to miss these in #2209 Signed-off-by: Ian Campbell --- projects/kubernetes/kube-master.yml | 2 +- projects/kubernetes/kube-node.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index f06ab0dd9..fec8f5ec3 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -16,7 +16,7 @@ onboot: - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts - image: linuxkit/mount:b24bd97ae43397b469dbaadd80f17f291c817bdf + image: linuxkit/mount:ac8939c4102f97c084d9ddfd445c1908fce6d768 command: ["/mount.sh", "/var/lib/"] - name: var image: library/alpine:3.6 diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index 165e52feb..a0ab43c84 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -16,7 +16,7 @@ onboot: - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts - image: linuxkit/mount:b24bd97ae43397b469dbaadd80f17f291c817bdf + image: linuxkit/mount:ac8939c4102f97c084d9ddfd445c1908fce6d768 command: ["/mount.sh", "/var/lib/"] - name: var image: library/alpine:3.6 From d208078aa059c379ed1d49a5632575db704eab63 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 14 Jul 2017 14:16:17 +0100 Subject: [PATCH 04/12] kubernetes: cmdline drop page_poison and put console=ttyS0 last Following the trend started in a5b9464a4ef50ca5fb1a4e62ba756d8208da8c66. Signed-off-by: Ian Campbell --- projects/kubernetes/kube-master.yml | 2 +- projects/kubernetes/kube-node.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index fec8f5ec3..31f34851f 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -1,6 +1,6 @@ kernel: image: linuxkit/kernel:4.9.38 - cmdline: "console=ttyS0 console=tty0 page_poison=1" + cmdline: "console=tty0 console=ttyS0" init: - linuxkit/init:059b2bb4b6efa5c58cf53fed4d0ea863521959fc - linuxkit/runc:4a35484aa6f90a1f06cdf1fb36f7056926a084b9 diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index a0ab43c84..15f88186d 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -1,6 +1,6 @@ kernel: image: linuxkit/kernel:4.9.38 - cmdline: "console=ttyS0 console=tty0 page_poison=1" + cmdline: "console=tty0 console=ttyS0" init: - linuxkit/init:059b2bb4b6efa5c58cf53fed4d0ea863521959fc - linuxkit/runc:4a35484aa6f90a1f06cdf1fb36f7056926a084b9 From cd43b44e7c3f3eafd107c84454937ad26d7e55ea Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 14 Jul 2017 16:34:52 +0100 Subject: [PATCH 05/12] kubernetes: Build image using Dockerfile and standard build system This is a pretty straight port of the previous box stuff, without much attempt to clean things up. Image label is a placeholder, will update once a batch of changes are complete. Signed-off-by: Ian Campbell --- projects/kubernetes/Makefile | 16 +--- projects/kubernetes/common.rb | 24 ------ projects/kubernetes/kube-master.yml | 2 +- projects/kubernetes/kube-node.yml | 2 +- projects/kubernetes/kubernetes.rb | 77 ------------------- projects/kubernetes/kubernetes/Dockerfile | 48 ++++++++++++ projects/kubernetes/kubernetes/Makefile | 6 ++ .../kubernetes/kubernetes/kubeadm-init.sh | 2 + projects/kubernetes/kubernetes/kubelet.sh | 2 + 9 files changed, 64 insertions(+), 115 deletions(-) delete mode 100644 projects/kubernetes/common.rb delete mode 100644 projects/kubernetes/kubernetes.rb create mode 100644 projects/kubernetes/kubernetes/Dockerfile create mode 100644 projects/kubernetes/kubernetes/Makefile create mode 100755 projects/kubernetes/kubernetes/kubeadm-init.sh create mode 100755 projects/kubernetes/kubernetes/kubelet.sh diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 71198c8c1..54afb1197 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,18 +1,10 @@ all: build-container-images build-vm-images -BOX_PLANS = kubernetes.rb +build-container-images: + make -C kubernetes tag -build-container-images: $(BOX_PLANS) - for plan in $(BOX_PLANS) ; do \ - docker run --rm -ti \ - -v $(PWD):$(PWD) \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -w $(PWD) \ - boxbuilder/box:master $$plan \ - ; done - -push-container-images: build-container-images cache-images - docker image push linuxkit/kubernetes:latest +push-container-images: cache-images + make -C kubernetes push docker image push linuxkit/kubernetes:latest-image-cache-common docker image push linuxkit/kubernetes:latest-image-cache-control-plane diff --git a/projects/kubernetes/common.rb b/projects/kubernetes/common.rb deleted file mode 100644 index d1a666e44..000000000 --- a/projects/kubernetes/common.rb +++ /dev/null @@ -1,24 +0,0 @@ -@image_name = "linuxkit/kubernetes" - -@versions = { - kubernetes: 'v1.6.1', - weave: 'v1.9.4', - cni: '0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff', -} - -def install_packages pkgs - cmds = [ - %(apk update), - %(apk add #{pkgs.join(' ')}), - ] - - cmds.each { |cmd| run cmd } -end - -def create_shell_wrapper script, path - run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}" -end - -def mount_bind src, dst - "mount --bind #{src} #{dst}" -end diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 31f34851f..9f4f3c791 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -62,7 +62,7 @@ services: binds: - /var/run:/var/run - name: kubelet - image: linuxkit/kubernetes:latest + image: linuxkitprojects/kubernetes:dev capabilities: - all net: host diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index 15f88186d..2ce74321f 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -58,7 +58,7 @@ services: binds: - /var/run:/var/run - name: kubelet - image: linuxkit/kubernetes:latest + image: linuxkitprojects/kubernetes:dev capabilities: - all net: host diff --git a/projects/kubernetes/kubernetes.rb b/projects/kubernetes/kubernetes.rb deleted file mode 100644 index f98eea807..000000000 --- a/projects/kubernetes/kubernetes.rb +++ /dev/null @@ -1,77 +0,0 @@ -import 'common.rb' - -from "alpine:edge" - -def install_node_dependencies - kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64" - cni_release_artefacts = "https://dl.k8s.io/network-plugins/cni-amd64-#{@versions[:cni]}.tar.gz" - weave_launcher = "https://cloud.weave.works/k8s/v1.6/net?v=#{@versions[:weave]}" - - download_files = [ - '/etc/weave.yaml' => { - url: weave_launcher, - mode: '0644', - }, - '/tmp/cni.tgz' => { - url: cni_release_artefacts, - mode: '0644', - }, - '/usr/bin/kubelet' => { - url: "#{kube_release_artefacts}/kubelet", - mode: '0755', - }, - '/usr/bin/kubeadm' => { - url: "#{kube_release_artefacts}/kubeadm", - mode: '0755', - }, - '/usr/bin/kubectl' => { - url: "#{kube_release_artefacts}/kubectl", - mode: '0755', - }, - ] - - download_files.each do |file| - file.each do |dest,info| - run %(curl --output "#{dest}" --fail --silent --location "#{info[:url]}") - run %(chmod "#{info[:mode]}" "#{dest}") - end - end - - run "mkdir -p /opt/cni/bin /etc/cni/net.d && tar xzf /tmp/cni.tgz -C /opt/cni && rm -f /tmp/cni.tgz" -end - -def kubelet_cmd - %w( - kubelet - --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true - --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true - --cluster-dns=10.96.0.10 --cluster-domain=cluster.local - --cgroups-per-qos=false --enforce-node-allocatable="" - --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin - ) -end - -kubelet_dependencies = %w(libc6-compat util-linux iproute2 iptables ebtables ethtool socat curl) -install_packages kubelet_dependencies -install_node_dependencies - -# Exploit shared mounts, give CNI paths back to the host -mount_cni_dirs = [ - mount_bind("/opt/cni", "/rootfs/opt/cni"), - mount_bind("/etc/cni", "/rootfs/etc/cni"), -] - -# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume -wait_for_node_metadata_or_sleep_until_master_init = "[ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config))" - -create_shell_wrapper "#{mount_cni_dirs.join(' && ')} && until #{kubelet_cmd.join(' ')} ; do #{wait_for_node_metadata_or_sleep_until_master_init} ; done", '/usr/bin/kubelet.sh' - -create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]} && kubectl create -n kube-system -f /etc/weave.yaml", '/usr/bin/kubeadm-init.sh' - -flatten - -env KUBECONFIG: "/etc/kubernetes/admin.conf" - -set_exec entrypoint: %w(kubelet.sh) - -tag "#{@image_name}:latest" diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile new file mode 100644 index 000000000..339be6085 --- /dev/null +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -0,0 +1,48 @@ +#FROM linuxkit/alpine:9bcf61f605ef0ce36cc94d59b8eac307862de6e1 AS build +# XXX needs ebtables ethtool iproute2 libc6-compat socat +FROM alpine:3.6 AS build + +ENV kubernetes_version v1.6.1 +ENV weave_version v1.9.4 +ENV cni_version 0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff + +ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" + +RUN apk add -U --no-cache \ + curl \ + && true + +RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +RUN apk add --no-cache --initdb -p /out \ + alpine-baselayout \ + busybox \ + ca-certificates \ + curl \ + ebtables \ + ethtool \ + iproute2 \ + iptables \ + libc6-compat \ + musl \ + socat \ + util-linux \ + && true +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + +ADD kubelet.sh /out/usr/bin/kubelet.sh +ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh + +RUN curl -fSL -o /tmp/cni.tgz https://dl.k8s.io/network-plugins/cni-amd64-${cni_version}.tar.gz && \ + mkdir -p /out/opt/cni /out/etc/cni/net.d && \ + tar -xzf /tmp/cni.tgz -C /out/opt/cni +RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.6/net?v=${weave_version} +RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet +RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm +RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl + +FROM scratch +WORKDIR / +ENTRYPOINT ["/usr/bin/kubelet.sh"] +COPY --from=build /out / +ENV KUBECONFIG "/etc/kubernetes/admin.conf" diff --git a/projects/kubernetes/kubernetes/Makefile b/projects/kubernetes/kubernetes/Makefile new file mode 100644 index 000000000..01aec0f84 --- /dev/null +++ b/projects/kubernetes/kubernetes/Makefile @@ -0,0 +1,6 @@ +ORG?=linuxkitprojects +IMAGE=kubernetes +NETWORK=1 +NOTRUST=1 + +include ../../../pkg/package.mk diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh new file mode 100755 index 000000000..d39f7a4eb --- /dev/null +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -0,0 +1,2 @@ +#!/bin/sh +kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 && kubectl create -n kube-system -f /etc/weave.yaml diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh new file mode 100755 index 000000000..a7bda13bc --- /dev/null +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -0,0 +1,2 @@ +#!/bin/sh +mount --bind /opt/cni /rootfs/opt/cni && mount --bind /etc/cni /rootfs/etc/cni && until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --cgroups-per-qos=false --enforce-node-allocatable= --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks $(cat /mnt/config)) ; done From f853fab1f9cf22377905130ad1702a973950ca40 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 14 Jul 2017 17:15:05 +0100 Subject: [PATCH 06/12] kubernetes: disable content trust for gcr.io It doesn't support it. This makes "make cache-images" work. Previously it would fail with various: Error: remote trust data does not exist for gcr.io/google_containers/pause-amd64: gcr.io does not have trust data for gcr.io/google_containers/pause-amd64 Signed-off-by: Ian Campbell --- projects/kubernetes/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 54afb1197..7c915da0b 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -36,7 +36,7 @@ CONTROL_PLANE_IMAGES := \ image-cache/%.tar: mkdir -p $(dir $@) - DOCKER_CONTENT_TRUST=1 docker image pull gcr.io/google_containers/$(shell basename $@ .tar) + docker image pull gcr.io/google_containers/$(shell basename $@ .tar) docker image save -o $@ gcr.io/google_containers/$(shell basename $@ .tar) cache-images: From 20c0d080df089a47f627e33f34afcfe37ec4e3e1 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 14 Jul 2017 17:28:56 +0100 Subject: [PATCH 07/12] kubernetes: use image labels to simplify yaml requirements Also `net: host` is the default, so drop. Signed-off-by: Ian Campbell --- projects/kubernetes/kube-master.yml | 14 -------------- projects/kubernetes/kube-node.yml | 14 -------------- projects/kubernetes/kubernetes/Dockerfile | 1 + 3 files changed, 1 insertion(+), 28 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 9f4f3c791..296fd04ee 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -63,20 +63,6 @@ services: - /var/run:/var/run - name: kubelet image: linuxkitprojects/kubernetes:dev - capabilities: - - all - net: host - pid: host - mounts: - - type: cgroup - options: ["rw","nosuid","noexec","nodev","relatime"] - binds: - - /dev:/dev - - /var:/var:rshared,rbind - - /var/lib/kubeadm:/etc/kubernetes - - /etc/cni:/rootfs/etc/cni:rshared,rbind - - /opt/cni:/rootfs/opt/cni:rshared,rbind - rootfsPropagation: shared files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index 2ce74321f..a28074651 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -59,20 +59,6 @@ services: - /var/run:/var/run - name: kubelet image: linuxkitprojects/kubernetes:dev - capabilities: - - all - net: host - pid: host - mounts: - - type: cgroup - options: ["rw","nosuid","noexec","nodev","relatime"] - binds: - - /dev:/dev - - /var:/var:rshared,rbind - - /var/lib/kubeadm:/etc/kubernetes - - /etc/cni:/rootfs/etc/cni:rshared,rbind - - /opt/cni:/rootfs/opt/cni:rshared,rbind - rootfsPropagation: shared files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 339be6085..3ddda9ca8 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -46,3 +46,4 @@ WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] COPY --from=build /out / ENV KUBECONFIG "/etc/kubernetes/admin.conf" +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/cni:/rootfs/etc/cni:rshared,rbind", "/opt/cni:/rootfs/opt/cni:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host"}' From a08a312d1f004325e51322bcfed5d9082a46b7eb Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 17 Jul 2017 10:29:38 +0100 Subject: [PATCH 08/12] kubernetes: Use image labels for image-cache bind mounts Signed-off-by: Ian Campbell --- projects/kubernetes/image-cache/Dockerfile | 1 + projects/kubernetes/kube-master.yml | 4 ---- projects/kubernetes/kube-node.yml | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/projects/kubernetes/image-cache/Dockerfile b/projects/kubernetes/image-cache/Dockerfile index ebdfbed2f..c3197868c 100644 --- a/projects/kubernetes/image-cache/Dockerfile +++ b/projects/kubernetes/image-cache/Dockerfile @@ -2,3 +2,4 @@ FROM linuxkit/docker-ce:9b937df179bdbebbc70243779978057df0b54190 ADD . /images ENTRYPOINT [ "/bin/sh", "-c" ] CMD [ "for image in /images/*.tar ; do docker image load -i $image && rm -f $image ; done" ] +LABEL org.mobyproject.config='{"binds": ["/var/run:/var/run"]}' diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 296fd04ee..450d12079 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -55,12 +55,8 @@ services: rootfsPropagation: shared - name: kubernetes-image-cache-common image: linuxkit/kubernetes:latest-image-cache-common - binds: - - /var/run:/var/run - name: kubernetes-image-cache-control-plane image: linuxkit/kubernetes:latest-image-cache-control-plane - binds: - - /var/run:/var/run - name: kubelet image: linuxkitprojects/kubernetes:dev files: diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index a28074651..0afd8cbeb 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -55,8 +55,6 @@ services: rootfsPropagation: shared - name: kubernetes-image-cache-common image: linuxkit/kubernetes:latest-image-cache-common - binds: - - /var/run:/var/run - name: kubelet image: linuxkitprojects/kubernetes:dev files: From f18ed90d4f8190850553a1c20e38390c60d220a9 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 17 Jul 2017 11:24:21 +0100 Subject: [PATCH 09/12] kubernetes: Add local scripts after downloads Caches much better. Signed-off-by: Ian Campbell --- projects/kubernetes/kubernetes/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 3ddda9ca8..6a39d6910 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -30,9 +30,6 @@ RUN apk add --no-cache --initdb -p /out \ # Remove apk residuals. We have a read-only rootfs, so apk is of no use. RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache -ADD kubelet.sh /out/usr/bin/kubelet.sh -ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh - RUN curl -fSL -o /tmp/cni.tgz https://dl.k8s.io/network-plugins/cni-amd64-${cni_version}.tar.gz && \ mkdir -p /out/opt/cni /out/etc/cni/net.d && \ tar -xzf /tmp/cni.tgz -C /out/opt/cni @@ -41,6 +38,9 @@ RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bi RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl +ADD kubelet.sh /out/usr/bin/kubelet.sh +ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh + FROM scratch WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] From 23d6649de0992d06bd2194b49daca34b19ec04bf Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 17 Jul 2017 11:43:39 +0100 Subject: [PATCH 10/12] kubernetes: Format scripts to be more readable. These were originally generated by some box builder runes and then taken wholesale here. Format them to be more readable. Signed-off-by: Ian Campbell --- .../kubernetes/kubernetes/kubeadm-init.sh | 4 +++- projects/kubernetes/kubernetes/kubelet.sh | 20 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index d39f7a4eb..5e953229d 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,2 +1,4 @@ #!/bin/sh -kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 && kubectl create -n kube-system -f /etc/weave.yaml +set -e +kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 +kubectl create -n kube-system -f /etc/weave.yaml diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index a7bda13bc..535844853 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -1,2 +1,20 @@ #!/bin/sh -mount --bind /opt/cni /rootfs/opt/cni && mount --bind /etc/cni /rootfs/etc/cni && until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true --cluster-dns=10.96.0.10 --cluster-domain=cluster.local --cgroups-per-qos=false --enforce-node-allocatable= --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks $(cat /mnt/config)) ; done +mount --bind /opt/cni /rootfs/opt/cni +mount --bind /etc/cni /rootfs/etc/cni +until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ + --require-kubeconfig=true \ + --pod-manifest-path=/var/lib/kubeadm/manifests \ + --allow-privileged=true \ + --cluster-dns=10.96.0.10 \ + --cluster-domain=cluster.local \ + --cgroups-per-qos=false \ + --enforce-node-allocatable= \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin ; do + if [ ! -e /dev/sr0 ] ; then + sleep 1 + else + mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks $(cat /mnt/config) + fi +done From d48cc7998b0b932966fe70f0c48bb86715654b18 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 17 Jul 2017 13:36:06 +0100 Subject: [PATCH 11/12] kubernetes: Use pkg/metadata Still a flat/unstructured config space, but at least uses the mounting machinery. `boot.sh` continues to just work without modification. Signed-off-by: Ian Campbell --- projects/kubernetes/kube-master.yml | 2 ++ projects/kubernetes/kube-node.yml | 2 ++ projects/kubernetes/kubernetes/kubelet.sh | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 450d12079..02e176285 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -13,6 +13,8 @@ onboot: image: linuxkit/sysfs:006a65b30cfdd9d751d7ab042fde7eca2c3bc9dc - name: binfmt image: linuxkit/binfmt:0bde4ebd422099f45c5ee03217413523ad2223e5 + - name: metadata + image: linuxkit/metadata:f122f1b4e873f1d08cd67bd9105385fd923af0cb - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index 0afd8cbeb..1976c89a4 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -13,6 +13,8 @@ onboot: image: linuxkit/sysfs:006a65b30cfdd9d751d7ab042fde7eca2c3bc9dc - name: binfmt image: linuxkit/binfmt:0bde4ebd422099f45c5ee03217413523ad2223e5 + - name: metadata + image: linuxkit/metadata:f122f1b4e873f1d08cd67bd9105385fd923af0cb - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index 535844853..fb8ef2e81 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -12,9 +12,9 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ --network-plugin=cni \ --cni-conf-dir=/etc/cni/net.d \ --cni-bin-dir=/opt/cni/bin ; do - if [ ! -e /dev/sr0 ] ; then + if [ ! -f /var/config/userdata ] ; then sleep 1 else - mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks $(cat /mnt/config) + kubeadm join --skip-preflight-checks $(cat /var/config/userdata) fi done From c495ccb8fb16f0931c47da8cf1b7c66963969a4e Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 17 Jul 2017 12:10:12 +0100 Subject: [PATCH 12/12] kubernetes: Update yml files. Signed-off-by: Ian Campbell --- projects/kubernetes/kube-master.yml | 2 +- projects/kubernetes/kube-node.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 02e176285..fb0a76d7f 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -60,7 +60,7 @@ services: - name: kubernetes-image-cache-control-plane image: linuxkit/kubernetes:latest-image-cache-control-plane - name: kubelet - image: linuxkitprojects/kubernetes:dev + image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index 1976c89a4..d586adb8a 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -58,7 +58,7 @@ services: - name: kubernetes-image-cache-common image: linuxkit/kubernetes:latest-image-cache-common - name: kubelet - image: linuxkitprojects/kubernetes:dev + image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub