From 73a29022ce89984fe3f9d20823d91ad9a7075c60 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Tue, 11 Apr 2017 12:23:40 +0100 Subject: [PATCH 1/4] Cache Kubernetes images, update image names, add `boot-master.sh` Signed-off-by: Ilya Dmitrichenko --- projects/kubernetes/.gitignore | 4 +++ projects/kubernetes/Boxfile | 3 +- projects/kubernetes/Makefile | 38 ++++++++++++++++++++-- projects/kubernetes/README.md | 2 +- projects/kubernetes/boot-master.sh | 3 ++ projects/kubernetes/image-cache/Dockerfile | 6 ++++ projects/kubernetes/kube-master.yml | 10 +++++- 7 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 projects/kubernetes/.gitignore create mode 100755 projects/kubernetes/boot-master.sh create mode 100644 projects/kubernetes/image-cache/Dockerfile diff --git a/projects/kubernetes/.gitignore b/projects/kubernetes/.gitignore new file mode 100644 index 000000000..87935610d --- /dev/null +++ b/projects/kubernetes/.gitignore @@ -0,0 +1,4 @@ +image-cache/common/*.tar +image-cache/common/Dockerfile +image-cache/control-plane/*.tar +image-cache/control-plane/Dockerfile diff --git a/projects/kubernetes/Boxfile b/projects/kubernetes/Boxfile index 37abfaa4e..19d26de64 100644 --- a/projects/kubernetes/Boxfile +++ b/projects/kubernetes/Boxfile @@ -83,8 +83,7 @@ def create_kubelet_wrapper end def create_kubeadm_wrappers - #create_shell_wrapper 'env KUBE_KUBERNETES_DIR=/var/lib/kubeadm kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 --cert-dir /var/lib/kubeadm/pki', '/usr/bin/kubeadm-init.sh' - create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1", '/usr/bin/kubeadm-init.sh' + create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh' end install_dependencies diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 992fa0307..e19c48d15 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -8,10 +8,44 @@ build-container-images: Boxfile boxbuilder/box:master Boxfile push-container-images: build-container-image - docker push mobylinux/kubernetes:latest + docker image push mobylinux/kubernetes:latest + docker image push mobylinux/kubernetes:latest-image-cache-common + docker image push mobylinux/kubernetes:latest-image-cache-control-plane build-vm-images: ../../bin/moby build -name kube-master kube-master.yml clean: - rm -f kube-master-bzImage kube-master-cmdline kube-master-disk.img kube-master-initrd.img + rm -f -r \ + kube-master-bzImage kube-master-cmdline kube-master-disk.img kube-master-initrd.img \ + image-cache/common image-cache/control-plane + +COMMON_IMAGES := \ + kube-proxy-amd64:v1.6.1 \ + k8s-dns-sidecar-amd64:1.14.1 \ + k8s-dns-kube-dns-amd64:1.14.1 \ + k8s-dns-dnsmasq-nanny-amd64:1.14.1 \ + pause-amd64:3.0 + +CONTROL_PLANE_IMAGES := \ + kube-apiserver-amd64:v1.6.1 \ + kube-controller-manager-amd64:v1.6.1 \ + kube-scheduler-amd64:v1.6.1 \ + etcd-amd64:3.0.17 + +image-cache/%.tar: + mkdir -p $(dir $@) + DOCKER_CONTENT_TRUST=1 docker image pull gcr.io/google_containers/$(shell basename $@ .tar) + docker image save -o $@ gcr.io/google_containers/$(shell basename $@ .tar) + +cache-images: + for image in $(COMMON_IMAGES) ; \ + do $(MAKE) "image-cache/common/$${image}.tar" \ + ; done + cp image-cache/Dockerfile image-cache/common + docker image build -t mobylinux/kubernetes:latest-image-cache-common image-cache/common + for image in $(CONTROL_PLANE_IMAGES) ; \ + do $(MAKE) "image-cache/control-plane/$${image}.tar" \ + ; done + cp image-cache/Dockerfile image-cache/control-plane + docker image build -t mobylinux/kubernetes:latest-image-cache-control-plane image-cache/control-plane diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index 4aff31326..33b2e936c 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -11,7 +11,7 @@ make Boot Kubernetes master OS image using `hyperkit` on macOS: ``` -../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master +./boot-master.sh ``` Manually initialise master with `kubeadm`: diff --git a/projects/kubernetes/boot-master.sh b/projects/kubernetes/boot-master.sh new file mode 100755 index 000000000..09b7fa866 --- /dev/null +++ b/projects/kubernetes/boot-master.sh @@ -0,0 +1,3 @@ +#!/bin/bash -eux +rm -f kube-master-disk.img +../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master diff --git a/projects/kubernetes/image-cache/Dockerfile b/projects/kubernetes/image-cache/Dockerfile new file mode 100644 index 000000000..e5e1ecafc --- /dev/null +++ b/projects/kubernetes/image-cache/Dockerfile @@ -0,0 +1,6 @@ +FROM mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08 + +ADD . /images + +ENTRYPOINT [ "/bin/sh", "-c" ] +CMD [ "for image in /images/*.tar ; do docker image load -i $image ; done" ] diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 4992884cd..1d0e872a4 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -77,8 +77,16 @@ services: - /var:/var:rshared,rbind - /var/lib/kubeadm:/etc/kubernetes rootfsPropagation: shared + - name: kubernetes-image-cache-common + image: "mobylinux/kubernetes:latest-image-cache-common" + binds: + - /var/run:/var/run + - name: kubernetes-image-cache-control-plane + image: "mobylinux/kubernetes:latest-image-cache-control-plane" + binds: + - /var/run:/var/run - name: kubelet - image: "errordeveloper/mobykube:master" + image: "mobylinux/kubernetes:latest" capabilities: - all net: host From 7bcca5cf7f3195b1c051c499bf7290ca059b0012 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Tue, 11 Apr 2017 12:32:57 +0100 Subject: [PATCH 2/4] Sync with `examples/doecker.yml` Signed-off-by: Ilya Dmitrichenko --- projects/kubernetes/kube-master.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 1d0e872a4..d45d8c08e 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -2,9 +2,9 @@ kernel: image: "mobylinux/kernel:4.9.x" cmdline: "console=ttyS0 console=tty0 page_poison=1" init: - - mobylinux/init:671bdce1ed0803daeb35e83e4bcd576bb449ea35 + - mobylinux/init:e10e2efc1b78ef41d196175cbc07e069391f406e - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9 - - mobylinux/containerd:c7f6ecdcbcb615a53edee556ba03c7c873bc8488 + - mobylinux/containerd:18eaf72f3f4f9a9f29ca1951f66df701f873060b - mobylinux/ca-certificates:eabc5a6e59f05aa91529d80e9a595b85b046f935 onboot: - name: sysctl @@ -15,6 +15,8 @@ onboot: capabilities: - CAP_SYS_ADMIN readonly: true + - name: sysfs + image: mobylinux/sysfs:6c1d06f28ddd9681799d3950cddf044b930b221c - name: binfmt image: "mobylinux/binfmt:bdb754f25a5d851b4f5f8d185a43dfcbb3c22d01" binds: @@ -54,6 +56,15 @@ services: - CAP_NET_RAW net: host oomScoreAdj: -800 + - name: ntpd + image: "mobylinux/openntpd:a570316d7fc49ca1daa29bd945499f4963d227af" + capabilities: + - CAP_SYS_TIME + - CAP_SYS_NICE + - CAP_SYS_CHROOT + - CAP_SETUID + - CAP_SETGID + net: host - name: sshd image: "mobylinux/sshd:160631d59fffc13d523ff7f09b3b49538d34b9cd" capabilities: From 89b8021f6259421427331a8e4c99bdedebf2988b Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Tue, 11 Apr 2017 13:58:02 +0100 Subject: [PATCH 3/4] Create `/var/lib/kubelet` at runtime, after `/var` is mounted Signed-off-by: Ilya Dmitrichenko --- projects/kubernetes/kube-master.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index d45d8c08e..a4b645828 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -38,6 +38,11 @@ onboot: - CAP_SYS_ADMIN rootfsPropagation: shared command: ["/mount.sh", "/var/lib"] + - name: mount + image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + binds: + - /var:/var + command: ["mkdir", "-p", "/var/lib/kubeadm"] services: - name: rngd image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92" @@ -115,7 +120,5 @@ services: files: - path: root/.ssh/authorized_keys contents: '# Your ssh key goes here' - - path: var/lib/kubeadm - directory: true outputs: - format: kernel+initrd From 52ba02b9c8708dbae9916d9f81691844187e28b7 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Tue, 11 Apr 2017 14:08:38 +0100 Subject: [PATCH 4/4] Basic multi-node support Signed-off-by: Ilya Dmitrichenko --- projects/kubernetes/Boxfile | 14 +--- projects/kubernetes/Makefile | 13 +++- projects/kubernetes/README.md | 15 +++- projects/kubernetes/boot-master.sh | 6 +- projects/kubernetes/boot-node.sh | 8 ++ projects/kubernetes/kube-node.yml | 120 +++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 17 deletions(-) create mode 100755 projects/kubernetes/boot-node.sh create mode 100644 projects/kubernetes/kube-node.yml diff --git a/projects/kubernetes/Boxfile b/projects/kubernetes/Boxfile index 19d26de64..bdb74422c 100644 --- a/projects/kubernetes/Boxfile +++ b/projects/kubernetes/Boxfile @@ -78,18 +78,12 @@ def create_shell_wrapper script, path run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}" end -def create_kubelet_wrapper - create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do sleep 1 ; done", '/usr/bin/kubelet.sh' -end - -def create_kubeadm_wrappers - create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh' -end - install_dependencies -create_kubelet_wrapper -create_kubeadm_wrappers +# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume +create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh' + +create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh' flatten diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index e19c48d15..873e4190a 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,23 +1,28 @@ all: build-container-images build-vm-images -build-container-images: Boxfile +build-container-image: Boxfile docker run --rm -ti \ -v $(PWD):$(PWD) \ -v /var/run/docker.sock:/var/run/docker.sock \ -w $(PWD) \ boxbuilder/box:master Boxfile -push-container-images: build-container-image +push-container-images: build-container-image cache-images docker image push mobylinux/kubernetes:latest docker image push mobylinux/kubernetes:latest-image-cache-common docker image push mobylinux/kubernetes:latest-image-cache-control-plane -build-vm-images: +build-vm-images: kube-master-initrd.img kube-node-initrd.img + +kube-master-initrd.img: kube-master.yml ../../bin/moby build -name kube-master kube-master.yml +kube-node-initrd.img: kube-node.yml + ../../bin/moby build -name kube-node kube-node.yml + clean: rm -f -r \ - kube-master-bzImage kube-master-cmdline kube-master-disk.img kube-master-initrd.img \ + kube-*-bzImage kube-*-cmdline kube-*-disk.img kube-*-initrd.img \ image-cache/common image-cache/control-plane COMMON_IMAGES := \ diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index 33b2e936c..ce179654a 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -19,4 +19,17 @@ Manually initialise master with `kubeadm`: runc exec kubelet kubeadm-init.sh ``` -Once `kubeadm` exits, try `runc exec kubelet kubectl get nodes`. +Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments, +and try `runc exec kubelet kubectl get nodes`. + +To boot a node use: +``` +./boot-node.sh [ ...] +``` + +More specifically, to start 3 nodes use 3 separate shells and run this: +``` +shell1> ./boot-node.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 +shell2> ./boot-node.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 +shell3> ./boot-node.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 +``` diff --git a/projects/kubernetes/boot-master.sh b/projects/kubernetes/boot-master.sh index 09b7fa866..6798caa8b 100755 --- a/projects/kubernetes/boot-master.sh +++ b/projects/kubernetes/boot-master.sh @@ -1,3 +1,5 @@ -#!/bin/bash -eux -rm -f kube-master-disk.img +#!/bin/bash -eu +disk="kube-master-disk.img" +set -x +rm -f "${disk}" ../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master diff --git a/projects/kubernetes/boot-node.sh b/projects/kubernetes/boot-node.sh new file mode 100755 index 000000000..76ca817f6 --- /dev/null +++ b/projects/kubernetes/boot-node.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eu +[ "${#@}" -gt 1 ] || (echo "Usage: ${0} " ; exit 1) +name="node-${1}" +shift +disk="kube-${name}-disk.img" +set -x +rm -f "${disk}" +../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml new file mode 100644 index 000000000..855df2c06 --- /dev/null +++ b/projects/kubernetes/kube-node.yml @@ -0,0 +1,120 @@ +kernel: + image: "mobylinux/kernel:4.9.x" + cmdline: "console=ttyS0 console=tty0 page_poison=1" +init: + - mobylinux/init:e10e2efc1b78ef41d196175cbc07e069391f406e + - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9 + - mobylinux/containerd:18eaf72f3f4f9a9f29ca1951f66df701f873060b + - mobylinux/ca-certificates:eabc5a6e59f05aa91529d80e9a595b85b046f935 +onboot: + - name: sysctl + image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c" + net: host + pid: host + ipc: host + capabilities: + - CAP_SYS_ADMIN + readonly: true + - name: sysfs + image: mobylinux/sysfs:6c1d06f28ddd9681799d3950cddf044b930b221c + - name: binfmt + image: "mobylinux/binfmt:bdb754f25a5d851b4f5f8d185a43dfcbb3c22d01" + binds: + - /proc/sys/fs/binfmt_misc:/binfmt_misc + readonly: true + - name: format + image: "mobylinux/format:53748000acf515549d398e6ae68545c26c0f3a2e" + binds: + - /dev:/dev + capabilities: + - CAP_SYS_ADMIN + - CAP_MKNOD + - name: mount + image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + binds: + - /dev:/dev + - /var:/var:rshared,rbind + capabilities: + - CAP_SYS_ADMIN + rootfsPropagation: shared + command: ["/mount.sh", "/var/lib"] + - name: mount + image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + binds: + - /var:/var + command: ["mkdir", "-p", "/var/lib/kubeadm"] +services: + - name: rngd + image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92" + capabilities: + - CAP_SYS_ADMIN + oomScoreAdj: -800 + readonly: true + - name: dhcpcd + image: "mobylinux/dhcpcd:57a8ef29d3a910645b2b24c124f9ce9ef53ce703" + binds: + - /var:/var + - /tmp/etc:/etc + capabilities: + - CAP_NET_ADMIN + - CAP_NET_BIND_SERVICE + - CAP_NET_RAW + net: host + oomScoreAdj: -800 + - name: ntpd + image: "mobylinux/openntpd:a570316d7fc49ca1daa29bd945499f4963d227af" + capabilities: + - CAP_SYS_TIME + - CAP_SYS_NICE + - CAP_SYS_CHROOT + - CAP_SETUID + - CAP_SETGID + net: host + - name: sshd + image: "mobylinux/sshd:160631d59fffc13d523ff7f09b3b49538d34b9cd" + capabilities: + - all + net: host + pid: host + binds: + - /root/.ssh:/root/.ssh + - name: docker + image: "mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08" + capabilities: + - all + net: host + pid: host + mounts: + - type: cgroup + options: ["rw","nosuid","noexec","nodev","relatime"] + binds: + - /dev:/dev + - /lib/modules:/lib/modules + - /var:/var:rshared,rbind + - /var/lib/kubeadm:/etc/kubernetes + rootfsPropagation: shared + - name: kubernetes-image-cache-common + image: "mobylinux/kubernetes:latest-image-cache-common" + binds: + - /var/run:/var/run + - name: kubelet + image: "mobylinux/kubernetes:latest" + capabilities: + - all + net: host + pid: host + mounts: + - type: cgroup + options: ["rw","nosuid","noexec","nodev","relatime"] + binds: + - /dev:/dev + - /var:/var:rshared,rbind + - /var/lib/kubeadm:/etc/kubernetes + #- /var/log:/var/log #/var/log/containers + # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared + rootfsPropagation: shared +files: + - path: root/.ssh/authorized_keys + contents: '# Your ssh key goes here' +outputs: + - format: kernel+initrd