diff --git a/projects/kubernetes/Boxfile b/projects/kubernetes/Boxfile index 19d26de64..bdb74422c 100644 --- a/projects/kubernetes/Boxfile +++ b/projects/kubernetes/Boxfile @@ -78,18 +78,12 @@ def create_shell_wrapper script, path run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}" end -def create_kubelet_wrapper - create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do sleep 1 ; done", '/usr/bin/kubelet.sh' -end - -def create_kubeadm_wrappers - create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh' -end - install_dependencies -create_kubelet_wrapper -create_kubeadm_wrappers +# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume +create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh' + +create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh' flatten diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index e19c48d15..873e4190a 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,23 +1,28 @@ all: build-container-images build-vm-images -build-container-images: Boxfile +build-container-image: Boxfile docker run --rm -ti \ -v $(PWD):$(PWD) \ -v /var/run/docker.sock:/var/run/docker.sock \ -w $(PWD) \ boxbuilder/box:master Boxfile -push-container-images: build-container-image +push-container-images: build-container-image cache-images docker image push mobylinux/kubernetes:latest docker image push mobylinux/kubernetes:latest-image-cache-common docker image push mobylinux/kubernetes:latest-image-cache-control-plane -build-vm-images: +build-vm-images: kube-master-initrd.img kube-node-initrd.img + +kube-master-initrd.img: kube-master.yml ../../bin/moby build -name kube-master kube-master.yml +kube-node-initrd.img: kube-node.yml + ../../bin/moby build -name kube-node kube-node.yml + clean: rm -f -r \ - kube-master-bzImage kube-master-cmdline kube-master-disk.img kube-master-initrd.img \ + kube-*-bzImage kube-*-cmdline kube-*-disk.img kube-*-initrd.img \ image-cache/common image-cache/control-plane COMMON_IMAGES := \ diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index 33b2e936c..ce179654a 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -19,4 +19,17 @@ Manually initialise master with `kubeadm`: runc exec kubelet kubeadm-init.sh ``` -Once `kubeadm` exits, try `runc exec kubelet kubectl get nodes`. +Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments, +and try `runc exec kubelet kubectl get nodes`. + +To boot a node use: +``` +./boot-node.sh [ ...] +``` + +More specifically, to start 3 nodes use 3 separate shells and run this: +``` +shell1> ./boot-node.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 +shell2> ./boot-node.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 +shell3> ./boot-node.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 +``` diff --git a/projects/kubernetes/boot-master.sh b/projects/kubernetes/boot-master.sh index 09b7fa866..6798caa8b 100755 --- a/projects/kubernetes/boot-master.sh +++ b/projects/kubernetes/boot-master.sh @@ -1,3 +1,5 @@ -#!/bin/bash -eux -rm -f kube-master-disk.img +#!/bin/bash -eu +disk="kube-master-disk.img" +set -x +rm -f "${disk}" ../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master diff --git a/projects/kubernetes/boot-node.sh b/projects/kubernetes/boot-node.sh new file mode 100755 index 000000000..76ca817f6 --- /dev/null +++ b/projects/kubernetes/boot-node.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eu +[ "${#@}" -gt 1 ] || (echo "Usage: ${0} " ; exit 1) +name="node-${1}" +shift +disk="kube-${name}-disk.img" +set -x +rm -f "${disk}" +../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml new file mode 100644 index 000000000..855df2c06 --- /dev/null +++ b/projects/kubernetes/kube-node.yml @@ -0,0 +1,120 @@ +kernel: + image: "mobylinux/kernel:4.9.x" + cmdline: "console=ttyS0 console=tty0 page_poison=1" +init: + - mobylinux/init:e10e2efc1b78ef41d196175cbc07e069391f406e + - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9 + - mobylinux/containerd:18eaf72f3f4f9a9f29ca1951f66df701f873060b + - mobylinux/ca-certificates:eabc5a6e59f05aa91529d80e9a595b85b046f935 +onboot: + - name: sysctl + image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c" + net: host + pid: host + ipc: host + capabilities: + - CAP_SYS_ADMIN + readonly: true + - name: sysfs + image: mobylinux/sysfs:6c1d06f28ddd9681799d3950cddf044b930b221c + - name: binfmt + image: "mobylinux/binfmt:bdb754f25a5d851b4f5f8d185a43dfcbb3c22d01" + binds: + - /proc/sys/fs/binfmt_misc:/binfmt_misc + readonly: true + - name: format + image: "mobylinux/format:53748000acf515549d398e6ae68545c26c0f3a2e" + binds: + - /dev:/dev + capabilities: + - CAP_SYS_ADMIN + - CAP_MKNOD + - name: mount + image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + binds: + - /dev:/dev + - /var:/var:rshared,rbind + capabilities: + - CAP_SYS_ADMIN + rootfsPropagation: shared + command: ["/mount.sh", "/var/lib"] + - name: mount + image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + binds: + - /var:/var + command: ["mkdir", "-p", "/var/lib/kubeadm"] +services: + - name: rngd + image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92" + capabilities: + - CAP_SYS_ADMIN + oomScoreAdj: -800 + readonly: true + - name: dhcpcd + image: "mobylinux/dhcpcd:57a8ef29d3a910645b2b24c124f9ce9ef53ce703" + binds: + - /var:/var + - /tmp/etc:/etc + capabilities: + - CAP_NET_ADMIN + - CAP_NET_BIND_SERVICE + - CAP_NET_RAW + net: host + oomScoreAdj: -800 + - name: ntpd + image: "mobylinux/openntpd:a570316d7fc49ca1daa29bd945499f4963d227af" + capabilities: + - CAP_SYS_TIME + - CAP_SYS_NICE + - CAP_SYS_CHROOT + - CAP_SETUID + - CAP_SETGID + net: host + - name: sshd + image: "mobylinux/sshd:160631d59fffc13d523ff7f09b3b49538d34b9cd" + capabilities: + - all + net: host + pid: host + binds: + - /root/.ssh:/root/.ssh + - name: docker + image: "mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08" + capabilities: + - all + net: host + pid: host + mounts: + - type: cgroup + options: ["rw","nosuid","noexec","nodev","relatime"] + binds: + - /dev:/dev + - /lib/modules:/lib/modules + - /var:/var:rshared,rbind + - /var/lib/kubeadm:/etc/kubernetes + rootfsPropagation: shared + - name: kubernetes-image-cache-common + image: "mobylinux/kubernetes:latest-image-cache-common" + binds: + - /var/run:/var/run + - name: kubelet + image: "mobylinux/kubernetes:latest" + capabilities: + - all + net: host + pid: host + mounts: + - type: cgroup + options: ["rw","nosuid","noexec","nodev","relatime"] + binds: + - /dev:/dev + - /var:/var:rshared,rbind + - /var/lib/kubeadm:/etc/kubernetes + #- /var/log:/var/log #/var/log/containers + # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared + rootfsPropagation: shared +files: + - path: root/.ssh/authorized_keys + contents: '# Your ssh key goes here' +outputs: + - format: kernel+initrd