Basic multi-node support

Signed-off-by: Ilya Dmitrichenko <errordeveloper@gmail.com>

projects/kubernetes/Boxfile

@@ -78,18 +78,12 @@ def create_shell_wrapper script, path
   run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
 end
 
-def create_kubelet_wrapper
-  create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do sleep 1 ; done", '/usr/bin/kubelet.sh'
-end
-
-def create_kubeadm_wrappers
-  create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh'
-end
-
 install_dependencies
 
-create_kubelet_wrapper
+# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
-create_kubeadm_wrappers
+create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh'
+
+create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh'
 
 flatten
 

projects/kubernetes/Makefile

@@ -1,23 +1,28 @@
 all: build-container-images build-vm-images
 
-build-container-images: Boxfile
+build-container-image: Boxfile
 	docker run --rm -ti \
 	  -v $(PWD):$(PWD) \
 	  -v /var/run/docker.sock:/var/run/docker.sock \
 	  -w $(PWD) \
 	    boxbuilder/box:master Boxfile
 
-push-container-images: build-container-image
+push-container-images: build-container-image cache-images
 	docker image push mobylinux/kubernetes:latest
 	docker image push mobylinux/kubernetes:latest-image-cache-common
 	docker image push mobylinux/kubernetes:latest-image-cache-control-plane
 
-build-vm-images:
+build-vm-images: kube-master-initrd.img kube-node-initrd.img
+
+kube-master-initrd.img: kube-master.yml
 	../../bin/moby build -name kube-master kube-master.yml
 
+kube-node-initrd.img: kube-node.yml
+	../../bin/moby build -name kube-node kube-node.yml
+
 clean:
 	rm -f -r \
-	  kube-master-bzImage kube-master-cmdline kube-master-disk.img kube-master-initrd.img \
+	  kube-*-bzImage kube-*-cmdline kube-*-disk.img kube-*-initrd.img \
 	  image-cache/common image-cache/control-plane
 
 COMMON_IMAGES := \

projects/kubernetes/README.md

@@ -19,4 +19,17 @@ Manually initialise master with `kubeadm`:
 runc exec kubelet kubeadm-init.sh
 ```
 
-Once `kubeadm` exits, try `runc exec kubelet kubectl get nodes`.
+Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments,
+and try `runc exec kubelet kubectl get nodes`.
+
+To boot a node use:
+```
+./boot-node.sh <n> [<join_args> ...]
+```
+
+More specifically, to start 3 nodes use 3 separate shells and run this:
+```
+shell1> ./boot-node.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
+shell2> ./boot-node.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
+shell3> ./boot-node.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
+```

projects/kubernetes/boot-master.sh

@@ -1,3 +1,5 @@
-#!/bin/bash -eux
+#!/bin/bash -eu
-rm -f kube-master-disk.img
+disk="kube-master-disk.img"
+set -x
+rm -f "${disk}"
 ../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master

projects/kubernetes/boot-node.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -eu
+[ "${#@}" -gt 1 ] || (echo "Usage: ${0} <node> <join_args>" ; exit 1)
+name="node-${1}"
+shift
+disk="kube-${name}-disk.img"
+set -x
+rm -f "${disk}"
+../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node

projects/kubernetes/kube-node.yml

@@ -0,0 +1,120 @@
+kernel:
+  image: "mobylinux/kernel:4.9.x"
+  cmdline: "console=ttyS0 console=tty0 page_poison=1"
+init:
+  - mobylinux/init:e10e2efc1b78ef41d196175cbc07e069391f406e
+  - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9
+  - mobylinux/containerd:18eaf72f3f4f9a9f29ca1951f66df701f873060b
+  - mobylinux/ca-certificates:eabc5a6e59f05aa91529d80e9a595b85b046f935
+onboot:
+  - name: sysctl
+    image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c"
+    net: host
+    pid: host
+    ipc: host
+    capabilities:
+     - CAP_SYS_ADMIN
+    readonly: true
+  - name: sysfs
+    image: mobylinux/sysfs:6c1d06f28ddd9681799d3950cddf044b930b221c
+  - name: binfmt
+    image: "mobylinux/binfmt:bdb754f25a5d851b4f5f8d185a43dfcbb3c22d01"
+    binds:
+     - /proc/sys/fs/binfmt_misc:/binfmt_misc
+    readonly: true
+  - name: format
+    image: "mobylinux/format:53748000acf515549d398e6ae68545c26c0f3a2e"
+    binds:
+     - /dev:/dev
+    capabilities:
+     - CAP_SYS_ADMIN
+     - CAP_MKNOD
+  - name: mount
+    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    binds:
+     - /dev:/dev
+     - /var:/var:rshared,rbind
+    capabilities:
+     - CAP_SYS_ADMIN
+    rootfsPropagation: shared
+    command: ["/mount.sh", "/var/lib"]
+  - name: mount
+    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    binds:
+     - /var:/var
+    command: ["mkdir", "-p", "/var/lib/kubeadm"]
+services:
+  - name: rngd
+    image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
+    capabilities:
+     - CAP_SYS_ADMIN
+    oomScoreAdj: -800
+    readonly: true
+  - name: dhcpcd
+    image: "mobylinux/dhcpcd:57a8ef29d3a910645b2b24c124f9ce9ef53ce703"
+    binds:
+     - /var:/var
+     - /tmp/etc:/etc
+    capabilities:
+     - CAP_NET_ADMIN
+     - CAP_NET_BIND_SERVICE
+     - CAP_NET_RAW
+    net: host
+    oomScoreAdj: -800
+  - name: ntpd
+    image: "mobylinux/openntpd:a570316d7fc49ca1daa29bd945499f4963d227af"
+    capabilities:
+      - CAP_SYS_TIME
+      - CAP_SYS_NICE
+      - CAP_SYS_CHROOT
+      - CAP_SETUID
+      - CAP_SETGID
+    net: host
+  - name: sshd
+    image: "mobylinux/sshd:160631d59fffc13d523ff7f09b3b49538d34b9cd"
+    capabilities:
+     - all
+    net: host
+    pid: host
+    binds:
+     - /root/.ssh:/root/.ssh
+  - name: docker
+    image: "mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08"
+    capabilities:
+     - all
+    net: host
+    pid: host
+    mounts:
+     - type: cgroup
+       options: ["rw","nosuid","noexec","nodev","relatime"]
+    binds:
+     - /dev:/dev
+     - /lib/modules:/lib/modules
+     - /var:/var:rshared,rbind
+     - /var/lib/kubeadm:/etc/kubernetes
+    rootfsPropagation: shared
+  - name: kubernetes-image-cache-common
+    image: "mobylinux/kubernetes:latest-image-cache-common"
+    binds:
+      - /var/run:/var/run
+  - name: kubelet
+    image: "mobylinux/kubernetes:latest"
+    capabilities:
+     - all
+    net: host
+    pid: host
+    mounts:
+     - type: cgroup
+       options: ["rw","nosuid","noexec","nodev","relatime"]
+    binds:
+     - /dev:/dev
+     - /var:/var:rshared,rbind
+     - /var/lib/kubeadm:/etc/kubernetes
+     #- /var/log:/var/log #/var/log/containers
+     # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
+    rootfsPropagation: shared
+files:
+  - path: root/.ssh/authorized_keys
+    contents: '# Your ssh key goes here'
+outputs:
+  - format: kernel+initrd