Merge pull request #1585 from errordeveloper/kubernetes

Basic multi-node Kubernetes cluster
2025-07-27 20:48:11 +00:00 · 2017-04-11 16:17:31 +01:00 · 2017-04-11 16:17:31 +01:00 · 25f8036bc4
commit 25f8036bc4
parent 9cf63a4aaa 52ba02b9c8
9 changed files with 233 additions and 23 deletions
--- a/projects/kubernetes/.gitignore
+++ b/projects/kubernetes/.gitignore
@ -0,0 +1,4 @@
+image-cache/common/*.tar
+image-cache/common/Dockerfile
+image-cache/control-plane/*.tar
+image-cache/control-plane/Dockerfile
--- a/projects/kubernetes/Boxfile
+++ b/projects/kubernetes/Boxfile
@ -78,19 +78,12 @@ def create_shell_wrapper script, path
  run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
 end

-def create_kubelet_wrapper
-  create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do sleep 1 ; done", '/usr/bin/kubelet.sh'
-end
-
-def create_kubeadm_wrappers
-  #create_shell_wrapper 'env KUBE_KUBERNETES_DIR=/var/lib/kubeadm kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 --cert-dir /var/lib/kubeadm/pki', '/usr/bin/kubeadm-init.sh'
-  create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1", '/usr/bin/kubeadm-init.sh'
-end
-
 install_dependencies

-create_kubelet_wrapper
-create_kubeadm_wrappers
+# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
+create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh'
+
+create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh'

 flatten

--- a/projects/kubernetes/Makefile
+++ b/projects/kubernetes/Makefile
@ -1,17 +1,56 @@
 all: build-container-images build-vm-images

-build-container-images: Boxfile
+build-container-image: Boxfile
 	docker run --rm -ti \
 	  -v $(PWD):$(PWD) \
 	  -v /var/run/docker.sock:/var/run/docker.sock \
 	  -w $(PWD) \
 	    boxbuilder/box:master Boxfile

-push-container-images: build-container-image
-	docker push mobylinux/kubernetes:latest
+push-container-images: build-container-image cache-images
+	docker image push mobylinux/kubernetes:latest
+	docker image push mobylinux/kubernetes:latest-image-cache-common
+	docker image push mobylinux/kubernetes:latest-image-cache-control-plane

-build-vm-images:
+build-vm-images: kube-master-initrd.img kube-node-initrd.img
+
+kube-master-initrd.img: kube-master.yml
 	../../bin/moby build -name kube-master kube-master.yml

+kube-node-initrd.img: kube-node.yml
+	../../bin/moby build -name kube-node kube-node.yml
+
 clean:
-	rm -f kube-master-bzImage kube-master-cmdline kube-master-disk.img kube-master-initrd.img 
+	rm -f -r \
+	  kube-*-bzImage kube-*-cmdline kube-*-disk.img kube-*-initrd.img \
+	  image-cache/common image-cache/control-plane
+
+COMMON_IMAGES := \
+  kube-proxy-amd64:v1.6.1 \
+  k8s-dns-sidecar-amd64:1.14.1 \
+  k8s-dns-kube-dns-amd64:1.14.1 \
+  k8s-dns-dnsmasq-nanny-amd64:1.14.1 \
+  pause-amd64:3.0
+
+CONTROL_PLANE_IMAGES := \
+  kube-apiserver-amd64:v1.6.1 \
+  kube-controller-manager-amd64:v1.6.1 \
+  kube-scheduler-amd64:v1.6.1 \
+  etcd-amd64:3.0.17
+
+image-cache/%.tar:
+	mkdir -p $(dir $@)
+	DOCKER_CONTENT_TRUST=1 docker image pull gcr.io/google_containers/$(shell basename $@ .tar)
+	docker image save -o $@ gcr.io/google_containers/$(shell basename $@ .tar)
+
+cache-images:
+	for image in $(COMMON_IMAGES) ; \
+	  do $(MAKE) "image-cache/common/$${image}.tar" \
+	  ; done
+	cp image-cache/Dockerfile image-cache/common
+	docker image build -t mobylinux/kubernetes:latest-image-cache-common image-cache/common
+	for image in $(CONTROL_PLANE_IMAGES) ; \
+	  do $(MAKE) "image-cache/control-plane/$${image}.tar" \
+	  ; done
+	cp image-cache/Dockerfile image-cache/control-plane
+	docker image build -t mobylinux/kubernetes:latest-image-cache-control-plane image-cache/control-plane
--- a/projects/kubernetes/README.md
+++ b/projects/kubernetes/README.md
@ -11,7 +11,7 @@ make

 Boot Kubernetes master OS image using `hyperkit` on macOS:
 ```
-../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master
+./boot-master.sh
 ```

 Manually initialise master with `kubeadm`:
@ -19,4 +19,17 @@ Manually initialise master with `kubeadm`:
 runc exec kubelet kubeadm-init.sh
 ```

-Once `kubeadm` exits, try `runc exec kubelet kubectl get nodes`.
+Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments,
+and try `runc exec kubelet kubectl get nodes`.
+
+To boot a node use:
+```
+./boot-node.sh <n> [<join_args> ...]
+```
+
+More specifically, to start 3 nodes use 3 separate shells and run this:
+```
+shell1> ./boot-node.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
+shell2> ./boot-node.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
+shell3> ./boot-node.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
+```
--- a/projects/kubernetes/boot-master.sh
+++ b/projects/kubernetes/boot-master.sh
@ -0,0 +1,5 @@
+#!/bin/bash -eu
+disk="kube-master-disk.img"
+set -x
+rm -f "${disk}"
+../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master
--- a/projects/kubernetes/boot-node.sh
+++ b/projects/kubernetes/boot-node.sh
@ -0,0 +1,8 @@
+#!/bin/bash -eu
+[ "${#@}" -gt 1 ] || (echo "Usage: ${0} <node> <join_args>" ; exit 1)
+name="node-${1}"
+shift
+disk="kube-${name}-disk.img"
+set -x
+rm -f "${disk}"
+../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node
--- a/projects/kubernetes/image-cache/Dockerfile
+++ b/projects/kubernetes/image-cache/Dockerfile
@ -0,0 +1,6 @@
+FROM mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08
+
+ADD . /images
+
+ENTRYPOINT [ "/bin/sh", "-c" ]
+CMD [ "for image in /images/*.tar ; do docker image load -i $image ; done" ]
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@ -2,9 +2,9 @@ kernel:
  image: "mobylinux/kernel:4.9.x"
  cmdline: "console=ttyS0 console=tty0 page_poison=1"
 init:
-  - mobylinux/init:671bdce1ed0803daeb35e83e4bcd576bb449ea35
+  - mobylinux/init:e10e2efc1b78ef41d196175cbc07e069391f406e
  - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9
-  - mobylinux/containerd:c7f6ecdcbcb615a53edee556ba03c7c873bc8488
+  - mobylinux/containerd:18eaf72f3f4f9a9f29ca1951f66df701f873060b
  - mobylinux/ca-certificates:eabc5a6e59f05aa91529d80e9a595b85b046f935
 onboot:
  - name: sysctl
@ -15,6 +15,8 @@ onboot:
    capabilities:
     - CAP_SYS_ADMIN
    readonly: true
+  - name: sysfs
+    image: mobylinux/sysfs:6c1d06f28ddd9681799d3950cddf044b930b221c
  - name: binfmt
    image: "mobylinux/binfmt:bdb754f25a5d851b4f5f8d185a43dfcbb3c22d01"
    binds:
@ -36,6 +38,11 @@ onboot:
     - CAP_SYS_ADMIN
    rootfsPropagation: shared
    command: ["/mount.sh", "/var/lib"]
+  - name: mount
+    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    binds:
+     - /var:/var
+    command: ["mkdir", "-p", "/var/lib/kubeadm"]
 services:
  - name: rngd
    image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
@ -54,6 +61,15 @@ services:
     - CAP_NET_RAW
    net: host
    oomScoreAdj: -800
+  - name: ntpd
+    image: "mobylinux/openntpd:a570316d7fc49ca1daa29bd945499f4963d227af"
+    capabilities:
+      - CAP_SYS_TIME
+      - CAP_SYS_NICE
+      - CAP_SYS_CHROOT
+      - CAP_SETUID
+      - CAP_SETGID
+    net: host
  - name: sshd
    image: "mobylinux/sshd:160631d59fffc13d523ff7f09b3b49538d34b9cd"
    capabilities:
@ -77,8 +93,16 @@ services:
     - /var:/var:rshared,rbind
     - /var/lib/kubeadm:/etc/kubernetes
    rootfsPropagation: shared
+  - name: kubernetes-image-cache-common
+    image: "mobylinux/kubernetes:latest-image-cache-common"
+    binds:
+      - /var/run:/var/run
+  - name: kubernetes-image-cache-control-plane
+    image: "mobylinux/kubernetes:latest-image-cache-control-plane"
+    binds:
+      - /var/run:/var/run
  - name: kubelet
-    image: "errordeveloper/mobykube:master"
+    image: "mobylinux/kubernetes:latest"
    capabilities:
     - all
    net: host
@ -96,7 +120,5 @@ services:
 files:
  - path: root/.ssh/authorized_keys
    contents: '# Your ssh key goes here'
-  - path: var/lib/kubeadm
-    directory: true
 outputs:
  - format: kernel+initrd
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@ -0,0 +1,120 @@
+kernel:
+  image: "mobylinux/kernel:4.9.x"
+  cmdline: "console=ttyS0 console=tty0 page_poison=1"
+init:
+  - mobylinux/init:e10e2efc1b78ef41d196175cbc07e069391f406e
+  - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9
+  - mobylinux/containerd:18eaf72f3f4f9a9f29ca1951f66df701f873060b
+  - mobylinux/ca-certificates:eabc5a6e59f05aa91529d80e9a595b85b046f935
+onboot:
+  - name: sysctl
+    image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c"
+    net: host
+    pid: host
+    ipc: host
+    capabilities:
+     - CAP_SYS_ADMIN
+    readonly: true
+  - name: sysfs
+    image: mobylinux/sysfs:6c1d06f28ddd9681799d3950cddf044b930b221c
+  - name: binfmt
+    image: "mobylinux/binfmt:bdb754f25a5d851b4f5f8d185a43dfcbb3c22d01"
+    binds:
+     - /proc/sys/fs/binfmt_misc:/binfmt_misc
+    readonly: true
+  - name: format
+    image: "mobylinux/format:53748000acf515549d398e6ae68545c26c0f3a2e"
+    binds:
+     - /dev:/dev
+    capabilities:
+     - CAP_SYS_ADMIN
+     - CAP_MKNOD
+  - name: mount
+    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    binds:
+     - /dev:/dev
+     - /var:/var:rshared,rbind
+    capabilities:
+     - CAP_SYS_ADMIN
+    rootfsPropagation: shared
+    command: ["/mount.sh", "/var/lib"]
+  - name: mount
+    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    binds:
+     - /var:/var
+    command: ["mkdir", "-p", "/var/lib/kubeadm"]
+services:
+  - name: rngd
+    image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
+    capabilities:
+     - CAP_SYS_ADMIN
+    oomScoreAdj: -800
+    readonly: true
+  - name: dhcpcd
+    image: "mobylinux/dhcpcd:57a8ef29d3a910645b2b24c124f9ce9ef53ce703"
+    binds:
+     - /var:/var
+     - /tmp/etc:/etc
+    capabilities:
+     - CAP_NET_ADMIN
+     - CAP_NET_BIND_SERVICE
+     - CAP_NET_RAW
+    net: host
+    oomScoreAdj: -800
+  - name: ntpd
+    image: "mobylinux/openntpd:a570316d7fc49ca1daa29bd945499f4963d227af"
+    capabilities:
+      - CAP_SYS_TIME
+      - CAP_SYS_NICE
+      - CAP_SYS_CHROOT
+      - CAP_SETUID
+      - CAP_SETGID
+    net: host
+  - name: sshd
+    image: "mobylinux/sshd:160631d59fffc13d523ff7f09b3b49538d34b9cd"
+    capabilities:
+     - all
+    net: host
+    pid: host
+    binds:
+     - /root/.ssh:/root/.ssh
+  - name: docker
+    image: "mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08"
+    capabilities:
+     - all
+    net: host
+    pid: host
+    mounts:
+     - type: cgroup
+       options: ["rw","nosuid","noexec","nodev","relatime"]
+    binds:
+     - /dev:/dev
+     - /lib/modules:/lib/modules
+     - /var:/var:rshared,rbind
+     - /var/lib/kubeadm:/etc/kubernetes
+    rootfsPropagation: shared
+  - name: kubernetes-image-cache-common
+    image: "mobylinux/kubernetes:latest-image-cache-common"
+    binds:
+      - /var/run:/var/run
+  - name: kubelet
+    image: "mobylinux/kubernetes:latest"
+    capabilities:
+     - all
+    net: host
+    pid: host
+    mounts:
+     - type: cgroup
+       options: ["rw","nosuid","noexec","nodev","relatime"]
+    binds:
+     - /dev:/dev
+     - /var:/var:rshared,rbind
+     - /var/lib/kubeadm:/etc/kubernetes
+     #- /var/log:/var/log #/var/log/containers
+     # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
+    rootfsPropagation: shared
+files:
+  - path: root/.ssh/authorized_keys
+    contents: '# Your ssh key goes here'
+outputs:
+  - format: kernel+initrd