Kubernetes with Weave CNI networking

For the time being I've decided to exploit shared mounts to make
`/opt/cni` and `/etc/cni` work as expected. We need these directories
to appear writable on the host, and allow Weave Net pod to bind-mount
out them in order to install plugin binaries, and allow for vanilla
CNI plugins to be also accessible to kubelet.

Signed-off-by: Ilya Dmitrichenko <errordeveloper@gmail.com>

projects/kubernetes/Boxfile

@@ -1,94 +0,0 @@
-@image_name = "mobylinux/kubernetes"
-
-@versions = {
-  kubernetes: "v1.6.1",
-  weave: "v1.9.4",
-  tini: "v0.14.0",
-}
-
-from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}"
-
-def install_packages pkgs
-  cmds = [
-    %(apt-get update -q),
-    %(apt-get upgrade -qy),
-    %(apt-get install -qy #{pkgs}),
-  ]
-
-  cmds.each { |cmd| run cmd }
-end
-
-def install_dependencies
-
-  prepare = [
-    'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -',
-    'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list',
-  ]
-
-  dependencies = %(curl apt-transport-https)
-
-  official_packages = %(kubernetes-cni)
-
-  kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64"
-  weave_launcher = "https://cloud.weave.works/k8s/v1.6/net.yaml?v=#{@versions[:weave]}"
-
-  download_files = [
-    "/etc/weave.yaml" => {
-      url: weave_launcher,
-      mode: '0644',
-    },
-    "/usr/bin/kubeadm" => {
-      url: "#{kube_release_artefacts}/kubeadm",
-      mode: '0755',
-    },
-    "/usr/bin/tini" => {
-      url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini",
-      mode: '0755',
-    },
-  ]
-
-  install_packages dependencies
-
-  prepare.each { |cmd| run cmd }
-
-  install_packages official_packages
-
-  download_files.each do |file|
-    file.each do |dest,info|
-      run %(curl --output "#{dest}" --fail --silent --location "#{info[:url]}")
-      run %(chmod "#{info[:mode]}" "#{dest}")
-    end
-  end
-
-end
-
-def kubelet_cmd
-  %w(
-    /hyperkube kubelet
-      --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true
-      --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true
-      --cluster-dns=10.96.0.10 --cluster-domain=cluster.local
-      --cgroups-per-qos=false --enforce-node-allocatable=""
-  )
-  #--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
-  #--node-ip="192.168.65.2"
-end
-
-def create_shell_wrapper script, path
-  run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
-end
-
-install_dependencies
-
-# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
-create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh'
-
-create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh'
-
-flatten
-
-env KUBECONFIG: "/etc/kubernetes/admin.conf"
-
-set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh)
-
-tag "#{@image_name}:latest"

projects/kubernetes/Makefile

@@ -1,14 +1,19 @@
 all: build-container-images build-vm-images
 
-build-container-image: Boxfile
+BOX_PLANS = kubernetes.rb mounts.rb
+
+build-container-images: $(BOX_PLANS)
+	for plan in $(BOX_PLANS) ; do \
 	  docker run --rm -ti \
 	    -v $(PWD):$(PWD) \
 	    -v /var/run/docker.sock:/var/run/docker.sock \
 	    -w $(PWD) \
-	    boxbuilder/box:master Boxfile
+	      boxbuilder/box:master $$plan \
+	; done
 
-push-container-images: build-container-image cache-images
+push-container-images: build-container-images cache-images
 	docker image push mobylinux/kubernetes:latest
+	docker image push mobylinux/kubernetes:latest-mounts
 	docker image push mobylinux/kubernetes:latest-image-cache-common
 	docker image push mobylinux/kubernetes:latest-image-cache-control-plane
 
@@ -47,10 +52,10 @@ cache-images:
 	for image in $(COMMON_IMAGES) ; \
 	  do $(MAKE) "image-cache/common/$${image}.tar" \
 	  ; done
-	cp image-cache/Dockerfile image-cache/common
+	cp image-cache/Dockerfile image-cache/.dockerignore image-cache/common
 	docker image build -t mobylinux/kubernetes:latest-image-cache-common image-cache/common
 	for image in $(CONTROL_PLANE_IMAGES) ; \
 	  do $(MAKE) "image-cache/control-plane/$${image}.tar" \
 	  ; done
-	cp image-cache/Dockerfile image-cache/control-plane
+	cp image-cache/Dockerfile image-cache/.dockerignore image-cache/control-plane
 	docker image build -t mobylinux/kubernetes:latest-image-cache-control-plane image-cache/control-plane

projects/kubernetes/README.md

@@ -4,9 +4,11 @@ This project aims to demonstrate how one can create minimal and immutable Kubern
 
 Make sure to `cd projects/kubernetes` first.
 
-Build container & OS images:
+Edit `kube-master.yml` and add your public SSH key to `files` section.
+
+Build OS images:
 ```
-make
+make build-vm-images
 ```
 
 Boot Kubernetes master OS image using `hyperkit` on macOS:
@@ -14,9 +16,19 @@ Boot Kubernetes master OS image using `hyperkit` on macOS:
 ./boot-master.sh
 ```
 
+Get IP address of the master:
+```
+ip addr show dev eth0
+```
+
+Login to the kubelet container:
+```
+./ssh_into_kubelet.sh <master-ip>
+```
+
 Manually initialise master with `kubeadm`:
 ```
-runc exec kubelet kubeadm-init.sh
+kubeadm-init.sh
 ```
 
 Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments,

projects/kubernetes/boot-master.sh

@@ -2,4 +2,4 @@
 disk="kube-master-disk.img"
 set -x
 rm -f "${disk}"
-../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master
+../../bin/moby run hyperkit -cpus 2 -mem 6144 -disk-size 6144 kube-master

projects/kubernetes/boot-node.sh

@@ -5,4 +5,4 @@ shift
 disk="kube-${name}-disk.img"
 set -x
 rm -f "${disk}"
-../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node
+../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 1024 -disk "${disk}" -data "${*}" kube-node

projects/kubernetes/common.rb

@@ -0,0 +1,54 @@
+@image_name = "mobylinux/kubernetes"
+
+@versions = {
+  kubernetes: "v1.6.1",
+  weave: "v1.9.4",
+  tini: "v0.14.0",
+}
+
+def install_packages pkgs
+  cmds = [
+    %(apt-get update -q),
+    %(apt-get upgrade -qy),
+    %(apt-get install -qy #{pkgs}),
+  ]
+
+  cmds.each { |cmd| run cmd }
+end
+
+def setup_apt_config
+  prepare = [
+    'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -',
+    'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list',
+  ]
+
+  dependencies = %(curl apt-transport-https)
+
+  install_packages dependencies
+
+  prepare.each { |cmd| run cmd }
+end
+
+def create_shell_wrapper script, path
+  run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
+end
+
+def mount_bind src, dst
+  "mount --bind #{src} #{dst}"
+end
+
+def mount_bind_hostns_self mnt
+  "nsenter --mount=/proc/1/ns/mnt mount -- --bind #{mnt} #{mnt}"
+end
+
+def mount_make_hostns_rshared mnt
+  "nsenter --mount=/proc/1/ns/mnt mount -- --make-rshared #{mnt}"
+end
+
+def mount_persistent_disk mnt
+  "/mount.sh #{mnt}"
+end
+
+def mkdir_p dir
+  "mkdir -p #{dir}"
+end

projects/kubernetes/image-cache/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

projects/kubernetes/image-cache/Dockerfile

@@ -1,6 +1,4 @@
 FROM mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08
-
 ADD . /images
-
 ENTRYPOINT [ "/bin/sh", "-c" ]
-CMD [ "for image in /images/*.tar ; do docker image load -i $image ; done" ]
+CMD [ "for image in /images/*.tar ; do docker image load -i $image && rm -f $image ; done" ]

projects/kubernetes/kube-master.yml

@@ -29,20 +29,15 @@ onboot:
     capabilities:
      - CAP_SYS_ADMIN
      - CAP_MKNOD
-  - name: mount
+  - name: mounts
-    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    image: "mobylinux/kubernetes:latest-mounts"
+    capabilities:
+     - all
+    pid: host
+    rootfsPropagation: shared
     binds:
      - /dev:/dev
      - /var:/var:rshared,rbind
-    capabilities:
-     - CAP_SYS_ADMIN
-    rootfsPropagation: shared
-    command: ["/mount.sh", "/var/lib"]
-  - name: mount
-    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
-    binds:
-     - /var:/var
-    command: ["mkdir", "-p", "/var/lib/kubeadm"]
 services:
   - name: rngd
     image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
@@ -92,6 +87,8 @@ services:
      - /lib/modules:/lib/modules
      - /var:/var:rshared,rbind
      - /var/lib/kubeadm:/etc/kubernetes
+     - /etc/cni:/etc/cni:rshared,rbind
+     - /opt/cni:/opt/cni:rshared,rbind
     rootfsPropagation: shared
   - name: kubernetes-image-cache-common
     image: "mobylinux/kubernetes:latest-image-cache-common"
@@ -114,11 +111,13 @@ services:
      - /dev:/dev
      - /var:/var:rshared,rbind
      - /var/lib/kubeadm:/etc/kubernetes
-     #- /var/log:/var/log #/var/log/containers
+     - /etc/cni:/rootfs/etc/cni:rshared,rbind
-     # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
+     - /opt/cni:/rootfs/opt/cni:rshared,rbind
     rootfsPropagation: shared
 files:
   - path: root/.ssh/authorized_keys
     contents: '# Your ssh key goes here'
+  - {path: etc/cni, directory: true}
+  - {path: opt/cni, directory: true}
 outputs:
   - format: kernel+initrd

projects/kubernetes/kube-node.yml

@@ -29,20 +29,15 @@ onboot:
     capabilities:
      - CAP_SYS_ADMIN
      - CAP_MKNOD
-  - name: mount
+  - name: mounts
-    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+    image: "mobylinux/kubernetes:latest-mounts"
+    capabilities:
+     - all
+    pid: host
+    rootfsPropagation: shared
     binds:
      - /dev:/dev
      - /var:/var:rshared,rbind
-    capabilities:
-     - CAP_SYS_ADMIN
-    rootfsPropagation: shared
-    command: ["/mount.sh", "/var/lib"]
-  - name: mount
-    image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
-    binds:
-     - /var:/var
-    command: ["mkdir", "-p", "/var/lib/kubeadm"]
 services:
   - name: rngd
     image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
@@ -92,6 +87,8 @@ services:
      - /lib/modules:/lib/modules
      - /var:/var:rshared,rbind
      - /var/lib/kubeadm:/etc/kubernetes
+     - /etc/cni:/etc/cni:rshared,rbind
+     - /opt/cni:/opt/cni:rshared,rbind
     rootfsPropagation: shared
   - name: kubernetes-image-cache-common
     image: "mobylinux/kubernetes:latest-image-cache-common"
@@ -110,11 +107,13 @@ services:
      - /dev:/dev
      - /var:/var:rshared,rbind
      - /var/lib/kubeadm:/etc/kubernetes
-     #- /var/log:/var/log #/var/log/containers
+     - /etc/cni:/rootfs/etc/cni:rshared,rbind
-     # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
+     - /opt/cni:/rootfs/opt/cni:rshared,rbind
     rootfsPropagation: shared
 files:
   - path: root/.ssh/authorized_keys
     contents: '# Your ssh key goes here'
+  - {path: etc/cni, directory: true}
+  - {path: opt/cni, directory: true}
 outputs:
   - format: kernel+initrd

projects/kubernetes/kubernetes.rb

@@ -0,0 +1,68 @@
+import 'common.rb'
+
+from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}"
+
+def install_node_dependencies
+  kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64"
+  weave_launcher = "https://frontend.dev.weave.works/k8s/v1.6/net?v=#{@versions[:weave]}"
+
+  download_files = [
+    "/etc/weave.yaml" => {
+      url: weave_launcher,
+      mode: '0644',
+    },
+    "/usr/bin/kubeadm" => {
+      url: "#{kube_release_artefacts}/kubeadm",
+      mode: '0755',
+    },
+    "/usr/bin/tini" => {
+      url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini",
+      mode: '0755',
+    },
+  ]
+
+  download_files.each do |file|
+    file.each do |dest,info|
+      run %(curl --insecure --output "#{dest}" --fail --silent --location "#{info[:url]}")
+      run %(chmod "#{info[:mode]}" "#{dest}")
+    end
+  end
+end
+
+def kubelet_cmd
+  %w(
+    /hyperkube kubelet
+      --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true
+      --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true
+      --cluster-dns=10.96.0.10 --cluster-domain=cluster.local
+      --cgroups-per-qos=false --enforce-node-allocatable=""
+      --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
+  )
+  #--node-ip="192.168.65.2"
+end
+
+setup_apt_config
+run "rm -f /etc/cni/net.d/10-containernet.conf"
+install_packages 'kubernetes-cni'
+install_node_dependencies
+
+# Exploit shared mounts, give CNI paths back to the host
+mount_cni_dirs = [
+  mount_bind("/opt/cni", "/rootfs/opt/cni"),
+  mount_bind("/etc/cni", "/rootfs/etc/cni"),
+]
+
+# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
+wait_for_node_metadata_or_sleep_until_master_init = "[ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config))"
+
+create_shell_wrapper "#{mount_cni_dirs.join(' && ')} && until #{kubelet_cmd.join(' ')} ; do #{wait_for_node_metadata_or_sleep_until_master_init} ; done", '/usr/bin/kubelet.sh'
+
+create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]} && kubectl create -n kube-system -f /etc/weave.yaml", '/usr/bin/kubeadm-init.sh'
+
+flatten
+
+env KUBECONFIG: "/etc/kubernetes/admin.conf"
+
+set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh)
+
+tag "#{@image_name}:latest"

projects/kubernetes/mounts.rb

@@ -0,0 +1,15 @@
+import 'common.rb'
+
+from "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
+
+script = [
+  mount_bind_hostns_self("/etc/cni"), mount_make_hostns_rshared("/etc/cni"),
+  mount_bind_hostns_self("/opt/cni"), mount_make_hostns_rshared("/opt/cni"),
+  mount_persistent_disk("/var/lib"),
+  mkdir_p("/var/lib/kubeadm"),
+]
+
+create_shell_wrapper script.join(' && '), '/usr/bin/kube-mounts.sh'
+set_exec cmd: [ '/usr/bin/kube-mounts.sh' ]
+
+tag "#{@image_name}:latest-mounts"

projects/kubernetes/ssh.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -eux
+docker run \
+  -ti \
+  -v ~/.ssh/:/root/.ssh \
+    jdeathe/centos-ssh \
+      ssh \
+        -o Compression=yes \
+	-o LogLevel=FATAL \
+	-o StrictHostKeyChecking=no \
+	-o UserKnownHostsFile=/dev/null \
+	-o IdentitiesOnly=yes \
+	  "$@"

projects/kubernetes/ssh_into_kubelet.sh

@@ -0,0 +1,2 @@
+#!/bin/bash -eux
+./ssh.sh -t "$1" nsenter --mount --target 1 runc exec --tty kubelet bash -l