diff --git a/projects/kubernetes/Boxfile b/projects/kubernetes/Boxfile deleted file mode 100644 index bdb74422c..000000000 --- a/projects/kubernetes/Boxfile +++ /dev/null @@ -1,94 +0,0 @@ -@image_name = "mobylinux/kubernetes" - -@versions = { - kubernetes: "v1.6.1", - weave: "v1.9.4", - tini: "v0.14.0", -} - -from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}" - -def install_packages pkgs - cmds = [ - %(apt-get update -q), - %(apt-get upgrade -qy), - %(apt-get install -qy #{pkgs}), - ] - - cmds.each { |cmd| run cmd } -end - -def install_dependencies - - prepare = [ - 'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -', - 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list', - ] - - dependencies = %(curl apt-transport-https) - - official_packages = %(kubernetes-cni) - - kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64" - weave_launcher = "https://cloud.weave.works/k8s/v1.6/net.yaml?v=#{@versions[:weave]}" - - download_files = [ - "/etc/weave.yaml" => { - url: weave_launcher, - mode: '0644', - }, - "/usr/bin/kubeadm" => { - url: "#{kube_release_artefacts}/kubeadm", - mode: '0755', - }, - "/usr/bin/tini" => { - url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini", - mode: '0755', - }, - ] - - install_packages dependencies - - prepare.each { |cmd| run cmd } - - install_packages official_packages - - download_files.each do |file| - file.each do |dest,info| - run %(curl --output "#{dest}" --fail --silent --location "#{info[:url]}") - run %(chmod "#{info[:mode]}" "#{dest}") - end - end - -end - -def kubelet_cmd - %w( - /hyperkube kubelet - --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true - --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true - --cluster-dns=10.96.0.10 --cluster-domain=cluster.local - --cgroups-per-qos=false --enforce-node-allocatable="" - ) - #--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin - #--node-ip="192.168.65.2" -end - -def create_shell_wrapper script, path - run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}" -end - -install_dependencies - -# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume -create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh' - -create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh' - -flatten - -env KUBECONFIG: "/etc/kubernetes/admin.conf" - -set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh) - -tag "#{@image_name}:latest" diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 873e4190a..b8878443a 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,14 +1,19 @@ all: build-container-images build-vm-images -build-container-image: Boxfile - docker run --rm -ti \ - -v $(PWD):$(PWD) \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -w $(PWD) \ - boxbuilder/box:master Boxfile +BOX_PLANS = kubernetes.rb mounts.rb -push-container-images: build-container-image cache-images +build-container-images: $(BOX_PLANS) + for plan in $(BOX_PLANS) ; do \ + docker run --rm -ti \ + -v $(PWD):$(PWD) \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -w $(PWD) \ + boxbuilder/box:master $$plan \ + ; done + +push-container-images: build-container-images cache-images docker image push mobylinux/kubernetes:latest + docker image push mobylinux/kubernetes:latest-mounts docker image push mobylinux/kubernetes:latest-image-cache-common docker image push mobylinux/kubernetes:latest-image-cache-control-plane @@ -47,10 +52,10 @@ cache-images: for image in $(COMMON_IMAGES) ; \ do $(MAKE) "image-cache/common/$${image}.tar" \ ; done - cp image-cache/Dockerfile image-cache/common + cp image-cache/Dockerfile image-cache/.dockerignore image-cache/common docker image build -t mobylinux/kubernetes:latest-image-cache-common image-cache/common for image in $(CONTROL_PLANE_IMAGES) ; \ do $(MAKE) "image-cache/control-plane/$${image}.tar" \ ; done - cp image-cache/Dockerfile image-cache/control-plane + cp image-cache/Dockerfile image-cache/.dockerignore image-cache/control-plane docker image build -t mobylinux/kubernetes:latest-image-cache-control-plane image-cache/control-plane diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index ce179654a..c88c18711 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -4,9 +4,11 @@ This project aims to demonstrate how one can create minimal and immutable Kubern Make sure to `cd projects/kubernetes` first. -Build container & OS images: +Edit `kube-master.yml` and add your public SSH key to `files` section. + +Build OS images: ``` -make +make build-vm-images ``` Boot Kubernetes master OS image using `hyperkit` on macOS: @@ -14,9 +16,19 @@ Boot Kubernetes master OS image using `hyperkit` on macOS: ./boot-master.sh ``` +Get IP address of the master: +``` +ip addr show dev eth0 +``` + +Login to the kubelet container: +``` +./ssh_into_kubelet.sh +``` + Manually initialise master with `kubeadm`: ``` -runc exec kubelet kubeadm-init.sh +kubeadm-init.sh ``` Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments, diff --git a/projects/kubernetes/boot-master.sh b/projects/kubernetes/boot-master.sh index 6798caa8b..b0515ca12 100755 --- a/projects/kubernetes/boot-master.sh +++ b/projects/kubernetes/boot-master.sh @@ -2,4 +2,4 @@ disk="kube-master-disk.img" set -x rm -f "${disk}" -../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master +../../bin/moby run hyperkit -cpus 2 -mem 6144 -disk-size 6144 kube-master diff --git a/projects/kubernetes/boot-node.sh b/projects/kubernetes/boot-node.sh index 76ca817f6..a88bc566a 100755 --- a/projects/kubernetes/boot-node.sh +++ b/projects/kubernetes/boot-node.sh @@ -5,4 +5,4 @@ shift disk="kube-${name}-disk.img" set -x rm -f "${disk}" -../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node +../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 1024 -disk "${disk}" -data "${*}" kube-node diff --git a/projects/kubernetes/common.rb b/projects/kubernetes/common.rb new file mode 100644 index 000000000..8fb0c2f52 --- /dev/null +++ b/projects/kubernetes/common.rb @@ -0,0 +1,54 @@ +@image_name = "mobylinux/kubernetes" + +@versions = { + kubernetes: "v1.6.1", + weave: "v1.9.4", + tini: "v0.14.0", +} + +def install_packages pkgs + cmds = [ + %(apt-get update -q), + %(apt-get upgrade -qy), + %(apt-get install -qy #{pkgs}), + ] + + cmds.each { |cmd| run cmd } +end + +def setup_apt_config + prepare = [ + 'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -', + 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list', + ] + + dependencies = %(curl apt-transport-https) + + install_packages dependencies + + prepare.each { |cmd| run cmd } +end + +def create_shell_wrapper script, path + run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}" +end + +def mount_bind src, dst + "mount --bind #{src} #{dst}" +end + +def mount_bind_hostns_self mnt + "nsenter --mount=/proc/1/ns/mnt mount -- --bind #{mnt} #{mnt}" +end + +def mount_make_hostns_rshared mnt + "nsenter --mount=/proc/1/ns/mnt mount -- --make-rshared #{mnt}" +end + +def mount_persistent_disk mnt + "/mount.sh #{mnt}" +end + +def mkdir_p dir + "mkdir -p #{dir}" +end diff --git a/projects/kubernetes/image-cache/.dockerignore b/projects/kubernetes/image-cache/.dockerignore new file mode 100644 index 000000000..94143827e --- /dev/null +++ b/projects/kubernetes/image-cache/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/projects/kubernetes/image-cache/Dockerfile b/projects/kubernetes/image-cache/Dockerfile index e5e1ecafc..ffde4331a 100644 --- a/projects/kubernetes/image-cache/Dockerfile +++ b/projects/kubernetes/image-cache/Dockerfile @@ -1,6 +1,4 @@ FROM mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08 - ADD . /images - ENTRYPOINT [ "/bin/sh", "-c" ] -CMD [ "for image in /images/*.tar ; do docker image load -i $image ; done" ] +CMD [ "for image in /images/*.tar ; do docker image load -i $image && rm -f $image ; done" ] diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 011614370..fd2defbdc 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -29,20 +29,15 @@ onboot: capabilities: - CAP_SYS_ADMIN - CAP_MKNOD - - name: mount - image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + - name: mounts + image: "mobylinux/kubernetes:latest-mounts" + capabilities: + - all + pid: host + rootfsPropagation: shared binds: - /dev:/dev - /var:/var:rshared,rbind - capabilities: - - CAP_SYS_ADMIN - rootfsPropagation: shared - command: ["/mount.sh", "/var/lib"] - - name: mount - image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" - binds: - - /var:/var - command: ["mkdir", "-p", "/var/lib/kubeadm"] services: - name: rngd image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92" @@ -92,6 +87,8 @@ services: - /lib/modules:/lib/modules - /var:/var:rshared,rbind - /var/lib/kubeadm:/etc/kubernetes + - /etc/cni:/etc/cni:rshared,rbind + - /opt/cni:/opt/cni:rshared,rbind rootfsPropagation: shared - name: kubernetes-image-cache-common image: "mobylinux/kubernetes:latest-image-cache-common" @@ -114,11 +111,13 @@ services: - /dev:/dev - /var:/var:rshared,rbind - /var/lib/kubeadm:/etc/kubernetes - #- /var/log:/var/log #/var/log/containers - # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared + - /etc/cni:/rootfs/etc/cni:rshared,rbind + - /opt/cni:/rootfs/opt/cni:rshared,rbind rootfsPropagation: shared files: - path: root/.ssh/authorized_keys contents: '# Your ssh key goes here' + - {path: etc/cni, directory: true} + - {path: opt/cni, directory: true} outputs: - format: kernel+initrd diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index 09fdab27d..9908c972a 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -29,20 +29,15 @@ onboot: capabilities: - CAP_SYS_ADMIN - CAP_MKNOD - - name: mount - image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + - name: mounts + image: "mobylinux/kubernetes:latest-mounts" + capabilities: + - all + pid: host + rootfsPropagation: shared binds: - /dev:/dev - /var:/var:rshared,rbind - capabilities: - - CAP_SYS_ADMIN - rootfsPropagation: shared - command: ["/mount.sh", "/var/lib"] - - name: mount - image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" - binds: - - /var:/var - command: ["mkdir", "-p", "/var/lib/kubeadm"] services: - name: rngd image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92" @@ -92,6 +87,8 @@ services: - /lib/modules:/lib/modules - /var:/var:rshared,rbind - /var/lib/kubeadm:/etc/kubernetes + - /etc/cni:/etc/cni:rshared,rbind + - /opt/cni:/opt/cni:rshared,rbind rootfsPropagation: shared - name: kubernetes-image-cache-common image: "mobylinux/kubernetes:latest-image-cache-common" @@ -110,11 +107,13 @@ services: - /dev:/dev - /var:/var:rshared,rbind - /var/lib/kubeadm:/etc/kubernetes - #- /var/log:/var/log #/var/log/containers - # TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared + - /etc/cni:/rootfs/etc/cni:rshared,rbind + - /opt/cni:/rootfs/opt/cni:rshared,rbind rootfsPropagation: shared files: - path: root/.ssh/authorized_keys contents: '# Your ssh key goes here' + - {path: etc/cni, directory: true} + - {path: opt/cni, directory: true} outputs: - format: kernel+initrd diff --git a/projects/kubernetes/kubernetes.rb b/projects/kubernetes/kubernetes.rb new file mode 100644 index 000000000..ee4c30ab4 --- /dev/null +++ b/projects/kubernetes/kubernetes.rb @@ -0,0 +1,68 @@ +import 'common.rb' + +from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}" + +def install_node_dependencies + kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64" + weave_launcher = "https://frontend.dev.weave.works/k8s/v1.6/net?v=#{@versions[:weave]}" + + download_files = [ + "/etc/weave.yaml" => { + url: weave_launcher, + mode: '0644', + }, + "/usr/bin/kubeadm" => { + url: "#{kube_release_artefacts}/kubeadm", + mode: '0755', + }, + "/usr/bin/tini" => { + url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini", + mode: '0755', + }, + ] + + download_files.each do |file| + file.each do |dest,info| + run %(curl --insecure --output "#{dest}" --fail --silent --location "#{info[:url]}") + run %(chmod "#{info[:mode]}" "#{dest}") + end + end +end + +def kubelet_cmd + %w( + /hyperkube kubelet + --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true + --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true + --cluster-dns=10.96.0.10 --cluster-domain=cluster.local + --cgroups-per-qos=false --enforce-node-allocatable="" + --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin + ) + #--node-ip="192.168.65.2" +end + +setup_apt_config +run "rm -f /etc/cni/net.d/10-containernet.conf" +install_packages 'kubernetes-cni' +install_node_dependencies + +# Exploit shared mounts, give CNI paths back to the host +mount_cni_dirs = [ + mount_bind("/opt/cni", "/rootfs/opt/cni"), + mount_bind("/etc/cni", "/rootfs/etc/cni"), +] + +# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume +wait_for_node_metadata_or_sleep_until_master_init = "[ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config))" + +create_shell_wrapper "#{mount_cni_dirs.join(' && ')} && until #{kubelet_cmd.join(' ')} ; do #{wait_for_node_metadata_or_sleep_until_master_init} ; done", '/usr/bin/kubelet.sh' + +create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]} && kubectl create -n kube-system -f /etc/weave.yaml", '/usr/bin/kubeadm-init.sh' + +flatten + +env KUBECONFIG: "/etc/kubernetes/admin.conf" + +set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh) + +tag "#{@image_name}:latest" diff --git a/projects/kubernetes/mounts.rb b/projects/kubernetes/mounts.rb new file mode 100644 index 000000000..d07b996a7 --- /dev/null +++ b/projects/kubernetes/mounts.rb @@ -0,0 +1,15 @@ +import 'common.rb' + +from "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a" + +script = [ + mount_bind_hostns_self("/etc/cni"), mount_make_hostns_rshared("/etc/cni"), + mount_bind_hostns_self("/opt/cni"), mount_make_hostns_rshared("/opt/cni"), + mount_persistent_disk("/var/lib"), + mkdir_p("/var/lib/kubeadm"), +] + +create_shell_wrapper script.join(' && '), '/usr/bin/kube-mounts.sh' +set_exec cmd: [ '/usr/bin/kube-mounts.sh' ] + +tag "#{@image_name}:latest-mounts" diff --git a/projects/kubernetes/ssh.sh b/projects/kubernetes/ssh.sh new file mode 100755 index 000000000..5452720ee --- /dev/null +++ b/projects/kubernetes/ssh.sh @@ -0,0 +1,12 @@ +#!/bin/bash -eux +docker run \ + -ti \ + -v ~/.ssh/:/root/.ssh \ + jdeathe/centos-ssh \ + ssh \ + -o Compression=yes \ + -o LogLevel=FATAL \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o IdentitiesOnly=yes \ + "$@" diff --git a/projects/kubernetes/ssh_into_kubelet.sh b/projects/kubernetes/ssh_into_kubelet.sh new file mode 100755 index 000000000..43791406a --- /dev/null +++ b/projects/kubernetes/ssh_into_kubelet.sh @@ -0,0 +1,2 @@ +#!/bin/bash -eux +./ssh.sh -t "$1" nsenter --mount --target 1 runc exec --tty kubelet bash -l