diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index 052802810..7c915da0b 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,19 +1,10 @@ all: build-container-images build-vm-images -BOX_PLANS = kubernetes.rb mounts.rb +build-container-images: + make -C kubernetes tag -build-container-images: $(BOX_PLANS) - for plan in $(BOX_PLANS) ; do \ - docker run --rm -ti \ - -v $(PWD):$(PWD) \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -w $(PWD) \ - boxbuilder/box:master $$plan \ - ; done - -push-container-images: build-container-images cache-images - docker image push linuxkit/kubernetes:latest - docker image push linuxkit/kubernetes:latest-mounts +push-container-images: cache-images + make -C kubernetes push docker image push linuxkit/kubernetes:latest-image-cache-common docker image push linuxkit/kubernetes:latest-image-cache-control-plane @@ -45,7 +36,7 @@ CONTROL_PLANE_IMAGES := \ image-cache/%.tar: mkdir -p $(dir $@) - DOCKER_CONTENT_TRUST=1 docker image pull gcr.io/google_containers/$(shell basename $@ .tar) + docker image pull gcr.io/google_containers/$(shell basename $@ .tar) docker image save -o $@ gcr.io/google_containers/$(shell basename $@ .tar) cache-images: diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index c11643a6b..2f69fc990 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -1,11 +1,25 @@ #!/bin/bash -eu +: ${KUBE_PORT_BASE:=2222} if [ $# -eq 0 ] ; then img="kube-master" + port=${KUBE_PORT_BASE} data="" state="kube-master-state" elif [ $# -gt 1 ] ; then + case $1 in + ''|*[!0-9]*) + echo "Node number must be a number" + exit 1 + ;; + 0) + echo "Node number must be greater than 0" + exit 1 + ;; + *) ;; + esac img="kube-node" name="node-${1}" + port=$((${KUBE_PORT_BASE} + $1)) shift data="${*}" state="kube-${name}-state" @@ -19,4 +33,4 @@ else fi set -x rm -rf "${state}" -../../bin/linuxkit run -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" +../../bin/linuxkit run -publish $port:22 -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" diff --git a/projects/kubernetes/common.rb b/projects/kubernetes/common.rb deleted file mode 100644 index b902cd0c1..000000000 --- a/projects/kubernetes/common.rb +++ /dev/null @@ -1,40 +0,0 @@ -@image_name = "linuxkit/kubernetes" - -@versions = { - kubernetes: 'v1.6.1', - weave: 'v1.9.4', - cni: '0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff', -} - -def install_packages pkgs - cmds = [ - %(apk update), - %(apk add #{pkgs.join(' ')}), - ] - - cmds.each { |cmd| run cmd } -end - -def create_shell_wrapper script, path - run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}" -end - -def mount_bind src, dst - "mount --bind #{src} #{dst}" -end - -def mount_bind_hostns_self mnt - "nsenter --mount=/proc/1/ns/mnt mount -- --bind #{mnt} #{mnt}" -end - -def mount_make_hostns_rshared mnt - "nsenter --mount=/proc/1/ns/mnt mount -- --make-rshared #{mnt}" -end - -def mount_persistent_disk mnt - "/mount.sh #{mnt}" -end - -def mkdir_p dir - "mkdir -p #{dir}" -end diff --git a/projects/kubernetes/image-cache/Dockerfile b/projects/kubernetes/image-cache/Dockerfile index ebdfbed2f..c3197868c 100644 --- a/projects/kubernetes/image-cache/Dockerfile +++ b/projects/kubernetes/image-cache/Dockerfile @@ -2,3 +2,4 @@ FROM linuxkit/docker-ce:9b937df179bdbebbc70243779978057df0b54190 ADD . /images ENTRYPOINT [ "/bin/sh", "-c" ] CMD [ "for image in /images/*.tar ; do docker image load -i $image && rm -f $image ; done" ] +LABEL org.mobyproject.config='{"binds": ["/var/run:/var/run"]}' diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 0445b5d70..fb0a76d7f 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -1,6 +1,6 @@ kernel: image: linuxkit/kernel:4.9.38 - cmdline: "console=ttyS0 console=tty0 page_poison=1" + cmdline: "console=tty0 console=ttyS0" init: - linuxkit/init:059b2bb4b6efa5c58cf53fed4d0ea863521959fc - linuxkit/runc:4a35484aa6f90a1f06cdf1fb36f7056926a084b9 @@ -13,17 +13,18 @@ onboot: image: linuxkit/sysfs:006a65b30cfdd9d751d7ab042fde7eca2c3bc9dc - name: binfmt image: linuxkit/binfmt:0bde4ebd422099f45c5ee03217413523ad2223e5 + - name: metadata + image: linuxkit/metadata:f122f1b4e873f1d08cd67bd9105385fd923af0cb - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts - image: linuxkit/kubernetes:latest-mounts - capabilities: - - all - pid: host - rootfsPropagation: shared + image: linuxkit/mount:ac8939c4102f97c084d9ddfd445c1908fce6d768 + command: ["/mount.sh", "/var/lib/"] + - name: var + image: library/alpine:3.6 + command: ["mkdir", "/var/lib/kubeadm"] binds: - - /dev:/dev - - /var:/var:rshared,rbind + - /var/lib:/var/lib services: - name: getty image: linuxkit/getty:deb9332e786e72591bd9be200bcc9c7a534eb754 @@ -56,28 +57,10 @@ services: rootfsPropagation: shared - name: kubernetes-image-cache-common image: linuxkit/kubernetes:latest-image-cache-common - binds: - - /var/run:/var/run - name: kubernetes-image-cache-control-plane image: linuxkit/kubernetes:latest-image-cache-control-plane - binds: - - /var/run:/var/run - name: kubelet - image: linuxkit/kubernetes:latest - capabilities: - - all - net: host - pid: host - mounts: - - type: cgroup - options: ["rw","nosuid","noexec","nodev","relatime"] - binds: - - /dev:/dev - - /var:/var:rshared,rbind - - /var/lib/kubeadm:/etc/kubernetes - - /etc/cni:/rootfs/etc/cni:rshared,rbind - - /opt/cni:/rootfs/opt/cni:rshared,rbind - rootfsPropagation: shared + image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index cb757a7c2..d586adb8a 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -1,6 +1,6 @@ kernel: image: linuxkit/kernel:4.9.38 - cmdline: "console=ttyS0 console=tty0 page_poison=1" + cmdline: "console=tty0 console=ttyS0" init: - linuxkit/init:059b2bb4b6efa5c58cf53fed4d0ea863521959fc - linuxkit/runc:4a35484aa6f90a1f06cdf1fb36f7056926a084b9 @@ -13,17 +13,18 @@ onboot: image: linuxkit/sysfs:006a65b30cfdd9d751d7ab042fde7eca2c3bc9dc - name: binfmt image: linuxkit/binfmt:0bde4ebd422099f45c5ee03217413523ad2223e5 + - name: metadata + image: linuxkit/metadata:f122f1b4e873f1d08cd67bd9105385fd923af0cb - name: format image: linuxkit/format:84a997e69051a1bf05b7c1926ab785bb07932954 - name: mounts - image: linuxkit/kubernetes:latest-mounts - capabilities: - - all - pid: host - rootfsPropagation: shared + image: linuxkit/mount:ac8939c4102f97c084d9ddfd445c1908fce6d768 + command: ["/mount.sh", "/var/lib/"] + - name: var + image: library/alpine:3.6 + command: ["mkdir", "/var/lib/kubeadm"] binds: - - /dev:/dev - - /var:/var:rshared,rbind + - /var/lib:/var/lib services: - name: getty image: linuxkit/getty:deb9332e786e72591bd9be200bcc9c7a534eb754 @@ -56,24 +57,8 @@ services: rootfsPropagation: shared - name: kubernetes-image-cache-common image: linuxkit/kubernetes:latest-image-cache-common - binds: - - /var/run:/var/run - name: kubelet - image: linuxkit/kubernetes:latest - capabilities: - - all - net: host - pid: host - mounts: - - type: cgroup - options: ["rw","nosuid","noexec","nodev","relatime"] - binds: - - /dev:/dev - - /var:/var:rshared,rbind - - /var/lib/kubeadm:/etc/kubernetes - - /etc/cni:/rootfs/etc/cni:rshared,rbind - - /opt/cni:/rootfs/opt/cni:rshared,rbind - rootfsPropagation: shared + image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kubernetes.rb b/projects/kubernetes/kubernetes.rb deleted file mode 100644 index f98eea807..000000000 --- a/projects/kubernetes/kubernetes.rb +++ /dev/null @@ -1,77 +0,0 @@ -import 'common.rb' - -from "alpine:edge" - -def install_node_dependencies - kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64" - cni_release_artefacts = "https://dl.k8s.io/network-plugins/cni-amd64-#{@versions[:cni]}.tar.gz" - weave_launcher = "https://cloud.weave.works/k8s/v1.6/net?v=#{@versions[:weave]}" - - download_files = [ - '/etc/weave.yaml' => { - url: weave_launcher, - mode: '0644', - }, - '/tmp/cni.tgz' => { - url: cni_release_artefacts, - mode: '0644', - }, - '/usr/bin/kubelet' => { - url: "#{kube_release_artefacts}/kubelet", - mode: '0755', - }, - '/usr/bin/kubeadm' => { - url: "#{kube_release_artefacts}/kubeadm", - mode: '0755', - }, - '/usr/bin/kubectl' => { - url: "#{kube_release_artefacts}/kubectl", - mode: '0755', - }, - ] - - download_files.each do |file| - file.each do |dest,info| - run %(curl --output "#{dest}" --fail --silent --location "#{info[:url]}") - run %(chmod "#{info[:mode]}" "#{dest}") - end - end - - run "mkdir -p /opt/cni/bin /etc/cni/net.d && tar xzf /tmp/cni.tgz -C /opt/cni && rm -f /tmp/cni.tgz" -end - -def kubelet_cmd - %w( - kubelet - --kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true - --pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true - --cluster-dns=10.96.0.10 --cluster-domain=cluster.local - --cgroups-per-qos=false --enforce-node-allocatable="" - --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin - ) -end - -kubelet_dependencies = %w(libc6-compat util-linux iproute2 iptables ebtables ethtool socat curl) -install_packages kubelet_dependencies -install_node_dependencies - -# Exploit shared mounts, give CNI paths back to the host -mount_cni_dirs = [ - mount_bind("/opt/cni", "/rootfs/opt/cni"), - mount_bind("/etc/cni", "/rootfs/etc/cni"), -] - -# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume -wait_for_node_metadata_or_sleep_until_master_init = "[ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config))" - -create_shell_wrapper "#{mount_cni_dirs.join(' && ')} && until #{kubelet_cmd.join(' ')} ; do #{wait_for_node_metadata_or_sleep_until_master_init} ; done", '/usr/bin/kubelet.sh' - -create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]} && kubectl create -n kube-system -f /etc/weave.yaml", '/usr/bin/kubeadm-init.sh' - -flatten - -env KUBECONFIG: "/etc/kubernetes/admin.conf" - -set_exec entrypoint: %w(kubelet.sh) - -tag "#{@image_name}:latest" diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile new file mode 100644 index 000000000..6a39d6910 --- /dev/null +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -0,0 +1,49 @@ +#FROM linuxkit/alpine:9bcf61f605ef0ce36cc94d59b8eac307862de6e1 AS build +# XXX needs ebtables ethtool iproute2 libc6-compat socat +FROM alpine:3.6 AS build + +ENV kubernetes_version v1.6.1 +ENV weave_version v1.9.4 +ENV cni_version 0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff + +ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" + +RUN apk add -U --no-cache \ + curl \ + && true + +RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +RUN apk add --no-cache --initdb -p /out \ + alpine-baselayout \ + busybox \ + ca-certificates \ + curl \ + ebtables \ + ethtool \ + iproute2 \ + iptables \ + libc6-compat \ + musl \ + socat \ + util-linux \ + && true +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + +RUN curl -fSL -o /tmp/cni.tgz https://dl.k8s.io/network-plugins/cni-amd64-${cni_version}.tar.gz && \ + mkdir -p /out/opt/cni /out/etc/cni/net.d && \ + tar -xzf /tmp/cni.tgz -C /out/opt/cni +RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.6/net?v=${weave_version} +RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet +RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm +RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl + +ADD kubelet.sh /out/usr/bin/kubelet.sh +ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh + +FROM scratch +WORKDIR / +ENTRYPOINT ["/usr/bin/kubelet.sh"] +COPY --from=build /out / +ENV KUBECONFIG "/etc/kubernetes/admin.conf" +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/cni:/rootfs/etc/cni:rshared,rbind", "/opt/cni:/rootfs/opt/cni:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host"}' diff --git a/projects/kubernetes/kubernetes/Makefile b/projects/kubernetes/kubernetes/Makefile new file mode 100644 index 000000000..01aec0f84 --- /dev/null +++ b/projects/kubernetes/kubernetes/Makefile @@ -0,0 +1,6 @@ +ORG?=linuxkitprojects +IMAGE=kubernetes +NETWORK=1 +NOTRUST=1 + +include ../../../pkg/package.mk diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh new file mode 100755 index 000000000..5e953229d --- /dev/null +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 +kubectl create -n kube-system -f /etc/weave.yaml diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh new file mode 100755 index 000000000..fb8ef2e81 --- /dev/null +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -0,0 +1,20 @@ +#!/bin/sh +mount --bind /opt/cni /rootfs/opt/cni +mount --bind /etc/cni /rootfs/etc/cni +until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ + --require-kubeconfig=true \ + --pod-manifest-path=/var/lib/kubeadm/manifests \ + --allow-privileged=true \ + --cluster-dns=10.96.0.10 \ + --cluster-domain=cluster.local \ + --cgroups-per-qos=false \ + --enforce-node-allocatable= \ + --network-plugin=cni \ + --cni-conf-dir=/etc/cni/net.d \ + --cni-bin-dir=/opt/cni/bin ; do + if [ ! -f /var/config/userdata ] ; then + sleep 1 + else + kubeadm join --skip-preflight-checks $(cat /var/config/userdata) + fi +done diff --git a/projects/kubernetes/mounts.rb b/projects/kubernetes/mounts.rb deleted file mode 100644 index a0cddb491..000000000 --- a/projects/kubernetes/mounts.rb +++ /dev/null @@ -1,15 +0,0 @@ -import 'common.rb' - -from "linuxkit/mount:ac8939c4102f97c084d9ddfd445c1908fce6d768" - -script = [ - mount_bind_hostns_self("/etc/cni"), mount_make_hostns_rshared("/etc/cni"), - mount_bind_hostns_self("/opt/cni"), mount_make_hostns_rshared("/opt/cni"), - mount_persistent_disk("/var/lib"), - mkdir_p("/var/lib/kubeadm"), -] - -create_shell_wrapper script.join(' && '), '/usr/bin/kube-mounts.sh' -set_exec cmd: [ '/usr/bin/kube-mounts.sh' ] - -tag "#{@image_name}:latest-mounts"