diff --git a/projects/kubernetes/Makefile b/projects/kubernetes/Makefile index c9e5da86d..034e519fd 100644 --- a/projects/kubernetes/Makefile +++ b/projects/kubernetes/Makefile @@ -1,3 +1,5 @@ +KUBE_RUNTIME ?= docker + all: tag-container-images build-vm-images tag-container-images: @@ -12,11 +14,11 @@ push-container-images: build-vm-images: kube-master.iso kube-node.iso -kube-master.iso: kube-master.yml - moby build -name kube-master -format iso-efi -format iso-bios kube-master.yml +kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml + moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml -kube-node.iso: kube-node.yml - moby build -name kube-node -format iso-efi -format iso-bios kube-node.yml +kube-node.iso: kube.yml $(KUBE_RUNTIME).yml + moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml clean: rm -f -r \ diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index 536b65402..d5e3a13a3 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -4,13 +4,16 @@ This project aims to demonstrate how one can create minimal and immutable Kubern Make sure to `cd projects/kubernetes` first. -Edit `kube-master.yml` and add your public SSH key to `files` section. - Build OS images: ``` make build-vm-images ``` +By default this will build images using Docker Engine for execution. To instead use cri-containerd use: +``` +make build-vm-images KUBE_RUNTIME=cri-containerd +``` + Boot Kubernetes master OS image using `hyperkit` on macOS: or `qemu` on Linux: ``` ./boot.sh diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 97f15aa89..3b041aef4 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -2,13 +2,19 @@ set -e -: ${KUBE_PORT_BASE:=2222} -: ${KUBE_VCPUS:=2} -: ${KUBE_MEM:=1024} -: ${KUBE_DISK:=4G} +: ${KUBE_MASTER_VCPUS:=2} +: ${KUBE_MASTER_MEM:=1024} +: ${KUBE_MASTER_DISK:=4G} + +: ${KUBE_NODE_VCPUS:=2} +: ${KUBE_NODE_MEM:=4096} +: ${KUBE_NODE_DISK:=8G} + : ${KUBE_NETWORKING:=default} : ${KUBE_RUN_ARGS:=} : ${KUBE_EFI:=} +: ${KUBE_MAC:=} +: ${KUBE_PRESERVE_STATE:=} [ "$(uname -s)" = "Darwin" ] && KUBE_EFI=1 @@ -19,7 +25,11 @@ if [ $# -eq 0 ] ; then img="kube-master" data="" state="kube-master-state" -elif [ $# -gt 1 ] ; then + + : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS} + : ${KUBE_MEM:=$KUBE_MASTER_MEM} + : ${KUBE_DISK:=$KUBE_MASTER_DISK} +elif [ $# -gt 1 ] || [ $# -eq 1 -a -n "${KUBE_PRESERVE_STATE}" ] ; then case $1 in ''|*[!0-9]*) echo "Node number must be a number" @@ -36,6 +46,10 @@ elif [ $# -gt 1 ] ; then shift data="${*}" state="kube-${name}-state" + + : ${KUBE_VCPUS:=$KUBE_NODE_VCPUS} + : ${KUBE_MEM:=$KUBE_NODE_MEM} + : ${KUBE_DISK:=$KUBE_NODE_DISK} else echo "Usage:" echo " - Boot master:" @@ -45,5 +59,11 @@ else exit 1 fi set -x -rm -rf "${state}" +if [ -z "${KUBE_PRESERVE_STATE}" ] ; then + rm -rf "${state}" + mkdir "${state}" + if [ -n "${KUBE_MAC}" ] ; then + echo -n "${KUBE_MAC}" > "${state}"/mac-addr + fi +fi linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" ${uefi} "${img}${suffix}" diff --git a/projects/kubernetes/cri-containerd-master.yml b/projects/kubernetes/cri-containerd-master.yml new file mode 100644 index 000000000..e69de29bb diff --git a/projects/kubernetes/cri-containerd.yml b/projects/kubernetes/cri-containerd.yml new file mode 100644 index 000000000..a3520ef5d --- /dev/null +++ b/projects/kubernetes/cri-containerd.yml @@ -0,0 +1,7 @@ +services: + - name: cri-containerd + image: linuxkitprojects/cri-containerd:b8b6a48426c2165055534b06fb0119f07e24506a +files: + - path: /etc/kubelet.conf + contents: | + KUBELET_ARGS="--container-runtime=remote --container-runtime-endpoint=unix:///var/run/cri-containerd.sock" diff --git a/projects/kubernetes/cri-containerd/Dockerfile b/projects/kubernetes/cri-containerd/Dockerfile new file mode 100644 index 000000000..707642370 --- /dev/null +++ b/projects/kubernetes/cri-containerd/Dockerfile @@ -0,0 +1,49 @@ +FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build + +RUN \ + apk add \ + bash \ + gcc \ + git \ + go \ + libc-dev \ + make \ + && true +ENV GOPATH=/go PATH=$PATH:/go/bin + +ENV CRI_CONTAINERD_URL https://github.com/kubernetes-incubator/cri-containerd.git +#ENV CRI_CONTAINERD_BRANCH pull/NNN/head +ENV CRI_CONTAINERD_COMMIT a8d49402859167a232b094d971e70c2f4b71b8ea +RUN mkdir -p $GOPATH/src/github.com/kubernetes-incubator && \ + cd $GOPATH/src/github.com/kubernetes-incubator && \ + git clone $CRI_CONTAINERD_URL cri-containerd +WORKDIR $GOPATH/src/github.com/kubernetes-incubator/cri-containerd +RUN set -e; \ + if [ -n "$CRI_CONTAINERD_BRANCH" ] ; then \ + git fetch origin "$CRI_CONTAINERD_BRANCH"; \ + fi; \ + git checkout $CRI_CONTAINERD_COMMIT +RUN make static-binaries + +RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +# util-linux because a full ns-enter is required. +# example commands: /usr/bin/nsenter --net= -F -- +# /usr/bin/nsenter --net=/var/run/netns/cni-5e8acebe-810d-c1b9-ced0-47be2f312fa8 -F -- +# NB the first ("--net=") is actually not valid -- see https://github.com/kubernetes-incubator/cri-containerd/issues/245 +RUN apk add --no-cache --initdb -p /out \ + alpine-baselayout \ + busybox \ + ca-certificates \ + iptables \ + util-linux \ + && true +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + +RUN make DESTDIR=/out install + +FROM scratch +WORKDIR / +ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/var/lib/cni/opt/bin", "--network-conf-dir", "/var/lib/cni/etc/net.d"] +COPY --from=build /out / +LABEL org.mobyproject.config='{"binds": ["/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/tmp:/tmp", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/etc/cni:rshared,rbind", "/var/lib/cni/opt:/opt/cni:rshared,rbind", "/run/containerd/containerd.sock:/run/containerd/containerd.sock"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc/net.d", "/var/lib/cni/opt"]}}' diff --git a/projects/kubernetes/cri-containerd/Makefile b/projects/kubernetes/cri-containerd/Makefile new file mode 100644 index 000000000..fe0253576 --- /dev/null +++ b/projects/kubernetes/cri-containerd/Makefile @@ -0,0 +1,7 @@ +ORG?=linuxkitprojects +IMAGE=cri-containerd +NETWORK=1 +NOTRUST=1 +ARCHES=x86_64 + +include ../../../pkg/package.mk diff --git a/projects/kubernetes/docker-master.yml b/projects/kubernetes/docker-master.yml new file mode 100644 index 000000000..ec6298647 --- /dev/null +++ b/projects/kubernetes/docker-master.yml @@ -0,0 +1,3 @@ +services: + - name: kubernetes-image-cache-control-plane + image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25 diff --git a/projects/kubernetes/docker.yml b/projects/kubernetes/docker.yml new file mode 100644 index 000000000..03388d91b --- /dev/null +++ b/projects/kubernetes/docker.yml @@ -0,0 +1,27 @@ +services: + - name: docker + image: docker:17.07.0-ce-dind + capabilities: + - all + pid: host + mounts: + - type: cgroup + options: ["rw","nosuid","noexec","nodev","relatime"] + binds: + - /dev:/dev + - /etc/resolv.conf:/etc/resolv.conf + - /lib/modules:/lib/modules + - /run:/run + - /var:/var:rshared,rbind + - /var/lib/kubeadm:/etc/kubernetes + - /var/lib/cni/etc:/etc/cni:rshared,rbind + - /var/lib/cni/opt:/opt/cni:rshared,rbind + rootfsPropagation: shared + command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] + runtime: + mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"] + - name: kubernetes-image-cache-common + image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25 +files: + - path: /etc/kubelet.conf + contents: "" diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml deleted file mode 100644 index 3745a89c1..000000000 --- a/projects/kubernetes/kube-master.yml +++ /dev/null @@ -1,66 +0,0 @@ -kernel: - image: linuxkit/kernel:4.9.50 - cmdline: "console=tty0 console=ttyS0" -init: - - linuxkit/init:851e9c3ad0574d640b733b92fdb26c368d2f7f8f - - linuxkit/runc:a1b564248a0d0b118c11e61db9f84ecf41dd2d2a - - linuxkit/containerd:06876ceef325e49e9ba119659357768d5df89075 - - linuxkit/ca-certificates:e44b0a66df5a102c0e220f0066b0d904710dcb10 -onboot: - - name: sysctl - image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051 - - name: sysfs - image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9 - - name: dhcpcd - image: linuxkit/dhcpcd:f3f5413abb78fae9020e35bd4788fa93df4530b7 - command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] - - name: metadata - image: linuxkit/metadata:da3138079c168e0c5608d8f3853366c113ed91d2 - - name: format - image: linuxkit/format:158d992b7bf7ab984100c697d7e72161ea7d7382 - - name: mounts - image: linuxkit/mount:4fe245efb01384e42622c36302e13e386bbaeb08 - command: ["/usr/bin/mountie", "/var/lib/"] -services: - - name: getty - image: linuxkit/getty:797cb79e0a229fcd16ebf44a0da74bcec03968ec - env: - - INSECURE=true - - name: rngd - image: linuxkit/rngd:558e86a36242bb74353bc9287b715ddb8567357e - - name: ntpd - image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67 - - name: sshd - image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0 - - name: docker - image: docker:17.07.0-ce-dind - capabilities: - - all - pid: host - mounts: - - type: cgroup - options: ["rw","nosuid","noexec","nodev","relatime"] - binds: - - /dev:/dev - - /etc/resolv.conf:/etc/resolv.conf - - /lib/modules:/lib/modules - - /run:/run - - /var:/var:rshared,rbind - - /var/lib/kubeadm:/etc/kubernetes - - /var/lib/cni/etc:/etc/cni:rshared,rbind - - /var/lib/cni/opt:/opt/cni:rshared,rbind - rootfsPropagation: shared - command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] - runtime: - mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"] - - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25 - - name: kubernetes-image-cache-control-plane - image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25 - - name: kubelet - image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2 -files: - - path: root/.ssh/authorized_keys - source: ~/.ssh/id_rsa.pub - mode: "0600" - optional: true diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube.yml similarity index 63% rename from projects/kubernetes/kube-node.yml rename to projects/kubernetes/kube.yml index 655314d8a..924648234 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube.yml @@ -9,6 +9,9 @@ init: onboot: - name: sysctl image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051 + binds: + - /etc/sysctl.d/01-kubernetes.conf:/etc/sysctl.d/01-kubernetes.conf + readonly: false - name: sysfs image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9 - name: dhcpcd @@ -32,32 +35,19 @@ services: image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67 - name: sshd image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0 - - name: docker - image: docker:17.07.0-ce-dind - capabilities: - - all - pid: host - mounts: - - type: cgroup - options: ["rw","nosuid","noexec","nodev","relatime"] - binds: - - /dev:/dev - - /etc/resolv.conf:/etc/resolv.conf - - /lib/modules:/lib/modules - - /run:/run - - /var:/var:rshared,rbind - - /var/lib/kubeadm:/etc/kubernetes - - /var/lib/cni/etc:/etc/cni:rshared,rbind - - /var/lib/cni/opt:/opt/cni:rshared,rbind - rootfsPropagation: shared - command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] - runtime: - mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"] - - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25 - name: kubelet - image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2 + image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a files: + - path: etc/linuxkit.yml + metadata: yaml + - path: /etc/kubernetes + symlink: "/var/lib/kubeadm" + - path: /etc/sysctl.d/01-kubernetes.conf + contents: 'net.ipv4.ip_forward = 1' + - path: /opt/cni + directory: true + - path: /etc/cni + directory: true - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub mode: "0600" diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 21f13eccc..a6cafa178 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -30,9 +30,7 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache RUN rmdir /out/var/run && ln -nfs /run /out/var/run -RUN curl -fSL -o /tmp/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz && \ - mkdir -p /out/opt/cni/bin /out/etc/cni/net.d && \ - tar -xzf /tmp/cni.tgz -C /out/opt/cni/bin +RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm @@ -47,4 +45,4 @@ WORKDIR / ENTRYPOINT ["/usr/bin/kubelet.sh"] COPY --from=build /out / ENV KUBECONFIG "/etc/kubernetes/admin.conf" -LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/rootfs/etc/cni:rshared,rbind", "/var/lib/cni/opt:/rootfs/opt/cni:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]}}' +LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}' diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index fb8ef2e81..f58067802 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -1,6 +1,12 @@ #!/bin/sh -mount --bind /opt/cni /rootfs/opt/cni -mount --bind /etc/cni /rootfs/etc/cni +if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then + mkdir -p /var/lib/cni/opt/bin + tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin + touch /var/lib/cni/.opt.defaults-extracted +fi +if [ -e /etc/kubelet.conf ] ; then + . /etc/kubelet.conf +fi until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ --require-kubeconfig=true \ --pod-manifest-path=/var/lib/kubeadm/manifests \ @@ -10,8 +16,9 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \ --cgroups-per-qos=false \ --enforce-node-allocatable= \ --network-plugin=cni \ - --cni-conf-dir=/etc/cni/net.d \ - --cni-bin-dir=/opt/cni/bin ; do + --cni-conf-dir=/var/lib/cni/etc/net.d \ + --cni-bin-dir=/var/lib/cni/opt/bin \ + $KUBELET_ARGS $@; do if [ ! -f /var/config/userdata ] ; then sleep 1 else