Merge pull request #2518 from ijc/kubernetes-cri

Kube project updates, including cri-containerd integration

projects/kubernetes/Makefile

@@ -1,3 +1,5 @@
+KUBE_RUNTIME ?= docker
+
 all: tag-container-images build-vm-images
 
 tag-container-images:
@@ -12,11 +14,11 @@ push-container-images:
 
 build-vm-images: kube-master.iso kube-node.iso
 
-kube-master.iso: kube-master.yml
+kube-master.iso: kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
-	moby build -name kube-master -format iso-efi -format iso-bios kube-master.yml
+	moby build -name kube-master -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml $(KUBE_RUNTIME)-master.yml
 
-kube-node.iso: kube-node.yml
+kube-node.iso: kube.yml $(KUBE_RUNTIME).yml
-	moby build -name kube-node -format iso-efi -format iso-bios kube-node.yml
+	moby build -name kube-node -format iso-efi -format iso-bios kube.yml $(KUBE_RUNTIME).yml
 
 clean:
 	rm -f -r \

projects/kubernetes/README.md

@@ -4,13 +4,16 @@ This project aims to demonstrate how one can create minimal and immutable Kubern
 
 Make sure to `cd projects/kubernetes` first.
 
-Edit `kube-master.yml` and add your public SSH key to `files` section.
-
 Build OS images:
 ```
 make build-vm-images
 ```
 
+By default this will build images using Docker Engine for execution. To instead use cri-containerd use:
+```
+make build-vm-images KUBE_RUNTIME=cri-containerd
+```
+
 Boot Kubernetes master OS image using `hyperkit` on macOS: or `qemu` on Linux:
 ```
 ./boot.sh

projects/kubernetes/boot.sh

@@ -2,13 +2,19 @@
 
 set -e
 
-: ${KUBE_PORT_BASE:=2222}
+: ${KUBE_MASTER_VCPUS:=2}
-: ${KUBE_VCPUS:=2}
+: ${KUBE_MASTER_MEM:=1024}
-: ${KUBE_MEM:=1024}
+: ${KUBE_MASTER_DISK:=4G}
-: ${KUBE_DISK:=4G}
+
+: ${KUBE_NODE_VCPUS:=2}
+: ${KUBE_NODE_MEM:=4096}
+: ${KUBE_NODE_DISK:=8G}
+
 : ${KUBE_NETWORKING:=default}
 : ${KUBE_RUN_ARGS:=}
 : ${KUBE_EFI:=}
+: ${KUBE_MAC:=}
+: ${KUBE_PRESERVE_STATE:=}
 
 [ "$(uname -s)" = "Darwin" ] && KUBE_EFI=1
 
@@ -19,7 +25,11 @@ if [ $# -eq 0 ] ; then
     img="kube-master"
     data=""
     state="kube-master-state"
-elif [ $# -gt 1 ] ; then
+
+    : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS}
+    : ${KUBE_MEM:=$KUBE_MASTER_MEM}
+    : ${KUBE_DISK:=$KUBE_MASTER_DISK}
+elif [ $# -gt 1 ] || [ $# -eq 1 -a -n "${KUBE_PRESERVE_STATE}" ] ; then
     case $1 in
 	''|*[!0-9]*)
 	    echo "Node number must be a number"
@@ -36,6 +46,10 @@ elif [ $# -gt 1 ] ; then
     shift
     data="${*}"
     state="kube-${name}-state"
+
+    : ${KUBE_VCPUS:=$KUBE_NODE_VCPUS}
+    : ${KUBE_MEM:=$KUBE_NODE_MEM}
+    : ${KUBE_DISK:=$KUBE_NODE_DISK}
 else
     echo "Usage:"
     echo " - Boot master:"
@@ -45,5 +59,11 @@ else
     exit 1
 fi
 set -x
-rm -rf "${state}"
+if [ -z "${KUBE_PRESERVE_STATE}" ] ; then
+    rm -rf "${state}"
+    mkdir "${state}"
+    if [ -n "${KUBE_MAC}" ] ; then
+	echo -n "${KUBE_MAC}" > "${state}"/mac-addr
+    fi
+fi
 linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" ${uefi} "${img}${suffix}"

projects/kubernetes/cri-containerd.yml

@@ -0,0 +1,7 @@
+services:
+  - name: cri-containerd
+    image: linuxkitprojects/cri-containerd:b8b6a48426c2165055534b06fb0119f07e24506a
+files:
+  - path: /etc/kubelet.conf
+    contents: |
+      KUBELET_ARGS="--container-runtime=remote --container-runtime-endpoint=unix:///var/run/cri-containerd.sock"

projects/kubernetes/cri-containerd/Dockerfile

@@ -0,0 +1,49 @@
+FROM linuxkit/alpine:a120ad6aead3fe583eaa20e9b75a05ac1b3487da AS build
+
+RUN \
+  apk add \
+  bash \
+  gcc \
+  git \
+  go \
+  libc-dev \
+  make \
+  && true
+ENV GOPATH=/go PATH=$PATH:/go/bin
+
+ENV CRI_CONTAINERD_URL https://github.com/kubernetes-incubator/cri-containerd.git
+#ENV CRI_CONTAINERD_BRANCH pull/NNN/head
+ENV CRI_CONTAINERD_COMMIT a8d49402859167a232b094d971e70c2f4b71b8ea
+RUN mkdir -p $GOPATH/src/github.com/kubernetes-incubator && \
+    cd $GOPATH/src/github.com/kubernetes-incubator && \
+    git clone $CRI_CONTAINERD_URL cri-containerd
+WORKDIR $GOPATH/src/github.com/kubernetes-incubator/cri-containerd
+RUN set -e; \
+    if [ -n "$CRI_CONTAINERD_BRANCH" ] ; then \
+        git fetch origin "$CRI_CONTAINERD_BRANCH"; \
+    fi; \
+    git checkout $CRI_CONTAINERD_COMMIT
+RUN make static-binaries
+
+RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
+# util-linux because a full ns-enter is required.
+# example commands: /usr/bin/nsenter --net= -F -- <ip commandline>
+#                   /usr/bin/nsenter --net=/var/run/netns/cni-5e8acebe-810d-c1b9-ced0-47be2f312fa8 -F -- <ip commandline>
+# NB the first ("--net=") is actually not valid -- see https://github.com/kubernetes-incubator/cri-containerd/issues/245
+RUN apk add --no-cache --initdb -p /out \
+    alpine-baselayout \
+    busybox \
+    ca-certificates \
+    iptables \
+    util-linux \
+    && true
+# Remove apk residuals. We have a read-only rootfs, so apk is of no use.
+RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
+
+RUN make DESTDIR=/out install
+
+FROM scratch
+WORKDIR /
+ENTRYPOINT ["cri-containerd", "-v", "2", "--alsologtostderr", "--network-bin-dir", "/var/lib/cni/opt/bin", "--network-conf-dir", "/var/lib/cni/etc/net.d"]
+COPY --from=build /out /
+LABEL org.mobyproject.config='{"binds": ["/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/tmp:/tmp", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/etc/cni:rshared,rbind", "/var/lib/cni/opt:/opt/cni:rshared,rbind", "/run/containerd/containerd.sock:/run/containerd/containerd.sock"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc/net.d", "/var/lib/cni/opt"]}}'

projects/kubernetes/cri-containerd/Makefile

@@ -0,0 +1,7 @@
+ORG?=linuxkitprojects
+IMAGE=cri-containerd
+NETWORK=1
+NOTRUST=1
+ARCHES=x86_64
+
+include ../../../pkg/package.mk

projects/kubernetes/docker-master.yml

@@ -0,0 +1,3 @@
+services:
+  - name: kubernetes-image-cache-control-plane
+    image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25

projects/kubernetes/docker.yml

@@ -0,0 +1,27 @@
+services:
+  - name: docker
+    image: docker:17.07.0-ce-dind
+    capabilities:
+     - all
+    pid: host
+    mounts:
+     - type: cgroup
+       options: ["rw","nosuid","noexec","nodev","relatime"]
+    binds:
+     - /dev:/dev
+     - /etc/resolv.conf:/etc/resolv.conf
+     - /lib/modules:/lib/modules
+     - /run:/run
+     - /var:/var:rshared,rbind
+     - /var/lib/kubeadm:/etc/kubernetes
+     - /var/lib/cni/etc:/etc/cni:rshared,rbind
+     - /var/lib/cni/opt:/opt/cni:rshared,rbind
+    rootfsPropagation: shared
+    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
+    runtime:
+      mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]
+  - name: kubernetes-image-cache-common
+    image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
+files:
+  - path: /etc/kubelet.conf
+    contents: ""

projects/kubernetes/kube-master.yml

@@ -1,66 +0,0 @@
-kernel:
-  image: linuxkit/kernel:4.9.50
-  cmdline: "console=tty0 console=ttyS0"
-init:
-  - linuxkit/init:851e9c3ad0574d640b733b92fdb26c368d2f7f8f
-  - linuxkit/runc:a1b564248a0d0b118c11e61db9f84ecf41dd2d2a
-  - linuxkit/containerd:06876ceef325e49e9ba119659357768d5df89075
-  - linuxkit/ca-certificates:e44b0a66df5a102c0e220f0066b0d904710dcb10
-onboot:
-  - name: sysctl
-    image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051
-  - name: sysfs
-    image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9
-  - name: dhcpcd
-    image: linuxkit/dhcpcd:f3f5413abb78fae9020e35bd4788fa93df4530b7
-    command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
-  - name: metadata
-    image: linuxkit/metadata:da3138079c168e0c5608d8f3853366c113ed91d2
-  - name: format
-    image: linuxkit/format:158d992b7bf7ab984100c697d7e72161ea7d7382
-  - name: mounts
-    image: linuxkit/mount:4fe245efb01384e42622c36302e13e386bbaeb08
-    command: ["/usr/bin/mountie", "/var/lib/"]
-services:
-  - name: getty
-    image: linuxkit/getty:797cb79e0a229fcd16ebf44a0da74bcec03968ec
-    env:
-     - INSECURE=true
-  - name: rngd
-    image: linuxkit/rngd:558e86a36242bb74353bc9287b715ddb8567357e
-  - name: ntpd
-    image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67
-  - name: sshd
-    image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0
-  - name: docker
-    image: docker:17.07.0-ce-dind
-    capabilities:
-     - all
-    pid: host
-    mounts:
-     - type: cgroup
-       options: ["rw","nosuid","noexec","nodev","relatime"]
-    binds:
-     - /dev:/dev
-     - /etc/resolv.conf:/etc/resolv.conf
-     - /lib/modules:/lib/modules
-     - /run:/run
-     - /var:/var:rshared,rbind
-     - /var/lib/kubeadm:/etc/kubernetes
-     - /var/lib/cni/etc:/etc/cni:rshared,rbind
-     - /var/lib/cni/opt:/opt/cni:rshared,rbind
-    rootfsPropagation: shared
-    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
-    runtime:
-      mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]
-  - name: kubernetes-image-cache-common
-    image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
-  - name: kubernetes-image-cache-control-plane
-    image: linuxkitprojects/kubernetes-image-cache-control-plane:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
-  - name: kubelet
-    image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
-files:
-  - path: root/.ssh/authorized_keys
-    source: ~/.ssh/id_rsa.pub
-    mode: "0600"
-    optional: true

projects/kubernetes/kube.yml

@@ -9,6 +9,9 @@ init:
 onboot:
   - name: sysctl
     image: linuxkit/sysctl:154913b72c6f1f33eb408609fca9963628e8c051
+    binds:
+     - /etc/sysctl.d/01-kubernetes.conf:/etc/sysctl.d/01-kubernetes.conf
+    readonly: false
   - name: sysfs
     image: linuxkit/sysfs:3ae01a25583ee37a5ff8b09a0e569cb4bd8cf2e9
   - name: dhcpcd
@@ -32,32 +35,19 @@ services:
     image: linuxkit/openntpd:0d7befc79842849d0b88d6c3b64200e340d7cf67
   - name: sshd
     image: linuxkit/sshd:505a985d7bd7a90f15eca9cb4dc6ec92789d51a0
-  - name: docker
-    image: docker:17.07.0-ce-dind
-    capabilities:
-     - all
-    pid: host
-    mounts:
-     - type: cgroup
-       options: ["rw","nosuid","noexec","nodev","relatime"]
-    binds:
-     - /dev:/dev
-     - /etc/resolv.conf:/etc/resolv.conf
-     - /lib/modules:/lib/modules
-     - /run:/run
-     - /var:/var:rshared,rbind
-     - /var/lib/kubeadm:/etc/kubernetes
-     - /var/lib/cni/etc:/etc/cni:rshared,rbind
-     - /var/lib/cni/opt:/opt/cni:rshared,rbind
-    rootfsPropagation: shared
-    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
-    runtime:
-      mkdir: ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]
-  - name: kubernetes-image-cache-common
-    image: linuxkitprojects/kubernetes-image-cache-common:0d818c5b1a7a0a0aa52c2a52e23de784d7fd5e25
   - name: kubelet
-    image: linuxkitprojects/kubernetes:c4a6ae5121df50471ad244b9fc153ff5eb674fb2
+    image: linuxkitprojects/kubernetes:b73aacdfaad2167f7b193d9b68f7e52186eb188a
 files:
+  - path: etc/linuxkit.yml
+    metadata: yaml
+  - path: /etc/kubernetes
+    symlink: "/var/lib/kubeadm"
+  - path: /etc/sysctl.d/01-kubernetes.conf
+    contents: 'net.ipv4.ip_forward = 1'
+  - path: /opt/cni
+    directory: true
+  - path: /etc/cni
+    directory: true
   - path: root/.ssh/authorized_keys
     source: ~/.ssh/id_rsa.pub
     mode: "0600"

projects/kubernetes/kubernetes/Dockerfile

@@ -30,9 +30,7 @@ RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
 
 RUN rmdir /out/var/run && ln -nfs /run /out/var/run
 
-RUN curl -fSL -o /tmp/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz && \
+RUN curl -fSL -o /out/root/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz
-    mkdir -p /out/opt/cni/bin /out/etc/cni/net.d && \
-    tar -xzf /tmp/cni.tgz -C /out/opt/cni/bin
 RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.7/net?v=${weave_version} 
 RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet
 RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm
@@ -47,4 +45,4 @@ WORKDIR /
 ENTRYPOINT ["/usr/bin/kubelet.sh"]
 COPY --from=build /out /
 ENV KUBECONFIG "/etc/kubernetes/admin.conf"
-LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/var/lib/cni/etc:/rootfs/etc/cni:rshared,rbind", "/var/lib/cni/opt:/rootfs/opt/cni:rshared,rbind"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"]}}'
+LABEL org.mobyproject.config='{"binds": ["/dev:/dev", "/etc/resolv.conf:/etc/resolv.conf", "/run:/run:rshared,rbind", "/var:/var:rshared,rbind", "/var/lib/kubeadm:/etc/kubernetes", "/etc/kubelet.conf:/etc/kubelet.conf"], "mounts": [{"type": "cgroup", "options": ["rw","nosuid","noexec","nodev","relatime"]}], "capabilities": ["all"], "rootfsPropagation": "shared", "pid": "host", "runtime": {"mkdir": ["/var/lib/kubeadm", "/var/lib/cni/etc", "/var/lib/cni/opt"], "mounts": [{"type": "bind", "source": "/var/lib/cni/opt", "destination": "/opt/cni", "options": ["rw", "bind"]}, {"type": "bind", "source": "/var/lib/cni/etc", "destination": "/etc/cni", "options": ["rw", "bind"]}]}}'

projects/kubernetes/kubernetes/kubelet.sh

@@ -1,6 +1,12 @@
 #!/bin/sh
-mount --bind /opt/cni /rootfs/opt/cni
+if [ ! -e /var/lib/cni/.opt.defaults-extracted ] ; then
-mount --bind /etc/cni /rootfs/etc/cni
+    mkdir -p /var/lib/cni/opt/bin
+    tar -xzf /root/cni.tgz -C /var/lib/cni/opt/bin
+    touch /var/lib/cni/.opt.defaults-extracted
+fi
+if [ -e /etc/kubelet.conf ] ; then
+    . /etc/kubelet.conf
+fi
 until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --require-kubeconfig=true \
 	      --pod-manifest-path=/var/lib/kubeadm/manifests \
@@ -10,8 +16,9 @@ until kubelet --kubeconfig=/var/lib/kubeadm/kubelet.conf \
 	      --cgroups-per-qos=false \
 	      --enforce-node-allocatable= \
 	      --network-plugin=cni \
-	      --cni-conf-dir=/etc/cni/net.d \
+	      --cni-conf-dir=/var/lib/cni/etc/net.d \
-	      --cni-bin-dir=/opt/cni/bin ; do
+	      --cni-bin-dir=/var/lib/cni/opt/bin \
+	      $KUBELET_ARGS $@; do
     if [ ! -f /var/config/userdata ] ; then
 	sleep 1
     else