diff --git a/README.md b/README.md index 7577246d7..82db01dff 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ # Kata Containers packaging -This repository is used to generate packages for Kata Containers components. +Kata Containers currently supports packages for many distributions. Tooling to aid in creating these +packages are contained within this repository. + +In addition, Kata build artifacts are available within a container image, created by a +[Dockerfile](kata-deploy/Dockerfile). Reference daemonsets are provided in [kata-deploy](kata-deploy), +which make installation of Kata Containers in a running Kubernetes Cluster very straightforward. diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile new file mode 100644 index 000000000..543804bde --- /dev/null +++ b/kata-deploy/Dockerfile @@ -0,0 +1,16 @@ +FROM centos/systemd +ARG KATA_VER=1.1.0 +ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA_VER} +ARG KUBECTL_VER=v1.10.2 + +RUN \ +curl -sOL ${KATA_URL}/kata-release-binaries.tar.xz && \ +mkdir -p /opt/kata-artifacts && \ +tar xvf kata-release-binaries.tar.xz -C /opt/kata-artifacts && \ +rm kata-release-binaries.tar.xz + +RUN \ +curl -s -o /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER}/bin/linux/amd64/kubectl && \ +chmod +x /bin/kubectl + +COPY scripts /opt/kata-artifacts/scripts diff --git a/kata-deploy/README.md b/kata-deploy/README.md new file mode 100644 index 000000000..92bb68b4c --- /dev/null +++ b/kata-deploy/README.md @@ -0,0 +1,132 @@ +# kata-deploy + + +- [kata-deploy](#kata-deploy) + * [Quick start](#quick-start-) + + [Install Kata on a running Kubernetes cluster](#install-kata-on-a-running-kubernetes-cluster) + + [Run a sample workload](#run-a-sample-workload-) + + [Remove Kata from the Kubernetes cluster](#remove-kata-from-the-kubernetes-cluster-) + * [kata-deploy details](#kata-deploy-details) + + [Dockerfile](#dockerfile) + + [Daemonsets and RBAC](#daemonsets-and-rbac-) + - [runtime-labeler](#runtime-labeler-) + - [CRI-O and containerd kata installer](#cri-o-and-containerd-kata-installer-) + + [Kata cleanup](#kata-cleanup-) + + +[kata-deploy](kata-deploy) provides a Dockerfile, which contains all of the binaries +and artifacts required to run Kata Containers, as well as reference daemonsets, which can +be utilized to install Kata Containers on a running Kubernetes cluster. + +Note, installation through daemonsets only succesfully installs `kata-containers.io/kata-runtime` on +a node if it uses either containerd or CRI-O CRI-shims. + +## Quick start: + +### Install Kata on a running Kubernetes cluster + +``` +kubectl apply -f kata-rbac.yaml +kubectl apply -f kata-deploy.yaml +``` + +### Run a sample workload + +Untrusted workloads can node-select based on ```kata-containers.io/kata-runtime=true```, and are +run through ```kata-containers.io/kata-runtime``` if they are marked with the appropriate CRIO or containerd +annotation: +``` +CRIO: io.kubernetes.cri-o.TrustedSandbox: "false" +containerd: io.kubernetes.cri.untrusted-workload: "true" +``` + +The following is a sample workload for running untrusted on a kata-enabled node: +``` +apiVersion: v1 +kind: Pod +metadata: + name: nginx + annotations: + io.kubernetes.cri-o.TrustedSandbox: "false" + io.kubernetes.cri.untrusted-workload: "true" + labels: + env: test +spec: + containers: + - name: nginx + image: nginx + imagePullPolicy: IfNotPresent + nodeSelector: + kata-containers.io/kata-runtime: "true" +``` + +To run: +``` +kubectl apply -f examples/nginx-untrusted.yaml +``` + +Now, you should see the pod start. You can verify that the pod is making use of +```kata-containers.io/kata-runtime``` by comparing the container ID observed with the following: +``` +/opt/kata/bin/kata-containers.io/kata-runtime list +kubectl describe pod nginx-untrusted +``` + +The following removes the test pod: +``` +kubectl delete -f examples/nginx-untrusted.yaml +``` + +### Remove Kata from the Kubernetes cluster + +``` +kubectl delete -f kata-deploy.yaml +kubectl apply -f kata-cleanup.yaml +kubectl delete -f kata-cleanup.yaml +kubectl delete -f kata-rbac.yaml +``` + +## kata-deploy Details + +### Dockerfile + +The Dockerfile used to create the container image deployed in the DaemonSet is provided here. +This image contains all the necessary artifacts for running Kata Containers. + +Host artifacts: +* kata-containers.io/kata-runtime: pulled from Kata GitHub releases page +* kata-proxy: pulled from Kata GitHub releases page +* kata-shim: pulled from Kata GitHub releases page +* qemu-system-x86_64: statically built and included in this repo, based on Kata's QEMU repo +* qemu/* : supporting binaries required for qemu-system-x86_64 + +Virtual Machine artifacts: +* kata-containers.img: pulled from Kata github releases page +* vmliuz.container: pulled from Kata github releases page + +### Daemonsets and RBAC: + +A few daemonsets are introduced for kata-deploy, as well as an RBAC to facilitate +appyling labels to the nodes. + +#### runtime-labeler: + +This daemonset creates a label on each node in +the cluster identifying the CRI shim in use. For example, +`kata-containers.io/container-runtime=crio` or `kata-containers.io/container-runtime=containerd.` + +#### CRI-O and containerd kata installer + +Depending the value of `kata-containers.io/container-runtime` label on the node, either the CRI-O or +containerd kata installation daemonset executes. These daemonsets install +the necessary kata binaries, configuration files, and virtual machine artifacts on +the node. Once installed, the daemonset adds a node label `kata-containers.io/kata-runtime=true` and reconfigures +either CRI-O or containerd to make use of Kata for untrusted workloads. As a final step the daemonset +restarts either CRI-O or containerd and kubelet. Upon deletion, the daemonset removes the kata binaries +and VM artifacts and updates the node label to `kata-containers.io/kata-runtime=cleanup.` + +### Kata cleanup: +This daemonset runs of the node has the label `kata-containers.io/kata-runtime=cleanup.` This daemonsets removes +the `kata-containers.io/container-runtime` and `kata-containers.io/kata-runtime` labels as well as restarts either CRI-O or containerd systemctl +daemon and kubelet. You cannot execute these restets during the preStopHook of the Kata installer daemonset, +which necessitated this final cleanup daemonset. diff --git a/kata-deploy/kata-cleanup.yaml b/kata-deploy/kata-cleanup.yaml new file mode 100644 index 000000000..868ba0513 --- /dev/null +++ b/kata-deploy/kata-cleanup.yaml @@ -0,0 +1,53 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-kata-cleanup + namespace: kube-system +spec: + selector: + matchLabels: + name: kubelet-kata-cleanup + template: + metadata: + labels: + name: kubelet-kata-cleanup + spec: + serviceAccountName: kata-label-node + nodeSelector: + kata-containers.io/kata-runtime: cleanup + containers: + - name: kube-kata-cleanup + image: egernst/kata-deploy + imagePullPolicy: Always + command: [ "sh", "-c" ] + args: + - kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime-; + systemctl daemon-reload; + systemctl restart containerd; + systemctl restart crio; + systemctl restart kubelet; + sleep infinity; + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + volumes: + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/kata-deploy/kata-deploy.yaml b/kata-deploy/kata-deploy.yaml new file mode 100644 index 000000000..e7d78747e --- /dev/null +++ b/kata-deploy/kata-deploy.yaml @@ -0,0 +1,181 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-runtime-labeler + namespace: kube-system +spec: + selector: + matchLabels: + name: kubelet-runtime-labeler + template: + metadata: + labels: + name: kubelet-runtime-labeler + spec: + serviceAccountName: kata-label-node + containers: + - name: kubelet-runtime-labeler-pod + image: egernst/kata-deploy + imagePullPolicy: Always + command: [ "sh", "-c" ] + args: + - printenv NODE_NAME; + kubectl get node $NODE_NAME --show-labels; + kubectl label node $NODE_NAME kata-containers.io/container-runtime=$(kubectl describe node $NODE_NAME | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '); + kubectl get node $NODE_NAME --show-labels; + sleep infinity; + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-cri-o-kata + namespace: kube-system +spec: + selector: + matchLabels: + name: kubelet-cri-o-kata + template: + metadata: + labels: + name: kubelet-cri-o-kata + spec: + serviceAccountName: kata-label-node + nodeSelector: + kata-containers.io/container-runtime: cri-o + containers: + - name: kube-kata + image: egernst/kata-deploy + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-crio.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] + command: [ "sh", "-ce" ] + args: + - /opt/kata-artifacts/scripts/install-kata-crio.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; + kubectl get node $NODE_NAME --show-labels; + sleep infinity; + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: kata-conf + mountPath: /usr/share/defaults/kata-containers/ + - name: kata-artifacts + mountPath: /opt/kata/ + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: kata-conf + hostPath: + path: /usr/share/defaults/kata-containers/ + type: DirectoryOrCreate + - name: kata-artifacts + hostPath: + path: /opt/kata/ + type: DirectoryOrCreate + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-cri-containerd-kata + namespace: kube-system +spec: + selector: + matchLabels: + name: kubelet-cri-containerd-kata + template: + metadata: + labels: + name: kubelet-cri-containerd-kata + spec: + serviceAccountName: kata-label-node + nodeSelector: + kata-containers.io/container-runtime: containerd + containers: + - name: kube-kata + image: egernst/kata-deploy + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-containerd.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] + command: [ "sh", "-c" ] + args: + - /opt/kata-artifacts/scripts/install-kata-containerd.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; + kubectl get node $NODE_NAME --show-labels; + sleep infinity; + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: containerd-conf + mountPath: /etc/containerd/ + - name: kata-conf + mountPath: /usr/share/defaults/kata-containers/ + - name: kata-artifacts + mountPath: /opt/kata/ + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + volumes: + - name: containerd-conf + hostPath: + path: /etc/containerd/ + type: DirectoryOrCreate + - name: kata-conf + hostPath: + path: /usr/share/defaults/kata-containers/ + type: DirectoryOrCreate + - name: kata-artifacts + hostPath: + path: /opt/kata/ + type: DirectoryOrCreate + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/kata-deploy/kata-rbac.yaml b/kata-deploy/kata-rbac.yaml new file mode 100644 index 000000000..408b5be90 --- /dev/null +++ b/kata-deploy/kata-rbac.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kata-label-node + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-labeler +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kata-label-node-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-labeler +subjects: +- kind: ServiceAccount + name: kata-label-node + namespace: kube-system + diff --git a/kata-deploy/scripts/install-kata-containerd.sh b/kata-deploy/scripts/install-kata-containerd.sh new file mode 100755 index 000000000..cd2ed9d1f --- /dev/null +++ b/kata-deploy/scripts/install-kata-containerd.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +echo "copying kata artifacts onto host" +cp -R /opt/kata-artifacts/bin /opt/kata/ +mkdir /opt/kata/share +mv /opt/kata/bin/qemu /opt/kata/share/ +chmod +x /opt/kata/bin/* +cp /opt/kata-artifacts/configuration.toml /usr/share/defaults/kata-containers/configuration.toml + +# Update Kata configuration for /opt/kata path usage +sed -i 's!/usr.*kata-containers/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!/usr/bin/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!qemu-lite!qemu!' /usr/share/defaults/kata-containers/configuration.toml + +# Configure containerd to use Kata: +echo "create containerd configuration for Kata" +mkdir -p /etc/containerd/ + +if [ -f /etc/containerd/config.toml ]; then + cp /etc/containerd/config.toml /etc/containerd/config.toml.bak +fi + +cat << EOT | tee /etc/containerd/config.toml +[plugins] + [plugins.cri.containerd] + [plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "/opt/kata/bin/kata-runtime" + runtime_root = "" +EOT + +echo "Reload systemd services" +systemctl daemon-reload +systemctl restart containerd diff --git a/kata-deploy/scripts/install-kata-crio.sh b/kata-deploy/scripts/install-kata-crio.sh new file mode 100755 index 000000000..4e5e7088e --- /dev/null +++ b/kata-deploy/scripts/install-kata-crio.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +echo "copying kata artifacts onto host" +cp -R /opt/kata-artifacts/bin /opt/kata/ +mkdir /opt/kata/share +mv /opt/kata/bin/qemu /opt/kata/share/ +chmod +x /opt/kata/bin/* +cp /opt/kata-artifacts/configuration.toml /usr/share/defaults/kata-containers/configuration.toml + +# Update Kata configuration for /opt/kata path usage +sed -i 's!/usr.*kata-containers/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!/usr/bin/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!qemu-lite!qemu!' /usr/share/defaults/kata-containers/configuration.toml + +echo "Set Kata containers as default runtime in CRI-O for untrusted workloads" +cp /etc/crio/crio.conf /etc/crio/crio.conf.bak +sed -i '/runtime_untrusted_workload = /c\runtime_untrusted_workload = "/opt/kata/bin/kata-runtime"' /etc/crio/crio.conf + +echo "Reload systemd services" +systemctl daemon-reload +systemctl restart crio diff --git a/kata-deploy/scripts/remove-kata-containerd.sh b/kata-deploy/scripts/remove-kata-containerd.sh new file mode 100755 index 000000000..797e4df3a --- /dev/null +++ b/kata-deploy/scripts/remove-kata-containerd.sh @@ -0,0 +1,10 @@ +#!/bin/sh +echo "delete kata artifacts" +rm -rf /opt/kata +rm -rf /usr/share/defaults/kata-containers +rm -f /etc/containerd/config.toml + +if [ -f /etc/containerd/config.toml.bak ]; then + mv /etc/containerd/config.toml.bak /etc/containerd/config.toml +fi + diff --git a/kata-deploy/scripts/remove-kata-crio.sh b/kata-deploy/scripts/remove-kata-crio.sh new file mode 100755 index 000000000..b23e40932 --- /dev/null +++ b/kata-deploy/scripts/remove-kata-crio.sh @@ -0,0 +1,5 @@ +#!/bin/sh +echo "deleting kata artifacts" +rm -rf /opt/kata/ +rm -rf /usr/share/defaults/kata-containers +mv /etc/crio/crio.conf.bak /etc/crio/crio.conf