diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile index bc1e046c26..1795179424 100644 --- a/kata-deploy/Dockerfile +++ b/kata-deploy/Dockerfile @@ -1,18 +1,23 @@ FROM centos/systemd -ARG KATA_VER=1.4.0 +ARG KATA_VER ARG ARCH=x86_64 +ARG KUBE_ARCH=amd64 ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA_VER} ARG KATA_FILE=kata-static-${KATA_VER}-${ARCH}.tar.xz -ARG KUBECTL_VER=v1.10.2 RUN \ +yum install -y epel-release && \ +yum install -y bzip2 jq && \ curl -sOL ${KATA_URL}/${KATA_FILE} && \ mkdir -p /opt/kata-artifacts && \ tar xvf ${KATA_FILE} -C /opt/kata-artifacts/ && \ rm ${KATA_FILE} RUN \ -curl -s -o /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER}/bin/linux/amd64/kubectl && \ +curl -Lso /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${KUBE_ARCH}/kubectl && \ chmod +x /bin/kubectl COPY scripts /opt/kata-artifacts/scripts +RUN \ +ln -s /opt/kata-artifacts/scripts/kata-deploy-docker.sh /usr/bin/kata-deploy-docker && \ +ln -s /opt/kata-artifacts/scripts/kata-deploy.sh /usr/bin/kata-deploy diff --git a/kata-deploy/README.md b/kata-deploy/README.md index 9a181a0cb1..f66f6b0e14 100644 --- a/kata-deploy/README.md +++ b/kata-deploy/README.md @@ -2,7 +2,11 @@ - [kata-deploy](#kata-deploy) - * [Quick start](#quick-start-) + * [Docker quick start](#docker-quick-start-) + + [Install Kata and configure Docker](#install-kata-and-configure-docker) + + [Run a sample workload](#run-a-sample-workload-utilizing-kata-containers) + + [Remove Kata](#remove-kata) + * [Kubernetes quick start](#kubernetes-quick-start-) + [Install Kata on a running Kubernetes cluster](#install-kata-on-a-running-kubernetes-cluster) + [Run a sample workload](#run-a-sample-workload-) + [Remove Kata from the Kubernetes cluster](#remove-kata-from-the-kubernetes-cluster-) @@ -16,12 +20,54 @@ [kata-deploy](kata-deploy) provides a Dockerfile, which contains all of the binaries and artifacts required to run Kata Containers, as well as reference daemonsets, which can -be utilized to install Kata Containers on a running Kubernetes cluster. +be utilized to install Kata Containers for both Docker and on a running Kubernetes cluster. -Note, installation through daemonsets successfully installs `kata-containers.io/kata-runtime` on +Note, installation through daemonsets successfully installs `katacontainers.io/kata-runtime` on a node only if it uses either containerd or CRI-O CRI-shims. -## Quick start: +## Docker quick start: + +The kata-deploy container image makes use of a script, `kata-deploy-docker`, for installation of +Kata artifacts and configuration of Docker to utilize the runtime. The following volumes are required to be mounted +to aid in this: +- /opt/kata: this is where all kata artifacts are installed on the system +- /var/run/dbus, /run/systemd: this is require for reloading the the Docker service +- /etc/docker: this is required for updating `daemon.json` in order to configure the kata runtimes in Docker + + +### Install Kata and configure Docker + +To install: + +``` +docker run -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker install +``` + +Once complete, `/etc/docker/daemon.json` is updated or created to include the Kata runtimes: kata-qemu and kata-fc, for utilizing +QEMU and Firecracker, respectively, for the VM isolation layer. + +### Run a sample workload utilizing Kata containers: + +Run a QEMU QEMU isolated Kata container: +``` +docker run --runtime=kata-qemu -itd alpine +``` + +Run a Firecracker isolated Kata container: +``` +docker run --runtime=kata-fc -itd alpine +``` + +### Remove Kata + +To uninstall: +``` +docker run -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker remove +``` + +After completing, the original daemon.json, if it existed, is restored and all Kata artifacts from /opt/kata are removed. + +## Kubernetes quick start ### Install Kata on a running Kubernetes cluster @@ -32,49 +78,44 @@ kubectl apply -f kata-deploy.yaml ### Run a sample workload -Untrusted workloads can node-select based on ```kata-containers.io/kata-runtime=true```, and are -run through ```kata-containers.io/kata-runtime``` if they are marked with the appropriate CRIO or containerd -annotation: -``` -CRIO: io.kubernetes.cri-o.TrustedSandbox: "false" -containerd: io.kubernetes.cri.untrusted-workload: "true" -``` -The following is a sample workload for running untrusted on a kata-enabled node: +Workloads which utilize Kata can node-select based on ```katacontainers.io/kata-runtime=true```, and are +run through an applicable runtime if they are marked with the appropriate runtimeClass annotation. + + +The following YAML snippet shows how to specify a workload should use Kata with QEMU: ``` -apiVersion: v1 -kind: Pod -metadata: - name: nginx - annotations: - io.kubernetes.cri-o.TrustedSandbox: "false" - io.kubernetes.cri.untrusted-workload: "true" - labels: - env: test spec: - containers: - - name: nginx - image: nginx - imagePullPolicy: IfNotPresent - nodeSelector: - kata-containers.io/kata-runtime: "true" -``` - -To run: -``` -kubectl apply -f examples/nginx-untrusted.yaml + template: + spec: + runtimeClassName: kata-qemu ``` -Now, you should see the pod start. You can verify that the pod is making use of -```kata-containers.io/kata-runtime``` by comparing the container ID observed with the following: +The following YAML snippet shows how to specify a workload should use Kata with Firecracker: ``` -/opt/kata/bin/kata-containers.io/kata-runtime list -kubectl describe pod nginx-untrusted +spec: + template: + spec: + runtimeClassName: kata-fc ``` -The following removes the test pod: + +To run an example with kata-qemu: + ``` -kubectl delete -f examples/nginx-untrusted.yaml +kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-qemu.yaml +``` + +To run an example with kata-fc: + +``` +kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-fc.yaml +``` + +The following removes the test pods: +``` +kubectl delete -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-qemu.yaml +kubectl delete -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-fc.yaml ``` ### Remove Kata from the Kubernetes cluster @@ -89,16 +130,18 @@ kubectl delete -f kata-rbac.yaml ## kata-deploy Details ### Dockerfile - -The Dockerfile used to create the container image deployed in the DaemonSet is provided here. -This image contains all the necessary artifacts for running Kata Containers. +The [Dockerfile](kata-deploy/Dockerfile) used to create the container image deployed in the DaemonSet is provided here. +This image contains all the necessary artifacts for running Kata Containers, all of which are pulled +from the [Kata Containers release page](https://github.com/kata-containers/runtime/releases). Host artifacts: -* kata-containers.io/kata-runtime: pulled from Kata GitHub releases page -* kata-proxy: pulled from Kata GitHub releases page -* kata-shim: pulled from Kata GitHub releases page -* qemu-system-x86_64: statically built and included in this repo, based on Kata's QEMU repo -* qemu/* : supporting binaries required for qemu-system-x86_64 +* kata-runtime +* kata-fc +* kata-qemu +* kata-proxy +* kata-shim +* firecracker +* qemu-system-x86_64 and supporting binaries Virtual Machine artifacts: * kata-containers.img: pulled from Kata github releases page @@ -106,27 +149,19 @@ Virtual Machine artifacts: ### Daemonsets and RBAC: -A few daemonsets are introduced for kata-deploy, as well as an RBAC to facilitate +Two daemonsets are introduced for kata-deploy, as well as an RBAC to facilitate applying labels to the nodes. -#### runtime-labeler: +#### Kata installer: kata-deploy -This daemonset creates a label on each node in -the cluster identifying the CRI shim in use. For example, -`kata-containers.io/container-runtime=crio` or `kata-containers.io/container-runtime=containerd.` - -#### CRI-O and containerd kata installer - -Depending on the value of `kata-containers.io/container-runtime` label on the node, either the CRI-O or -containerd kata installation daemonset executes. These daemonsets install -the necessary kata binaries, configuration files, and virtual machine artifacts on -the node. Once installed, the daemonset adds a node label `kata-containers.io/kata-runtime=true` and reconfigures -either CRI-O or containerd to make use of Kata for untrusted workloads. As a final step the daemonset -restarts either CRI-O or containerd and kubelet. Upon deletion, the daemonset removes the kata binaries -and VM artifacts and updates the node label to `kata-containers.io/kata-runtime=cleanup.` +This daemonset installs the necessary kata binaries, configuration files, and virtual machine artifacts on +the node. Once installed, the daemonset adds a node label `katacontainers.io/kata-runtime=true` and reconfigures +either CRI-O or containerd to register two runtimeClasses: `kata-qemu` (for QEMU isolation) and `kata-fc` (for Firecracker isolation). +As a final step the daemonset restarts either CRI-O or containerd. Upon deletion, the daemonset removes the +Kata binaries and VM artifacts and updates the node label to `katacontainers.io/kata-runtime=cleanup.` ### Kata cleanup: -This daemonset runs of the node has the label `kata-containers.io/kata-runtime=cleanup.` These daemonsets removes -the `kata-containers.io/container-runtime` and `kata-containers.io/kata-runtime` labels as well as restarts either CRI-O or containerd systemctl -daemon and kubelet. You cannot execute these resets during the preStopHook of the Kata installer daemonset, +This daemonset runs of the node has the label `katacontainers.io/kata-runtime=cleanup.` These daemonsets removes +the `katacontainers.io/kata-runtime` label as well as restarts either CRI-O or containerd systemctl +daemon. You cannot execute these resets during the preStopHook of the Kata installer daemonset, which necessitated this final cleanup daemonset. diff --git a/kata-deploy/examples/test-deploy-kata-fc.yaml b/kata-deploy/examples/test-deploy-kata-fc.yaml new file mode 100644 index 0000000000..498de27ae9 --- /dev/null +++ b/kata-deploy/examples/test-deploy-kata-fc.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: php-apache-kata-fc + name: php-apache-kata-fc +spec: + replicas: 1 + selector: + matchLabels: + run: php-apache-kata-fc + template: + metadata: + labels: + run: php-apache-kata-fc + spec: + runtimeClassName: kata-fc + containers: + - image: k8s.gcr.io/hpa-example + imagePullPolicy: Always + name: php-apache + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 200m + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache-kata-fc +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: php-apache-kata-fc + sessionAffinity: None + type: ClusterIP diff --git a/kata-deploy/examples/test-deploy-kata-qemu.yaml b/kata-deploy/examples/test-deploy-kata-qemu.yaml new file mode 100644 index 0000000000..84fd1bfeda --- /dev/null +++ b/kata-deploy/examples/test-deploy-kata-qemu.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: php-apache-kata-qemu + name: php-apache-kata-qemu +spec: + replicas: 1 + selector: + matchLabels: + run: php-apache-kata-qemu + template: + metadata: + annotations: + io.kubernetes.cri-o.TrustedSandbox: "false" + io.kubernetes.cri.untrusted-workload: "true" + labels: + run: php-apache-kata-qemu + spec: + runtimeClassName: kata-qemu + containers: + - image: k8s.gcr.io/hpa-example + imagePullPolicy: Always + name: php-apache + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 200m + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache-kata-qemu +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: php-apache-kata-qemu + sessionAffinity: None + type: ClusterIP diff --git a/kata-deploy/examples/test-deploy-runc.yaml b/kata-deploy/examples/test-deploy-runc.yaml new file mode 100644 index 0000000000..c7702bc0f5 --- /dev/null +++ b/kata-deploy/examples/test-deploy-runc.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: php-apache-runc + name: php-apache-runc +spec: + replicas: 1 + selector: + matchLabels: + run: php-apache-runc + template: + metadata: + labels: + run: php-apache-runc + spec: + containers: + - image: k8s.gcr.io/hpa-example + imagePullPolicy: Always + name: php-apache + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 200m + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache-runc +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: php-apache-runc + sessionAffinity: None + type: ClusterIP diff --git a/kata-deploy/kata-cleanup.yaml b/kata-deploy/kata-cleanup.yaml index 92b75d9efe..f2c26d0243 100644 --- a/kata-deploy/kata-cleanup.yaml +++ b/kata-deploy/kata-cleanup.yaml @@ -15,19 +15,12 @@ spec: spec: serviceAccountName: kata-label-node nodeSelector: - kata-containers.io/kata-runtime: cleanup + katacontainers.io/kata-runtime: cleanup containers: - name: kube-kata-cleanup image: katadocker/kata-deploy imagePullPolicy: Always - command: [ "sh", "-c" ] - args: - - kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime-; - systemctl daemon-reload; - systemctl restart containerd; - systemctl restart crio; - systemctl restart kubelet; - sleep infinity; + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "reset" ] env: - name: NODE_NAME valueFrom: diff --git a/kata-deploy/kata-deploy.yaml b/kata-deploy/kata-deploy.yaml index f618784fe4..a30c93a5e6 100644 --- a/kata-deploy/kata-deploy.yaml +++ b/kata-deploy/kata-deploy.yaml @@ -2,71 +2,27 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: kubelet-runtime-labeler + name: kata-deploy namespace: kube-system spec: selector: matchLabels: - name: kubelet-runtime-labeler + name: kata-deploy template: metadata: labels: - name: kubelet-runtime-labeler + name: kata-deploy spec: serviceAccountName: kata-label-node containers: - - name: kubelet-runtime-labeler-pod - image: katadocker/kata-deploy - imagePullPolicy: Always - command: [ "sh", "-c" ] - args: - - printenv NODE_NAME; - kubectl get node $NODE_NAME --show-labels; - kubectl label node $NODE_NAME kata-containers.io/container-runtime=$(kubectl describe node $NODE_NAME | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '); - kubectl get node $NODE_NAME --show-labels; - sleep infinity; - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubelet-cri-o-kata - namespace: kube-system -spec: - selector: - matchLabels: - name: kubelet-cri-o-kata - template: - metadata: - labels: - name: kubelet-cri-o-kata - spec: - serviceAccountName: kata-label-node - nodeSelector: - kata-containers.io/container-runtime: cri-o - containers: - name: kube-kata image: katadocker/kata-deploy imagePullPolicy: Always lifecycle: preStop: exec: - command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-crio.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] - command: [ "sh", "-ce" ] - args: - - /opt/kata-artifacts/scripts/install-kata-crio.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; - kubectl get node $NODE_NAME --show-labels; - sleep infinity; + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "cleanup"] + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "install" ] env: - name: NODE_NAME valueFrom: @@ -77,6 +33,8 @@ spec: volumeMounts: - name: crio-conf mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ - name: kata-artifacts mountPath: /opt/kata/ - name: dbus @@ -87,6 +45,9 @@ spec: - name: crio-conf hostPath: path: /etc/crio/ + - name: containerd-conf + hostPath: + path: /etc/containerd/ - name: kata-artifacts hostPath: path: /opt/kata/ @@ -101,69 +62,3 @@ spec: rollingUpdate: maxUnavailable: 1 type: RollingUpdate ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubelet-cri-containerd-kata - namespace: kube-system -spec: - selector: - matchLabels: - name: kubelet-cri-containerd-kata - template: - metadata: - labels: - name: kubelet-cri-containerd-kata - spec: - serviceAccountName: kata-label-node - nodeSelector: - kata-containers.io/container-runtime: containerd - containers: - - name: kube-kata - image: katadocker/kata-deploy - imagePullPolicy: Always - lifecycle: - preStop: - exec: - command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-containerd.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] - command: [ "sh", "-c" ] - args: - - /opt/kata-artifacts/scripts/install-kata-containerd.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; - kubectl get node $NODE_NAME --show-labels; - sleep infinity; - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - volumeMounts: - - name: containerd-conf - mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - volumes: - - name: containerd-conf - hostPath: - path: /etc/containerd/ - type: DirectoryOrCreate - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate diff --git a/kata-deploy/scripts/install-kata-containerd.sh b/kata-deploy/scripts/install-kata-containerd.sh deleted file mode 100755 index 085b83971b..0000000000 --- a/kata-deploy/scripts/install-kata-containerd.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -echo "copying kata artifacts onto host" -cp -R /opt/kata-artifacts/opt/kata/* /opt/kata/ -chmod +x /opt/kata/bin/* - -# Configure containerd to use Kata: -echo "create containerd configuration for Kata" -mkdir -p /etc/containerd/ - -if [ -f /etc/containerd/config.toml ]; then - cp /etc/containerd/config.toml /etc/containerd/config.toml.bak -fi - -cat <&2 + exit 1 +} + +function print_usage() { + echo "Usage: $0 [install/remove]" +} + +function install_artifacts() { + echo "copying kata artifacts onto host" + cp -a /opt/kata-artifacts/opt/kata/* /opt/kata/ + chmod +x /opt/kata/bin/* +} + +function configure_docker() { + echo "configuring docker" + + cat < "${tmp_file}" + mv "${tmp_file}" "${conf_file}" + rm "${snippet}" + else + mv "${snippet}" "${conf_file}" + fi + + systemctl daemon-reload + systemctl reload docker +} + +function remove_artifacts() { + echo "deleting kata artifacts" + rm -rf /opt/kata/ +} + +function cleanup_runtime() { + echo "cleanup docker" + rm -f "${conf_file}" + + if [ -f "${conf_file_backup}" ]; then + cp "${conf_file_backup}" "${conf_file}" + fi + systemctl daemon-reload + systemctl reload docker +} + +function main() { + # script requires that user is root + euid=`id -u` + if [[ $euid -ne 0 ]]; then + die "This script must be run as root" + fi + + action=${1:-} + if [ -z $action ]; then + print_usage + die "invalid arguments" + fi + + case $action in + install) + install_artifacts + configure_docker + ;; + remove) + cleanup_runtime + remove_artifacts + ;; + *) + echo invalid arguments + print_usage + ;; + esac +} + + +main $@ diff --git a/kata-deploy/scripts/kata-deploy.sh b/kata-deploy/scripts/kata-deploy.sh new file mode 100755 index 0000000000..c073e5a4ac --- /dev/null +++ b/kata-deploy/scripts/kata-deploy.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +crio_conf_file="/etc/crio/crio.conf" +crio_conf_file_backup="${crio_conf_file}.bak" +containerd_conf_file="/etc/containerd/config.toml" +containerd_conf_file_backup="${containerd_conf_file}.bak" +# If we fail for any reason a message will be displayed +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function print_usage() { + echo "Usage: $0 [install/cleanup/reset]" +} + +function get_container_runtime() { + local runtime=$(kubectl describe node $NODE_NAME) + if [ "$?" -ne 0 ]; then + die "invalid node name" + fi + echo "$runtime" | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' ' +} + +function install_artifacts() { + echo "copying kata artifacts onto host" + cp -a /opt/kata-artifacts/opt/kata/* /opt/kata/ + chmod +x /opt/kata/bin/* +} + +function configure_cri_runtime() { + case $1 in + crio) + configure_crio + ;; + containerd) + configure_containerd + ;; + esac + systemctl daemon-reload + systemctl restart $1 +} + +function configure_crio() { + # Configure crio to use Kata: + echo "Add Kata Containers as a supported runtime for CRIO:" + + # backup the CRIO.conf only if a backup doesn't already exist (don't override original) + cp -n "$crio_conf_file" "$crio_conf_file_backup" + + cat <