kata-deploy: add support for runtime class, firecracker

Simplify the yaml and combine the prior scripts. The resulting script,
kata-deploy.sh, is used for  install and configuration and
removal for CRI-O and containerd. While this could be used standalone
outside of daemonsets, today it will sleep infinity after processing the
request, since it is assumed to be called by a daemon.

By checking the CRI runtime within the script itself, we no longer need
to support many daemonsets for deploy - just a single. Still requires a
seperate cleanup daemonset (for restarting the CRI runtime), and an
RBAC.

Verified with CRI-O -- containerd testing WIP

Throwing this up now for feedback since I do not bash good.

Signed-off-by: Eric Ernst <eric.ernst@intel.com>
Signed-off-by: Saikrishna Edupuganti <saikrishna.edupuganti@intel.com>

kata-deploy/Dockerfile

@@ -1,9 +1,9 @@
 FROM centos/systemd
-ARG KATA_VER=1.4.0
+ARG KATA_VER
 ARG ARCH=x86_64
+ARG KUBE_ARCH=amd64
 ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA_VER}
 ARG KATA_FILE=kata-static-${KATA_VER}-${ARCH}.tar.xz
-ARG KUBECTL_VER=v1.10.2
 
 RUN \
 curl -sOL ${KATA_URL}/${KATA_FILE} && \
@@ -12,7 +12,7 @@ tar xvf ${KATA_FILE} -C /opt/kata-artifacts/ && \
 rm ${KATA_FILE}
 
 RUN \
-curl -s -o /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER}/bin/linux/amd64/kubectl && \
+curl -Lso /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${KUBE_ARCH}/kubectl && \
 chmod +x /bin/kubectl
 
 COPY scripts /opt/kata-artifacts/scripts

kata-deploy/kata-cleanup.yaml

@@ -20,14 +20,7 @@ spec:
       - name: kube-kata-cleanup
         image: katadocker/kata-deploy
         imagePullPolicy: Always
-        command: [ "sh", "-c" ]
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "reset" ]
-        args:
-        - kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime-;
-          systemctl daemon-reload;
-          systemctl restart containerd;
-          systemctl restart crio;
-          systemctl restart kubelet;
-          sleep infinity;
         env:
         - name: NODE_NAME
           valueFrom:

kata-deploy/kata-deploy.yaml

@@ -2,71 +2,27 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kubelet-runtime-labeler
+  name: kata-deploy
   namespace: kube-system
 spec:
   selector:
       matchLabels:
-        name: kubelet-runtime-labeler
+        name: kata-deploy
   template:
     metadata:
         labels:
-          name: kubelet-runtime-labeler
+          name: kata-deploy
     spec:
       serviceAccountName: kata-label-node
       containers:
-      - name: kubelet-runtime-labeler-pod
-        image: katadocker/kata-deploy
-        imagePullPolicy: Always
-        command: [ "sh", "-c" ]
-        args:
-        - printenv NODE_NAME;
-          kubectl get node $NODE_NAME --show-labels;
-          kubectl label node $NODE_NAME kata-containers.io/container-runtime=$(kubectl describe node $NODE_NAME | awk -F'[:]' '/Container Runtime Version/ {print $2}' |  tr -d ' ');
-          kubectl get node $NODE_NAME --show-labels;
-          sleep infinity;
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        securityContext:
-          privileged: false
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kubelet-cri-o-kata
-  namespace: kube-system
-spec:
-  selector:
-      matchLabels:
-        name: kubelet-cri-o-kata
-  template:
-    metadata:
-        labels:
-          name: kubelet-cri-o-kata
-    spec:
-      serviceAccountName: kata-label-node
-      nodeSelector:
-          kata-containers.io/container-runtime: cri-o
-      containers:
       - name: kube-kata
         image: katadocker/kata-deploy
         imagePullPolicy: Always
         lifecycle:
           preStop:
             exec:
-              command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-crio.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"]
+              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "cleanup"]
-        command: [ "sh", "-ce" ]
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "install" ]
-        args:
-        - /opt/kata-artifacts/scripts/install-kata-crio.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true;
-          kubectl get node $NODE_NAME --show-labels;
-          sleep infinity;
         env:
         - name: NODE_NAME
           valueFrom:
@@ -77,6 +33,8 @@ spec:
         volumeMounts:
         - name: crio-conf
           mountPath: /etc/crio/
+        - name: containerd-conf
+          mountPath: /etc/containerd/
         - name: kata-artifacts
           mountPath: /opt/kata/
         - name: dbus
@@ -87,72 +45,9 @@ spec:
         - name: crio-conf
           hostPath:
             path: /etc/crio/
-        - name: kata-artifacts
-          hostPath:
-            path: /opt/kata/
-            type: DirectoryOrCreate
-        - name: dbus
-          hostPath:
-            path: /var/run/dbus
-        - name: systemd
-          hostPath:
-            path: /run/systemd
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kubelet-cri-containerd-kata
-  namespace: kube-system
-spec:
-  selector:
-      matchLabels:
-        name: kubelet-cri-containerd-kata
-  template:
-    metadata:
-        labels:
-          name: kubelet-cri-containerd-kata
-    spec:
-      serviceAccountName: kata-label-node
-      nodeSelector:
-          kata-containers.io/container-runtime: containerd
-      containers:
-      - name: kube-kata
-        image: katadocker/kata-deploy
-        imagePullPolicy: Always
-        lifecycle:
-          preStop:
-            exec:
-              command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-containerd.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"]
-        command: [ "sh", "-c" ]
-        args:
-        - /opt/kata-artifacts/scripts/install-kata-containerd.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true;
-          kubectl get node $NODE_NAME --show-labels;
-          sleep infinity;
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        securityContext:
-          privileged: false
-        volumeMounts:
-        - name: containerd-conf
-          mountPath: /etc/containerd/
-        - name: kata-artifacts
-          mountPath: /opt/kata/
-        - name: dbus
-          mountPath: /var/run/dbus
-        - name: systemd
-          mountPath: /run/systemd
-      volumes:
         - name: containerd-conf
           hostPath:
             path: /etc/containerd/
-            type: DirectoryOrCreate
         - name: kata-artifacts
           hostPath:
             path: /opt/kata/

kata-deploy/scripts/install-kata-containerd.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-echo "copying kata artifacts onto host"
-cp -R /opt/kata-artifacts/opt/kata/* /opt/kata/
-chmod +x /opt/kata/bin/*
-
-# Configure containerd to use Kata:
-echo "create containerd configuration for Kata"
-mkdir -p /etc/containerd/
-
-if [ -f /etc/containerd/config.toml ]; then
-	cp /etc/containerd/config.toml /etc/containerd/config.toml.bak
-fi
-
-cat <<EOT | tee /etc/containerd/config.toml
-[plugins]
-    [plugins.cri.containerd]
-      [plugins.cri.containerd.untrusted_workload_runtime]
-        runtime_type = "io.containerd.runtime.v1.linux"
-        runtime_engine = "/opt/kata/bin/kata-runtime"
-        runtime_root = ""
-EOT
-
-echo "Reload systemd services"
-systemctl daemon-reload
-systemctl restart containerd

kata-deploy/scripts/install-kata-crio.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-echo "copying kata artifacts onto host"
-cp -R /opt/kata-artifacts/opt/kata/* /opt/kata/
-chmod +x /opt/kata/bin/*
-
-# Configure crio to use Kata:
-echo "Set Kata containers as default runtime in CRI-O for untrusted workloads"
-cp /etc/crio/crio.conf /etc/crio/crio.conf.bak
-sed -i '/runtime_untrusted_workload = /c\runtime_untrusted_workload = "/opt/kata/bin/kata-runtime"' /etc/crio/crio.conf
-
-echo "Reload systemd services"
-systemctl daemon-reload
-systemctl restart crio

kata-deploy/scripts/kata-deploy.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+crio_conf_file="/etc/crio/crio.conf"
+crio_conf_file_backup="${crio_conf_file}.bak"
+containerd_conf_file="/etc/containerd/config.toml"
+containerd_conf_file_backup="${containerd_conf_file}.bak"
+# If we fail for any reason a message will be displayed
+die() {
+        msg="$*"
+        echo "ERROR: $msg" >&2
+        exit 1
+}
+
+function print_usage() {
+	echo "Usage: $0 [install/cleanup/reset]"
+}
+
+function get_container_runtime() {
+	local runtime=$(kubectl describe node $NODE_NAME)
+	if [ "$?" -ne 0 ]; then
+                die "invalid node name"
+	fi
+	echo "$runtime" | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '
+}
+
+function install_artifacts() {
+	echo "copying kata artifacts onto host"
+	cp -a /opt/kata-artifacts/opt/kata/* /opt/kata/
+	chmod +x /opt/kata/bin/*
+}
+
+function configure_cri_runtime() {
+	case $1 in
+	crio)
+		configure_crio
+		;;
+	containerd)
+		configure_containerd
+		;;
+	esac
+	systemctl daemon-reload
+	systemctl restart $1
+}
+
+function configure_crio() {
+	# Configure crio to use Kata:
+	echo "Add Kata Containers as a supported runtime for CRIO:"
+
+	# backup the CRIO.conf only if a backup doesn't already exist (don't override original)
+	cp -n "$crio_conf_file" "$crio_conf_file_backup"
+
+	cat <<EOT | tee -a "$crio_conf_file"
+[crio.runtime.runtimes.kata-qemu]
+  runtime_path = "/opt/kata/bin/kata-qemu"
+
+[crio.runtime.runtimes.kata-fc]
+  runtime_path = "/opt/kata/bin/kata-fc"
+EOT
+
+	sed -i 's|\(\[crio\.runtime\]\)|\1\nmanage_network_ns_lifecycle = true|' "$crio_conf_file"
+}
+
+function configure_containerd() {
+	# Configure containerd to use Kata:
+	echo "Add Kata Containers as a supported runtime for containerd"
+	mkdir -p /etc/containerd/
+
+	if [ -f "$containerd_conf_file" ]; then
+		cp "$containerd_conf_file" "$containerd_conf_file_backup"
+	fi
+	# TODO: While there isn't a default here anyway, it'd probably be best to
+	#  add sed magic to insert into appropriate location if config.toml already exists
+	# https://github.com/kata-containers/packaging/issues/307
+	cat <<EOT | tee "$containerd_conf_file"
+[plugins]
+    [plugins.cri.containerd]
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = "/opt/kata/bin/kata-runtime"
+        runtime_root = ""
+EOT
+}
+
+function remove_artifacts() {
+	echo "deleting kata artifacts"
+	rm -rf /opt/kata/
+}
+
+function cleanup_cri_runtime() {
+	case $1 in
+	crio)
+		cleanup_crio
+		;;
+	containerd)
+		cleanup_containerd
+		;;
+	esac
+
+}
+function cleanup_crio() {
+	if [ -f "$crio_conf_file_backup" ]; then
+		cp "$crio_conf_file_backup" "$crio_conf_file"
+	fi
+}
+
+function cleanup_containerd() {
+	rm -f /etc/containerd/config.toml
+	if [ -f "$containerd_conf_file_backup" ]; then
+		mv "$containerd_conf_file_backup" "$containerd_conf_file"
+	fi
+
+}
+
+function reset_runtime() {
+	kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime-
+	systemctl daemon-reload
+	systemctl restart $1
+	systemctl restart kubelet
+}
+
+function main() {
+	# script requires that user is root
+	euid=`id -u`
+	if [[ $euid -ne 0 ]]; then
+	   die  "This script must be run as root"
+	fi
+
+	runtime=$(get_container_runtime)
+
+	# CRI-O isn't consistent with the naming -- let's use crio to match the service file
+	if [ "$runtime" == "cri-o" ]; then
+		runtime="crio"
+	fi
+
+	action=${1:-}
+	if [ -z $action ]; then
+		print_usage
+		die "invalid arguments"
+	fi
+
+	# only install / remove / update if we are dealing with CRIO or containerd
+	if [ "$runtime" == "crio" ] || [ "$runtime" == "containerd" ]; then
+
+		case $action in
+		install)
+
+			install_artifacts
+			configure_cri_runtime $runtime
+			;;
+		cleanup)
+			remove_artifacts
+			cleanup_cri_runtime $runtime
+			kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup
+			;;
+		reset)
+			reset_runtime $runtime
+			;;
+		*)
+			echo invalid arguments
+			print_usage
+			;;
+		esac
+	fi
+
+	#It is assumed this script will be called as a daemonset. As a result, do
+        # not return, otherwise the daemon will restart and rexecute the script
+	sleep infinity
+}
+
+main $@

kata-deploy/scripts/remove-kata-containerd.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-echo "delete kata artifacts"
-rm -rf /opt/kata
-rm -f /etc/containerd/config.toml
-
-if [ -f /etc/containerd/config.toml.bak ]; then
-	mv /etc/containerd/config.toml.bak /etc/containerd/config.toml
-fi

kata-deploy/scripts/remove-kata-crio.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-echo "deleting kata artifacts"
-rm -rf /opt/kata/
-mv /etc/crio/crio.conf.bak /etc/crio/crio.conf