Merge pull request #7411 from fidencio/topic/kata-deploy-create-runtime-classes

kata-deploy: Allow runtimeclasses to be created by the daemonset

.github/workflows/kata-runtime-classes-sync.yaml

@@ -0,0 +1,36 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-deploy-runtime-classes-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Ensure the split out runtime classes match the all-in-one file
+      run: |
+        pushd tools/packaging/kata-deploy/runtimeclasses/
+        echo "::group::Combine runtime classes"
+        for runtimeClass in `find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort`; do
+            echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml"
+            cat ${runtimeClass} >> resultingRuntimeClasses.yaml;
+        done
+        echo "::endgroup::"
+        echo "::group::Displaying the content of resultingRuntimeClasses.yaml"
+        cat resultingRuntimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        echo "::group::Displaying the content of kata-runtimeClasses.yaml"
+        cat kata-runtimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        diff resultingRuntimeClasses.yaml kata-runtimeClasses.yaml

tests/integration/kubernetes/gha-run.sh

@@ -71,14 +71,19 @@ function deploy_kata() {
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
     # Enable debug for Kata Containers
-    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' "\"yes\""
+    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[1].value' --tag '!!str' "true"
+    # Let the `kata-deploy` script take care of the runtime class creation / removal
+    yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
 
     if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
         yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS"
         yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}"
     fi
+
+    echo "::group::Final kata-deploy.yaml that is used in the test"
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
     cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
+    echo "::endgroup::"
 
     kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
     if [ "${platform}" = "tdx" ]; then
@@ -87,10 +92,6 @@ function deploy_kata() {
         kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
     fi
     kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-    kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
-
-    echo "Gather information about the nodes and pods after having kata-deploy ready"
-    get_nodes_and_pods_info
 
     # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
     # which may cause issues like not having the node properly labeled or the artefacts
@@ -100,6 +101,14 @@ function deploy_kata() {
     else
         sleep 60s
     fi
+
+    echo "::group::kata-deploy logs"
+    kubectl -n kube-system logs -l name=kata-deploy
+    echo "::endgroup::"
+
+    echo "::group::Runtime classes"
+    kubectl get runtimeclass
+    echo "::endgroup::"
 }
 
 function run_tests() {
@@ -110,9 +119,6 @@ function run_tests() {
     kubectl apply -f ${kubernetes_dir}/runtimeclass_workloads/tests-namespace.yaml
     kubectl config set-context --current --namespace=kata-containers-k8s-tests
 
-    echo "Gather information about the nodes and pods just before starting the tests"
-    get_nodes_and_pods_info
-
     pushd "${kubernetes_dir}"
     bash setup.sh
     bash run_kubernetes_tests.sh
@@ -121,9 +127,10 @@ function run_tests() {
 
 function cleanup() {
     platform="${1}"
+    ensure_yq
 
     echo "Gather information about the nodes and pods before cleaning up the node"
-    get_nodes_and_pods_info "yes"
+    get_nodes_and_pods_info
 
     if [ "${platform}" = "aks" ]; then
         delete_cluster
@@ -145,6 +152,9 @@ function cleanup() {
     kubectl delete ${deploy_spec}
     kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
 
+    # Let the `kata-deploy` script take care of the runtime class creation / removal
+    yq write -i "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" 'spec.template.spec.containers[0].env[4].value' --tag '!!str' "true"
+
     sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
     cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
     cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
@@ -153,7 +163,6 @@ function cleanup() {
 
     kubectl delete ${cleanup_spec}
     kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
-    kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
 }
 
 function delete_cluster() {
@@ -164,21 +173,6 @@ function delete_cluster() {
 }
 
 function get_nodes_and_pods_info() {
-    describe_pods="${1:-"no"}"
-
-    echo "::group::Get node information"
-    kubectl get nodes -o wide --show-labels=true
-    echo "::endgroup::"
-    echo ""
-    echo "::group::Get all the pods running"
-    kubectl get pods -A
-    echo "::endgroup::"
-    echo ""
-    if [[ "${describe_pods}" == "yes" ]]; then
-	echo "::group::Describe all the pods"
-    	kubectl describe pods -A
-	echo "::endgroup::"
-    fi
     kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest
     kubectl get pods -o name | grep node-debugger | xargs kubectl delete
 }

tools/packaging/kata-deploy/Dockerfile

@@ -28,3 +28,4 @@ tar xvf ${WORKDIR}/${KATA_ARTIFACTS} -C ${DESTINATION} && \
 rm -f ${WORKDIR}/${KATA_ARTIFACTS}
 
 COPY scripts ${DESTINATION}/scripts
+COPY runtimeclasses ${DESTINATION}/runtimeclasses

tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml

@@ -13,7 +13,7 @@ spec:
         labels:
           name: kubelet-kata-cleanup
     spec:
-      serviceAccountName: kata-label-node
+      serviceAccountName: kata-deploy-sa
       nodeSelector:
           katacontainers.io/kata-runtime: cleanup
       containers:
@@ -27,11 +27,15 @@ spec:
             fieldRef:
               fieldPath: spec.nodeName
         - name: DEBUG
-          value: "no"
+          value: "false"
         - name: SHIMS
           value: "clh dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu"
         - name: DEFAULT_SHIM
           value: "qemu"
+        - name: CREATE_RUNTIMECLASSES
+          value: "false"
+        - name: CREATE_DEFAULT_RUNTIMECLASS
+          value: "false"
         securityContext:
           privileged: true
         volumeMounts:

tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml

@@ -13,7 +13,7 @@ spec:
         labels:
           name: kata-deploy
     spec:
-      serviceAccountName: kata-label-node
+      serviceAccountName: kata-deploy-sa
       containers:
       - name: kube-kata
         image: quay.io/kata-containers/kata-deploy:latest
@@ -29,11 +29,15 @@ spec:
             fieldRef:
               fieldPath: spec.nodeName
         - name: DEBUG
-          value: "no"
+          value: "false"
         - name: SHIMS
           value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx"
         - name: DEFAULT_SHIM
           value: "qemu"
+        - name: CREATE_RUNTIMECLASSES
+          value: "false"
+        - name: CREATE_DEFAULT_RUNTIMECLASS
+          value: "false"
         securityContext:
           privileged: true
         volumeMounts:

tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml

@@ -2,28 +2,30 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: kata-label-node
+  name: kata-deploy-sa
   namespace: kube-system
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: node-labeler
+  name: kata-deploy-role
 rules:
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "patch"]
+- apiGroups: ["node.k8s.io"]
+  resources: ["runtimeclasses"]
+  verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: kata-label-node-rb
+  name: kata-deploy-rb
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: node-labeler
+  name: kata-deploy-role
 subjects:
 - kind: ServiceAccount
-  name: kata-label-node
+  name: kata-deploy-sa
   namespace: kube-system
-

tools/packaging/kata-deploy/runtimeclasses/kata-clh.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-clh
+handler: kata-clh
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-dragonball.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-dragonball
+handler: kata-dragonball
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-fc.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-fc
+handler: kata-fc
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-qemu-nvidia-gpu.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-nvidia-gpu
+handler: kata-qemu-nvidia-gpu
+overhead:
+    podFixed:
+        memory: "160Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-qemu-sev.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-sev
+handler: kata-qemu-sev
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+    nodeSelector:
+        katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-qemu-snp.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-snp
+handler: kata-qemu-snp
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-qemu-tdx.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-tdx
+handler: kata-qemu-tdx
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-qemu.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu
+handler: kata-qemu
+overhead:
+    podFixed:
+        memory: "160Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

@@ -1,58 +1,6 @@
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
-metadata:
-    name: kata-qemu
-handler: kata-qemu
-overhead:
-    podFixed:
-        memory: "160Mi"
-        cpu: "250m"
-scheduling:
-  nodeSelector:
-    katacontainers.io/kata-runtime: "true"
----
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1
-metadata:
-    name: kata-qemu-sev
-handler: kata-qemu-sev
-overhead:
-    podFixed:
-        memory: "2048Mi"
-        cpu: "1.0"
-scheduling:
-    nodeSelector:
-        katacontainers.io/kata-runtime: "true"
----
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1
-metadata:
-    name: kata-qemu-tdx
-handler: kata-qemu-tdx
-overhead:
-    podFixed:
-        memory: "2048Mi"
-        cpu: "1.0"
-scheduling:
-  nodeSelector:
-    katacontainers.io/kata-runtime: "true"
----
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1
-metadata:
-    name: kata-qemu-snp
-handler: kata-qemu-snp
-overhead:
-    podFixed:
-        memory: "2048Mi"
-        cpu: "1.0"
-scheduling:
-  nodeSelector:
-    katacontainers.io/kata-runtime: "true"
----
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1
 metadata:
     name: kata-clh
 handler: kata-clh
@@ -66,19 +14,6 @@ scheduling:
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
-metadata:
-    name: kata-fc
-handler: kata-fc
-overhead:
-    podFixed:
-        memory: "130Mi"
-        cpu: "250m"
-scheduling:
-  nodeSelector:
-    katacontainers.io/kata-runtime: "true"
----
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1
 metadata:
     name: kata-dragonball
 handler: kata-dragonball
@@ -92,6 +27,19 @@ scheduling:
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-fc
+handler: kata-fc
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
 metadata:
     name: kata-qemu-nvidia-gpu
 handler: kata-qemu-nvidia-gpu
@@ -102,3 +50,55 @@ overhead:
 scheduling:
   nodeSelector:
     katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-sev
+handler: kata-qemu-sev
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+    nodeSelector:
+        katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-snp
+handler: kata-qemu-snp
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu-tdx
+handler: kata-qemu-tdx
+overhead:
+    podFixed:
+        memory: "2048Mi"
+        cpu: "1.0"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-qemu
+handler: kata-qemu
+overhead:
+    podFixed:
+        memory: "160Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -29,6 +29,41 @@ function print_usage() {
 	echo "Usage: $0 [install/cleanup/reset]"
 }
 
+function create_runtimeclasses() {
+	echo "Creating the runtime classes"
+
+	for shim in "${shims[@]}"; do
+		echo "Creating the kata-${shim} runtime class"
+		kubectl apply -f /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml
+	done
+
+	if [[ "${CREATE_DEFAULT_RUNTIMECLASS}" == "true" ]]; then
+		echo "Creating the kata runtime class for the default shim (an alias for kata-${default_shim})"
+		cp /opt/kata-artifacts/runtimeclasses/kata-${default_shim}.yaml /tmp/kata.yaml
+		sed -i -e 's/kata-'${default_shim}'/kata/g' /tmp/kata.yaml
+		kubectl apply -f /tmp/kata.yaml
+		rm -f /tmp/kata.yaml
+	fi
+}
+
+function delete_runtimeclasses() {
+	echo "Deleting the runtime classes"
+
+	for shim in "${shims[@]}"; do
+		echo "Deleting the kata-${shim} runtime class"
+		kubectl delete -f /opt/kata-artifacts/runtimeclasses/kata-${shim}.yaml
+	done
+
+
+	if [[ "${CREATE_DEFAULT_RUNTIMECLASS}" == "true" ]]; then
+		echo "Deleting the kata runtime class for the default shim (an alias for kata-${default_shim})"
+		cp /opt/kata-artifacts/runtimeclasses/kata-${default_shim}.yaml /tmp/kata.yaml
+		sed -i -e 's/kata-'${default_shim}'/kata/g' /tmp/kata.yaml
+		kubectl delete -f /tmp/kata.yaml
+		rm -f /tmp/kata.yaml
+	fi
+}
+
 function get_container_runtime() {
 
 	local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}')
@@ -58,7 +93,7 @@ function install_artifacts() {
 		chmod +x /opt/kata/runtime-rs/bin/*
 
 	# Allow enabling debug for Kata Containers
-	if [[ "${DEBUG:-"no"}" == "yes" ]]; then
+	if [[ "${DEBUG}" == "true" ]]; then
 		config_path="/opt/kata/share/defaults/kata-containers/"
 		for shim in "${shims[@]}"; do
 			sed -i -e 's/^#\(enable_debug\).*=.*$/\1 = true/g' "${config_path}/configuration-${shim}.toml"
@@ -75,6 +110,10 @@ function install_artifacts() {
 		sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}"
 		sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}"
 	fi
+
+	if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then
+		create_runtimeclasses
+	fi
 }
 
 function wait_till_node_is_ready() {
@@ -174,6 +213,10 @@ function cleanup_different_shims_base() {
 
 	rm "${default_shim_file}" || true
 	restore_shim "${default_shim_file}"
+
+	if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then
+		delete_runtimeclasses
+	fi
 }
 
 function configure_crio_runtime() {
@@ -216,7 +259,7 @@ function configure_crio() {
 	done
 
 
-	if [ "${DEBUG:-"no"}" == "yes" ]; then
+	if [ "${DEBUG}" == "true" ]; then
 		cat <<EOF | tee -a $crio_drop_in_conf_file_debug
 [crio]
 log_level = "debug"
@@ -261,7 +304,7 @@ EOF
 EOF
 	fi
 
-	if [ "${DEBUG:-"no"}" == "yes" ]; then
+	if [ "${DEBUG}" == "true" ]; then
 		if grep -q "\[debug\]" $containerd_conf_file; then
 			sed -i 's/level.*/level = \"debug\"/' $containerd_conf_file
 		else
@@ -314,7 +357,7 @@ function cleanup_cri_runtime() {
 
 function cleanup_crio() {
 	rm $crio_drop_in_conf_file
-	if [[ "${DEBUG:-"no"}" == "yes" ]]; then
+	if [[ "${DEBUG}" == "true" ]]; then
 		rm $crio_drop_in_conf_file_debug
 	fi
 }
@@ -338,6 +381,14 @@ function reset_runtime() {
 }
 
 function main() {
+	echo "Environment variables passed to this script"
+	echo "* NODE_NAME: ${NODE_NAME}"
+	echo "* DEBUG: ${DEBUG}"
+	echo "* SHIMS: ${SHIMS}"
+	echo "* DEFAULT_SHIM: ${DEFAULT_SHIM}"
+	echo "* CREATE_RUNTIMECLASSES: ${CREATE_RUNTIMECLASSES}"
+	echo "* CREATE_DEFAULT_RUNTIMECLASS: ${CREATE_DEFAULT_RUNTIMECLASS}"
+
 	# script requires that user is root
 	euid=$(id -u)
 	if [[ $euid -ne 0 ]]; then