Merge pull request #7399 from fidencio/topic/add-kata-debug

packaging/tools: Add kata-debug and use it as part of our CI

Makefile

@@ -24,6 +24,10 @@ TOOLS += trace-forwarder
 
 STANDARD_TARGETS = build check clean install static-checks-build test vendor
 
+# Variables for the build-and-publish-kata-debug target
+KATA_DEBUG_REGISTRY ?= ""
+KATA_DEBUG_TAG ?= ""
+
 default: all
 
 include utils.mk
@@ -44,6 +48,9 @@ static-checks: static-checks-build
 docs-url-alive-check:
 	bash ci/docs-url-alive-check.sh
 
+build-and-publish-kata-debug:
+	bash tools/packaging/kata-debug/kata-debug-build-and-upload-payload.sh ${KATA_DEBUG_REGISTRY} ${KATA_DEBUG_TAG} 
+
 .PHONY: \
 	all \
 	kata-tarball \

README.md

@@ -134,6 +134,7 @@ The table below lists the remaining parts of the project:
 | [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
+| [kata-debug](tools/packaging/kata-debug/README.md) | infrastructure | Utility tool to gather Kata Containers debug information from Kubernetes clusters. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
 | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
 | [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. |

tests/integration/kubernetes/gha-run.sh

@@ -152,6 +152,10 @@ function cleanup() {
     kubectl delete ${cleanup_spec}
     kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
     kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
+
+   if [ "${platform}" = "aks" ]; then
+	   delete_cluster
+   fi
 }
 
 function delete_cluster() {
@@ -164,11 +168,20 @@ function delete_cluster() {
 function get_nodes_and_pods_info() {
     describe_pods="${1:-"no"}"
 
+    echo "::group::Get node information"
     kubectl get nodes -o wide --show-labels=true
+    echo "::endgroup::"
+    echo ""
+    echo "::group::Get all the pods running"
     kubectl get pods -A
+    echo "::endgroup::"
+    echo ""
     if [[ "${describe_pods}" == "yes" ]]; then
+	echo "::group::Describe all the pods"
     	kubectl describe pods -A
+	echo "::endgroup::"
     fi
+    kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest
 }
 
 function main() {
@@ -190,7 +203,7 @@ function main() {
         cleanup-sev) cleanup "sev" ;;
         cleanup-snp) cleanup "snp" ;;
         cleanup-tdx) cleanup "tdx" ;;
-        delete-cluster) delete_cluster ;;
+        delete-cluster) cleanup "aks" ;;
         *) >&2 echo "Invalid argument"; exit 2 ;;
     esac
 }

tools/packaging/kata-debug/Dockerfile

@@ -0,0 +1,16 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+FROM ubuntu:22.04
+
+COPY debug.sh /usr/bin/debug.sh
+
+RUN \
+apt-get update && \
+apt-get install -y --no-install-recommends tree && \
+apt-get clean && \
+rm -rf /var/lib/apt/lists/
+
+CMD ["/usr/bin/debug.sh"]

tools/packaging/kata-debug/README.md

@@ -0,0 +1,28 @@
+# kata-debug
+
+`kata-debug` is a tool that is used as part of the Kata Containers CI to gather
+information from the node, in order to help debugging issues with Kata
+Containers.
+
+As one can imagine, this can be expanded and used outside of the CI context,
+and any contribution back to the script is very much welcome.
+
+The resulting container is stored at the [Kata Containers `quay.io`
+space](https://quay.io/repository/kata-containers/kata-debug) and can
+be used as shown below:
+```sh
+kubectl debug $NODE_NAME -it --image=quay.io/kata-containers/kata-debug:latest
+```
+
+## Building and publishing
+The project can be built and publish by calling the following command from the
+Kata Containers top directory:
+```sh
+make build-and-publish-kata-debug
+```
+
+Users can specify the following environment variables to the build:
+* `KATA_DEBUG_REGISTRY` - The container registry to be used
+                          default: `quay.io/kata-containers/kata-debug`
+- `KATA_DEBUG_TAG`      - A tag to the be used for the image
+                          default: `$(git rev-parse HEAD)-$(uname -a)`

tools/packaging/kata-debug/debug.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+echo "Let's gather Kata Containers debug information"
+echo ""
+echo "::group::Check Kata Containers logs"
+chroot /host /bin/bash -c "sudo journalctl -xe -t kata | tee"
+echo "::endgroup::"
+echo ""
+echo "::group::Checking the loaded kernel modules"
+chroot /host /bin/bash -c "sudo lsmod"
+echo "::endgroup::"
+echo ""
+echo "::group::Check Kata Containers deployed binaries"
+tree /host/opt/kata /host/usr/local/bin
+echo "::endgroup::"
+echo ""
+echo "::group:: Check node's dmesg"
+chroot /host /bin/bash -c "sudo dmesg"
+echo "::endgroup::"

tools/packaging/kata-debug/kata-debug-build-and-upload-payload.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Copyright 2023 Intel
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+[ -z "${DEBUG}" ] || set -x
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+KATA_DEBUG_DIR="`dirname ${0}`"
+
+REGISTRY="${1:-"quay.io/kata-containers/kata-debug"}"
+TAG="${2:-}"
+
+arch=$(uname -m)
+[ "$arch" = "x86_64" ] && arch="amd64"
+IMAGE_TAG="${REGISTRY}:$(git rev-parse HEAD)-${arch}"
+
+pushd ${KATA_DEBUG_DIR}
+
+echo "Building the image"
+docker build --tag ${IMAGE_TAG} .
+
+echo "Pushing the image to the registry"
+docker push ${IMAGE_TAG}
+
+if [ -n "${TAG}" ]; then
+	ADDITIONAL_TAG="${REGISTRY}:${TAG}"
+
+	echo "Building the ${ADDITIONAL_TAG} image"
+
+	docker build --tag ${ADDITIONAL_TAG} .
+
+	echo "Pushing the image ${ADDITIONAL_TAG} to the registry"
+	docker push ${ADDITIONAL_TAG}
+fi
+
+popd