Merge pull request #7896 from fidencio/topic/ground-work-for-testing-all-k8s-flavours-we-support

ci: kata-deploy: Enable all k8s flavours that we support

.github/workflows/ci.yaml

@@ -101,6 +101,18 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
+  run-kata-deploy-tests-on-garm:
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
   run-kata-deploy-tests-on-tdx:
     needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
     uses: ./.github/workflows/run-kata-deploy-tests-on-tdx.yaml

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -44,6 +44,7 @@ jobs:
       GH_PR_NUMBER: ${{ inputs.pr-number }}
       KATA_HOST_OS: ${{ matrix.host_os }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
       USING_NFD: "false"
     steps:
       - uses: actions/checkout@v3

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -0,0 +1,65 @@
+name: CI | Run kata-deploy tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+        k8s:
+          - k0s
+          - k3s
+          - rke2
+    runs-on: garm-ubuntu-2004-small
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests

.github/workflows/run-kata-deploy-tests-on-tdx.yaml

@@ -36,6 +36,7 @@ jobs:
       DOCKER_TAG: ${{ inputs.tag }}
       PR_NUMBER: ${{ inputs.pr-number }}
       KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
       USING_NFD: "true"
     steps:
       - uses: actions/checkout@v3

tests/functional/kata-deploy/gha-run.sh

@@ -56,6 +56,7 @@ function main() {
         install-azure-cli) install_azure_cli ;;
         login-azure) login_azure ;;
         create-cluster) create_cluster "kata-deploy" ;;
+        deploy-k8s) deploy_k8s ;;
         install-bats) install_bats ;;
         install-kubectl) install_kubectl ;;
         get-cluster-credentials) get_cluster_credentials "kata-deploy" ;;

tests/functional/kata-deploy/kata-deploy.bats

@@ -48,8 +48,12 @@ setup() {
 	echo "::endgroup::"
 	
 	kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
-	if [ "${platform}" = "tdx" ]; then
+	if [ "${KUBERNETES}" = "k0s" ]; then
+		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
+	elif [ "${KUBERNETES}" = "k3s" ]; then
 		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
+	elif [ "${KUBERNETES}" = "rke2" ]; then
+		kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
 	else
 		kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	fi
@@ -74,12 +78,18 @@ setup() {
 teardown() {
 	kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
 
-	if [ "${platform}" = "tdx" ]; then
+	if [ "${KUBERNETES}" = "k0s" ]; then
-		deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s""
+		deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
-		cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s""
+		cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
+	elif [ "${KUBERNETES}" = "k3s" ]; then
+		deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
+		cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
+	elif [ "${KUBERNETES}" = "rke2" ]; then
+		deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
+		cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
 	else
-		deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
+		deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
-		cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
+		cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
 	fi
 
 	kubectl delete ${deploy_spec}

tests/gha-run-k8s-common.sh

@@ -111,3 +111,85 @@ function get_nodes_and_pods_info() {
     kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest || true
     kubectl get pods -o name | grep node-debugger | xargs kubectl delete || true
 }
+
+function deploy_k0s() {
+	curl -sSLf https://get.k0s.sh | sudo sh
+
+	sudo k0s install controller --single
+
+	sudo k0s start
+
+	# This is an arbitrary value that came up from local tests
+	sleep 120s
+
+	# Download the kubectl binary into /usr/bin so we can avoid depending
+	# on `k0s kubectl` command
+	ARCH=$(uname -m)
+	if [ "${ARCH}" = "x86_64" ]; then
+		ARCH=amd64
+	fi
+	kubectl_version=$(sudo k0s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //')
+	sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl
+	sudo chmod +x /usr/bin/kubectl
+
+	mkdir -p ~/.kube
+	sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config
+	sudo chown ${USER}:${USER} ~/.kube/config
+}
+
+function deploy_k3s() {
+	curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
+
+	# This is an arbitrary value that came up from local tests
+	sleep 120s
+
+	# Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl
+	#
+	# We need to do this to avoid hitting issues like:
+	# ```sh
+	# error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied
+	# ```
+	# Which happens basically because k3s links `/usr/local/bin/kubectl`
+	# to `/usr/local/bin/k3s`, and that does extra stuff that vanilla
+	# `kubectl` doesn't do.
+	ARCH=$(uname -m)
+	if [ "${ARCH}" = "x86_64" ]; then
+		ARCH=amd64
+	fi
+	kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//')
+	sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl
+	sudo chmod +x /usr/bin/kubectl
+	sudo rm -rf /usr/local/bin/kubectl
+
+	mkdir -p ~/.kube
+	cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
+}
+
+function deploy_rke2() {
+	curl -sfL https://get.rke2.io | sudo sh -
+
+	systemctl enable --now rke2-server.service
+
+	# This is an arbitrary value that came up from local tests
+	sleep 120s
+
+	# Link the kubectl binary into /usr/bin
+	sudo ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl
+
+	mkdir -p ~/.kube
+	sudo cp /etc/rancher/rke2/rke2.yaml ~/.kube/config
+	sudo chown ${USER}:${USER} ~/.kube/config
+}
+
+function deploy_k8s() {
+	echo "::group::Deploying ${KUBERNETES}"
+
+	case ${KUBERNETES} in
+		k0s) deploy_k0s ;;
+		k3s) deploy_k3s ;;
+		rke2) deploy_rke2 ;;
+		*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
+	esac
+
+	echo "::endgroup::"
+}

tests/integration/kubernetes/gha-run.sh

@@ -143,45 +143,6 @@ function deploy_kata() {
     echo "::endgroup::"
 }
 
-function deploy_k3s() {
-	curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
-
-	# This is an arbitrary value that came up from local tests
-	sleep 120s
-
-	# Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl
-	#
-	# We need to do this to avoid hitting issues like:
-	# ```sh
-	# error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied
-	# ```
-	# Which happens basically because k3s links `/usr/local/bin/kubectl`
-	# to `/usr/local/bin/k3s`, and that does extra stuff that vanilla
-	# `kubectl` doesn't do.
-	ARCH=$(uname -m)
-	if [ "${ARCH}" = "x86_64" ]; then
-		ARCH=amd64
-	fi
-	kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//')
-	sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl
-	sudo chmod +x /usr/bin/kubectl
-	sudo rm -rf /usr/local/bin/kubectl
-
-	mkdir -p ~/.kube
-	cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
-}
-
-function deploy_k8s() {
-	echo "::group::Deploying ${KUBERNETES}"
-
-	case ${KUBERNETES} in
-		k3s) deploy_k3s ;;
-		*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
-	esac
-
-	echo "::endgroup::"
-}
-
 function run_tests() {
     # Delete any spurious tests namespace that was left behind
     kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true