ci: Enable cri-containerd-tests for arm64

This change enables cri-containerd-test for arm64.

Signed-off-by: Seunguk Shin <seunguk.shin@arm.com>
Reviewed-by: Nick Connolly <nick.connolly@arm.com>

.github/workflows/basic-ci-amd64.yaml

@@ -17,49 +17,6 @@ permissions:
   contents: read
 
 jobs:
-  run-cri-containerd:
-    strategy:
-      # We can set this to true whenever we're 100% sure that
-      # the all the tests are not flaky, otherwise we'll fail
-      # all the tests due to a single flaky instance.
-      fail-fast: false
-      matrix:
-        containerd_version: ['lts', 'active']
-        vmm: ['clh', 'dragonball', 'qemu', 'stratovirt', 'cloud-hypervisor', 'qemu-runtime-rs']
-    runs-on: ubuntu-22.04
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.commit-hash }}
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Rebase atop of the latest target branch
-        run: |
-          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-        env:
-          TARGET_BRANCH: ${{ inputs.target-branch }}
-
-      - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v4
-        with:
-          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
-
-      - name: Run cri-containerd tests
-        timeout-minutes: 10
-        run: bash tests/integration/cri-containerd/gha-run.sh run
-
   run-containerd-sandboxapi:
     strategy:
       # We can set this to true whenever we're 100% sure that

.github/workflows/basic-ci-s390x.yaml

@@ -17,48 +17,6 @@ permissions:
   contents: read
 
 jobs:
-  run-cri-containerd:
-    strategy:
-      # We can set this to true whenever we're 100% sure that
-      # the all the tests are not flaky, otherwise we'll fail
-      # all the tests due to a single flaky instance
-      fail-fast: false
-      matrix:
-        containerd_version: ['active']
-        vmm: ['qemu', 'qemu-runtime-rs']
-    runs-on: s390x-large
-    env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
-      GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.commit-hash }}
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Rebase atop of the latest target branch
-        run: |
-          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
-        env:
-          TARGET_BRANCH: ${{ inputs.target-branch }}
-
-      - name: Install dependencies
-        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
-
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v4
-        with:
-          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
-          path: kata-artifacts
-
-      - name: Install kata
-        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
-
-      - name: Run cri-containerd tests
-        run: bash tests/integration/cri-containerd/gha-run.sh run
-
   run-containerd-sandboxapi:
     strategy:
       # We can set this to true whenever we're 100% sure that

.github/workflows/ci.yaml

@@ -415,11 +415,86 @@ jobs:
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
 
-  run-cri-containerd-tests-ppc64le:
+  run-cri-containerd-amd64:
     if: ${{ inputs.skip-test != 'yes' }}
-    needs: build-kata-static-tarball-ppc64le
+    needs: build-kata-static-tarball-amd64
-    uses: ./.github/workflows/run-cri-containerd-tests-ppc64le.yaml
+    strategy:
+      matrix:
+        params: [
+          { containerd_version: lts,    vmm: clh              },
+          { containerd_version: lts,    vmm: dragonball       },
+          { containerd_version: lts,    vmm: qemu             },
+          { containerd_version: lts,    vmm: stratovirt       },
+          { containerd_version: lts,    vmm: cloud-hypervisor },
+          { containerd_version: lts,    vmm: qemu-runtime-rs  },
+          { containerd_version: active, vmm: clh              },
+          { containerd_version: active, vmm: dragonball       },
+          { containerd_version: active, vmm: qemu             },
+          { containerd_version: active, vmm: stratovirt       },
+          { containerd_version: active, vmm: cloud-hypervisor },
+          { containerd_version: active, vmm: qemu-runtime-rs  },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
     with:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
+      runner: ubuntu-22.04
+      arch: amd64
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}
+
+  run-cri-containerd-s390x:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-s390x
+    strategy:
+      matrix:
+        params: [
+          { containerd_version: active, vmm: qemu            },
+          { containerd_version: active, vmm: qemu-runtime-rs },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: s390x-large
+      arch: s390x
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}
+
+  run-cri-containerd-tests-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-ppc64le
+    strategy:
+      matrix:
+        params: [
+          { containerd_version: active, vmm: qemu },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: ppc64le
+      arch: ppc64le
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}
+
+  run-cri-containerd-tests-arm64:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-arm64
+    strategy:
+      matrix:
+        params: [
+          { containerd_version: active, vmm: qemu },
+         ]
+    uses: ./.github/workflows/run-cri-containerd-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+      runner: arm64-non-k8s
+      arch: arm64
+      containerd_version: ${{ matrix.params.containerd_version }}
+      vmm: ${{ matrix.params.vmm }}

.github/workflows/run-cri-containerd-tests.yaml

@@ -1,4 +1,4 @@
-name: CI | Run cri-containerd tests on ppc64le
+name: CI | Run cri-containerd tests
 
 permissions:
   contents: read
@@ -16,22 +16,33 @@ on:
         required: false
         type: string
         default: ""
+      runner:
+        description: The runner to execute the workflow on.
+        required: true
+        type: string
+      arch:
+        description: The arch of the tarball.
+        required: true
+        type: string
+      containerd_version:
+        description: The version of containerd for testing.
+        required: true
+        type: string
+      vmm:
+        description: The kata hypervisor for testing.
+        required: true
+        type: string
 
 jobs:
   run-cri-containerd:
+    name: run-cri-containerd-${{ inputs.arch }} (${{ inputs.containerd_version }}, ${{ inputs.vmm }})
     strategy:
-      # We can set this to true whenever we're 100% sure that
-      # the all the tests are not flaky, otherwise we'll fail
-      # all the tests due to a single flaky instance
       fail-fast: false
-      matrix:
+    runs-on: ${{ inputs.runner }}
-        containerd_version: ['active']
-        vmm: ['qemu']
-    runs-on: ppc64le
     env:
-      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      CONTAINERD_VERSION: ${{ inputs.containerd_version }}
       GOPATH: ${{ github.workspace }}
-      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KATA_HYPERVISOR: ${{ inputs.vmm }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -49,14 +60,15 @@ jobs:
         timeout-minutes: 15
         run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
 
-      - name: get-kata-tarball
+      - name: get-kata-tarball for ${{ inputs.arch }}
         uses: actions/download-artifact@v4
         with:
-          name: kata-static-tarball-ppc64le${{ inputs.tarball-suffix }}
+          name: kata-static-tarball-${{ inputs.arch }}${{ inputs.tarball-suffix }}
           path: kata-artifacts
 
       - name: Install kata
         run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
 
-      - name: Run cri-containerd tests
+      - name: Run cri-containerd tests for ${{ inputs.arch }}
+        timeout-minutes: 10
         run: bash tests/integration/cri-containerd/gha-run.sh run

tests/integration/cri-containerd/gha-run.sh

@@ -16,9 +16,17 @@ source "${cri_containerd_dir}/../../common.bash"
 function install_dependencies() {
 	info "Installing the dependencies needed for running the cri-containerd tests"
 
+	# Remove go if it's installed as it conflicts with another version of go
+	sudo apt-get remove -y golang-* || true
+	sudo rm -rf /usr/local/go
+
 	# Remove Docker if it's installed as it conflicts with podman-docker
 	sudo apt-get remove -y docker-ce-cli || true
 
+	# Remove containerd if it's installed as it conflicts with another version of containerd
+	sudo apt-get remove -y containerd containerd.io || true
+	sudo rm -rf /etc/systemd/system/containerd.service
+
 	# Dependency list of projects that we can rely on the system packages
 	# - build-essential
 	#   - Theoretically we only need `make`, but doesn't hurt to install

tests/integration/cri-containerd/integration-tests.sh

@@ -32,7 +32,7 @@ SANDBOXER=${SANDBOXER:-"podsandbox"}
 
 containerd_runtime_type="io.containerd.kata-${KATA_HYPERVISOR}.v2"
 
-containerd_shim_path="$(command -v containerd-shim)"
+containerd_shim_path="$(command -v containerd-shim || true)"
 
 #containerd config file
 readonly tmp_dir=$(mktemp -t -d test-cri-containerd.XXXX)
@@ -101,27 +101,36 @@ function create_containerd_config() {
 		runtime_config_path=""
 		runtime_binary_path=""
 	fi
+
+	# check containerd config version
+	if containerd config default | grep -q "version = 3\>"; then
+		pluginid=\"io.containerd.cri.v1.runtime\"
+	else
+		pluginid="cri"
+	fi
 	info "Kata Config Path ${runtime_config_path}, Runtime Binary Name ${runtime_binary_path}"
 
 cat << EOF | sudo tee "${CONTAINERD_CONFIG_FILE}"
 [debug]
   level = "debug"
 [plugins]
-  [plugins.cri]
+  [plugins.${pluginid}]
-    [plugins.cri.containerd]
+    [plugins.${pluginid}.containerd]
         default_runtime_name = "$runtime"
-      [plugins.cri.containerd.runtimes.${runtime}]
+      [plugins.${pluginid}.containerd.runtimes.${runtime}]
         runtime_type = "${runtime_type}"
         sandboxer = "${SANDBOXER}"
         $( [ $kata_annotations -eq 1 ] && \
         echo 'pod_annotations = ["io.katacontainers.*"]' && \
         echo '        container_annotations = ["io.katacontainers.*"]'
         )
-        [plugins.cri.containerd.runtimes.${runtime}.options]
+        [plugins.${pluginid}.containerd.runtimes.${runtime}.options]
           ConfigPath = "${runtime_config_path}"
           BinaryName = "${runtime_binary_path}"
-[plugins.linux]
+$( [[ -n "$containerd_shim_path" ]] && \
-       shim = "${containerd_shim_path}"
+echo "[plugins.linux]" && \
+echo "  shim = \"${containerd_shim_path}\""
+)
 EOF
 }
 
@@ -271,7 +280,7 @@ function PrepareContainerMemoryUpdate() {
 	test_virtio_mem=$1
 
 	if [ $test_virtio_mem -eq 1 ]; then
-		if [[ "$ARCH" != "x86_64" ]]; then
+		if [[ "$ARCH" != "x86_64" ]] && [[ "$ARCH" != "aarch64" ]]; then
 			return
 		fi
 		info "Test container memory update with virtio-mem"
@@ -601,6 +610,17 @@ function TestDeviceCgroup() {
 
 function main() {
 
+	info "Clean up containers and pods"
+	restart_containerd_service
+	containers=( $(sudo crictl ps --all -o json | jq -r '.containers[].id') )
+	for c in "${containers[@]}"; do
+		sudo crictl rm -f $c
+	done
+	pods=( $(sudo crictl pods -o json | jq -r '.items[].id') )
+	for p in "${pods[@]}"; do
+		sudo crictl rmp -f $p
+	done
+
 	info "Stop crio service"
 	systemctl is-active --quiet crio && sudo systemctl stop crio