Merge pull request #10890 from zvonkok/arm64-fix-release

release: Remove artifacts for release

.github/actionlint.yaml

@@ -0,0 +1,24 @@
+# Copyright (c) 2024 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Configuration file with rules for the actionlint tool.
+#
+self-hosted-runner:
+  # Labels of self-hosted runner that linter should ignore
+  labels:
+    - ubuntu-22.04-arm
+    - garm-ubuntu-2004
+    - garm-ubuntu-2004-smaller
+    - garm-ubuntu-2204
+    - garm-ubuntu-2304
+    - garm-ubuntu-2304-smaller
+    - garm-ubuntu-2204-smaller
+    - k8s-ppc64le
+    - metrics
+    - ppc64le
+    - sev
+    - sev-snp
+    - s390x
+    - s390x-large
+    - tdx

.github/cargo-deny-composite-action/cargo-deny-generator.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2022 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+script_dir=$(dirname "$(readlink -f "$0")")
+parent_dir=$(realpath "${script_dir}/../..")
+cidir="${parent_dir}/ci"
+source "${cidir}/../tests/common.bash"
+
+cargo_deny_file="${script_dir}/action.yaml"
+
+cat cargo-deny-skeleton.yaml.in > "${cargo_deny_file}"
+
+changed_files_status=$(run_get_pr_changed_file_details)
+changed_files_status=$(echo "$changed_files_status" | grep "Cargo\.toml$" || true)
+changed_files=$(echo "$changed_files_status" | awk '{print $NF}' || true)
+
+if [ -z "$changed_files" ]; then
+  cat >> "${cargo_deny_file}" << EOF
+    - run: echo "No Cargo.toml files to check"
+      shell: bash
+EOF
+fi
+
+for path in $changed_files
+do
+    cat >> "${cargo_deny_file}" << EOF
+
+    - name: ${path}
+      continue-on-error: true
+      shell: bash
+      run: |
+        pushd $(dirname ${path})
+        cargo deny check
+        popd
+EOF
+done

.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2022 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+name: 'Cargo Crates Check'
+description: 'Checks every Cargo.toml file using cargo-deny'
+
+env:
+  CARGO_TERM_COLOR: always
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: nightly 
+        override: true
+
+    - name: Cache
+      uses: Swatinem/rust-cache@v2
+
+    - name: Install Cargo deny
+      shell: bash
+      run: |
+        which cargo
+        cargo install --locked cargo-deny || true

.github/workflows/PR-wip-checks.yaml

@@ -9,9 +9,13 @@ on:
       - labeled
       - unlabeled
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pr_wip_check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: WIP Check
     steps:
     - name: WIP Check

.github/workflows/actionlint.yaml

@@ -0,0 +1,33 @@
+name: Lint GHA workflows
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths:
+      - '.github/workflows/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-actionlint:
+    env:
+      GH_TOKEN: ${{ github.token }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install actionlint gh extension
+        run: gh extension install https://github.com/cschleiden/gh-actionlint
+
+      - name: Run actionlint
+        run:  gh actionlint

.github/workflows/add-issues-to-project.yaml

@@ -11,9 +11,13 @@ on:
       - opened
       - reopened
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   add-new-issues-to-backlog:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Install hub
         run: |
@@ -29,13 +33,13 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install hub-util.sh /usr/local/bin
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Add issue to issue backlog
         env:

.github/workflows/add-pr-sizing-label.yaml

@@ -12,18 +12,31 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   add-pr-size-label:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Install PR sizing label script
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install pr-add-size-label.sh /usr/local/bin
           popd &>/dev/null
@@ -33,6 +46,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
         run: |
           pr=${{ github.event.number }}
+          # Removing man-db, workflow kept failing, fixes: #4480
+          sudo apt -y remove --purge man-db
           sudo apt -y install diffstat patchutils
 
           pr-add-size-label.sh -p "$pr"

.github/workflows/basic-ci-amd64.yaml

@@ -0,0 +1,428 @@
+name: CI | Basic amd64 tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'dragonball', 'qemu', 'stratovirt', 'cloud-hypervisor', 'qemu-runtime-rs']
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        timeout-minutes: 10
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+  run-containerd-sandboxapi:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['latest']
+        vmm: ['dragonball', 'cloud-hypervisor', 'qemu-runtime-rs']
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      #the latest containerd from 2.0 need to set the CGROUP_DRIVER for e2e testing
+      CGROUP_DRIVER: ""
+      SANDBOXER: "shim"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run containerd-sandboxapi tests
+        timeout-minutes: 10
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+  run-containerd-stability:
+    strategy:
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'cloud-hypervisor', 'dragonball', 'qemu', 'stratovirt']
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      SANDBOXER: "podsandbox"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/stability/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/stability/gha-run.sh install-kata kata-artifacts
+
+      - name: Run containerd-stability tests
+        timeout-minutes: 15
+        run: bash tests/stability/gha-run.sh run
+
+  run-nydus:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        containerd_version: ['lts', 'active']
+        vmm: ['clh', 'qemu', 'dragonball', 'stratovirt']
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nydus/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nydus tests
+        timeout-minutes: 10
+        run: bash tests/integration/nydus/gha-run.sh run
+
+  run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run runk tests
+        timeout-minutes: 10
+        run: bash tests/integration/runk/gha-run.sh run
+
+  run-tracing:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh # cloud-hypervisor
+          - qemu
+    # TODO: enable me when https://github.com/kata-containers/kata-containers/issues/9763 is fixed
+    # TODO: Transition to free runner (see #9940).
+    if: false
+    runs-on: garm-ubuntu-2204-smaller
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/tracing/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/tracing/gha-run.sh install-kata kata-artifacts
+
+      - name: Run tracing tests
+        timeout-minutes: 15
+        run: bash tests/functional/tracing/gha-run.sh run
+
+  run-vfio:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+    # TODO: enable with clh when https://github.com/kata-containers/kata-containers/issues/9764 is fixed
+    # TODO: enable with qemu when https://github.com/kata-containers/kata-containers/issues/9851 is fixed
+    # TODO: Transition to free runner (see #9940).
+    if: false
+    runs-on: garm-ubuntu-2304
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/vfio/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Run vfio tests
+        timeout-minutes: 15
+        run: bash tests/functional/vfio/gha-run.sh run
+
+  run-docker-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+          - dragonball
+          - cloud-hypervisor
+    runs-on: ubuntu-22.04
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/docker/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/docker/gha-run.sh install-kata kata-artifacts
+
+      - name: Run docker smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/docker/gha-run.sh run
+
+  run-nerdctl-tests:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # all the tests are not flaky, otherwise we'll fail them
+      # all due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+          - cloud-hypervisor
+    runs-on: ubuntu-22.04
+    env:
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/nerdctl/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/nerdctl/gha-run.sh install-kata kata-artifacts
+
+      - name: Run nerdctl smoke test
+        timeout-minutes: 5
+        run: bash tests/integration/nerdctl/gha-run.sh run
+
+      - name: Collect artifacts ${{ matrix.vmm }}
+        if: always()
+        run: bash tests/integration/nerdctl/gha-run.sh collect-artifacts
+        continue-on-error: true
+
+      - name: Archive artifacts ${{ matrix.vmm }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nerdctl-tests-garm-${{ matrix.vmm }}
+          path: /tmp/artifacts
+          retention-days: 1
+
+  run-kata-agent-apis:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/kata-agent-apis/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/kata-agent-apis/gha-run.sh install-kata kata-artifacts
+
+      - name: Run kata agent api tests with agent-ctl
+        run: bash tests/functional/kata-agent-apis/gha-run.sh run

.github/workflows/build-checks.yaml

@@ -0,0 +1,108 @@
+on:
+  workflow_call:
+    inputs:
+      instance:
+        required: true
+        type: string
+
+name: Build checks
+jobs:
+  check:
+    runs-on: ${{ inputs.instance }}
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - agent
+          - dragonball
+          - runtime
+          - runtime-rs
+          - agent-ctl
+          - kata-ctl
+          - trace-forwarder
+          - genpolicy
+        command:
+          - "make vendor"
+          - "make check"
+          - "make test"
+          - "sudo -E PATH=\"$PATH\" make test"
+        include:
+          - component: agent
+            component-path: src/agent
+          - component: dragonball
+            component-path: src/dragonball
+          - component: runtime
+            component-path: src/runtime
+          - component: runtime-rs
+            component-path: src/runtime-rs
+          - component: agent-ctl
+            component-path: src/tools/agent-ctl
+          - component: kata-ctl
+            component-path: src/tools/kata-ctl
+          - component: trace-forwarder
+            component-path: src/tools/trace-forwarder
+          - install-libseccomp: no
+          - component: agent
+            install-libseccomp: yes
+          - component: genpolicy
+            component-path: src/tools/genpolicy
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE" "$HOME"
+          sudo rm -rf "$GITHUB_WORKSPACE"/* || { sleep 10 && sudo rm -rf "$GITHUB_WORKSPACE"/*; }
+          sudo rm -f /tmp/kata_hybrid*  # Sometime we got leftover from test_setup_hvsock_failed()
+
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install yq
+        run: |
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        if: ${{ matrix.component == 'runtime' }}
+        run: |
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
+      - name: Install rust
+        if: ${{ matrix.component != 'runtime' }}
+        run: |
+          ./tests/install_rust.sh
+          echo "${HOME}/.cargo/bin" >> "$GITHUB_PATH"
+      - name: Install musl-tools
+        if: ${{ matrix.component != 'runtime' }}
+        run: sudo apt-get -y install musl-tools
+      - name: Install devicemapper
+        if: ${{ matrix.command == 'make check' && matrix.component == 'agent' }}
+        run: sudo apt-get -y install libdevmapper-dev
+      - name: Install libseccomp
+        if: ${{ matrix.command != 'make vendor'  &&  matrix.command != 'make check' &&  matrix.install-libseccomp == 'yes' }}
+        run: |
+          libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+          gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+          ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+          echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+          echo "LIBSECCOMP_LINK_TYPE=static" >> "$GITHUB_ENV"
+          echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> "$GITHUB_ENV"
+      - name: Install protobuf-compiler
+        if: ${{ matrix.command != 'make vendor' && (matrix.component == 'agent' || matrix.component == 'genpolicy' || matrix.component == 'agent-ctl') }}
+        run: sudo apt-get -y install protobuf-compiler
+      - name: Install clang
+        if: ${{ matrix.command == 'make check' && (matrix.component == 'agent' || matrix.component == 'agent-ctl') }}
+        run: sudo apt-get -y install clang
+      - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
+        if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
+        run: |
+          XDG_RUNTIME_DIR=$(mktemp -d "/tmp/kata-tests-$USER.XXX" | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> "$GITHUB_ENV"
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+          SKIP_GO_VERSION_CHECK: "1"

.github/workflows/build-kata-static-tarball-amd64.yaml

@@ -0,0 +1,331 @@
+name: CI | Build kata-static tarball for amd64
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - agent-ctl
+          - busybox
+          - cloud-hypervisor
+          - cloud-hypervisor-glibc
+          - coco-guest-components
+          - csi-kata-directvolume
+          - firecracker
+          - genpolicy
+          - kata-ctl
+          - kata-manager
+          - kernel
+          - kernel-confidential
+          - kernel-dragonball-experimental
+          - kernel-nvidia-gpu
+          - kernel-nvidia-gpu-confidential
+          - nydus
+          - ovmf
+          - ovmf-sev
+          - pause-image
+          - qemu
+          - qemu-snp-experimental
+          - stratovirt
+          - trace-forwarder
+          - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+        exclude:
+          - asset: cloud-hypervisor-glibc
+            stage: release
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      - uses: oras-project/setup-oras@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          version: "1.2.0"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+      - name: store-extratarballs-artifact ${{ matrix.asset }}
+        if: ${{ startsWith(matrix.asset, 'kernel-nvidia-gpu') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: ubuntu-22.04
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-image-mariner
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+          - rootfs-nvidia-gpu-initrd
+          - rootfs-nvidia-gpu-confidential-initrd
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - busybox
+          - coco-guest-components
+          - kernel-nvidia-gpu-headers
+          - kernel-nvidia-gpu-confidential-headers
+          - pause-image
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts-for-release:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-amd64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts, remove-rootfs-binary-artifacts-for-release]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-amd64-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-22.04
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-amd64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 15
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-arm64.yaml

@@ -0,0 +1,301 @@
+name: CI | Build kata-static tarball for arm64
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: ubuntu-22.04-arm
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - busybox
+          - cloud-hypervisor
+          - firecracker
+          - kernel
+          - kernel-dragonball-experimental
+          - kernel-nvidia-gpu
+          - nydus
+          - qemu
+          - stratovirt
+          - virtiofsd
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      - uses: oras-project/setup-oras@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          version: "1.2.0"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+      - name: store-extratarballs-artifact ${{ matrix.asset }}
+        if: ${{ startsWith(matrix.asset, 'kernel-nvidia-gpu') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}-headers${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}-headers.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: ubuntu-22.04-arm
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-initrd
+          - rootfs-nvidia-gpu-initrd
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04-arm
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - busybox
+          - kernel-nvidia-gpu-headers
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts-for-release:
+    runs-on: ubuntu-22.04-arm
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-arm64-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ubuntu-22.04-arm
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts, remove-rootfs-binary-artifacts-for-release]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-arm64-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ubuntu-22.04-arm
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-arm64-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 15
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-ppc64le.yaml

@@ -0,0 +1,260 @@
+name: CI | Build kata-static tarball for ppc64le
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: ppc64le
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - kernel
+          - qemu
+          - virtiofsd
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: ppc64le
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-initrd
+        stage:
+          - ${{ inputs.stage }}
+    steps:
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: build-asset-rootfs
+    strategy:
+      matrix:
+        asset:
+          - agent
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-ppc64le-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+            "${HOME}/scripts/prepare_runner.sh"
+            sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-ppc64le-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: ppc64le
+    needs: [build-asset, build-asset-rootfs, build-asset-shim-v2]
+    steps:
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-ppc64le-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-static-tarball-ppc64le${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/build-kata-static-tarball-s390x.yaml

@@ -0,0 +1,324 @@
+name: CI | Build kata-static tarball for s390x
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: false
+        type: string
+        default: test
+      tarball-suffix:
+        required: false
+        type: string
+      push-to-registry:
+        required: false
+        type: string
+        default: no
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-asset:
+    runs-on: s390x
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+      attestations: write
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - coco-guest-components
+          - kernel
+          - kernel-confidential
+          - pause-image
+          - qemu
+          - virtiofsd
+    env:
+      PERFORM_ATTESTATION: ${{ matrix.asset == 'agent' && inputs.push-to-registry == 'yes' && 'yes' || 'no' }}
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: Parse OCI image name and digest
+        id: parse-oci-segments
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        run: |
+          oci_image="$(<"build/${{ matrix.asset }}-oci-image")"
+          echo "oci-name=${oci_image%@*}" >> "$GITHUB_OUTPUT"
+          echo "oci-digest=${oci_image#*@}" >> "$GITHUB_OUTPUT"
+
+      # for pushing attestations to the registry
+      - uses: docker/login-action@v3
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/attest-build-provenance@v1
+        if: ${{ env.PERFORM_ATTESTATION == 'yes' }}
+        with:
+          subject-name: ${{ steps.parse-oci-segments.outputs.oci-name }}
+          subject-digest: ${{ steps.parse-oci-segments.outputs.oci-digest }}
+          push-to-registry: true
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-rootfs:
+    runs-on: s390x
+    needs: build-asset
+    strategy:
+      matrix:
+        asset:
+          - rootfs-image
+          - rootfs-image-confidential
+          - rootfs-initrd
+          - rootfs-initrd-confidential
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build ${{ matrix.asset }}
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: ${{ matrix.asset }}
+          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+
+      - name: store-artifact ${{ matrix.asset }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset }}${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  build-asset-boot-image-se:
+    runs-on: s390x
+    needs: [build-asset, build-asset-rootfs]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Place a host key document
+        run: |
+          mkdir -p "host-key-document"
+          cp "${CI_HKD_PATH}" "host-key-document"
+        env:
+          CI_HKD_PATH: ${{ secrets.CI_HKD_PATH }}
+
+      - name: Build boot-image-se
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "boot-image-se"
+          make boot-image-se-tarball
+          build_dir=$(readlink -f build)
+          sudo cp -r "${build_dir}" "kata-build"
+          sudo chown -R "$(id -u)":"$(id -g)" "kata-build"
+        env:
+          HKD_PATH: "host-key-document"
+
+      - name: store-artifact boot-image-se
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-boot-image-se.tar.xz
+          retention-days: 1
+          if-no-files-found: error
+
+  # We don't need the binaries installed in the rootfs as part of the release tarball, so can delete them now we've built the rootfs
+  remove-rootfs-binary-artifacts:
+    runs-on: ubuntu-22.04
+    needs: [build-asset-rootfs, build-asset-boot-image-se]
+    strategy:
+      matrix:
+        asset:
+          - agent
+          - coco-guest-components
+          - pause-image
+    steps:
+      - uses: geekyeggo/delete-artifact@v5
+        if: ${{ inputs.stage == 'release' }}
+        with:
+          name: kata-artifacts-s390x-${{ matrix.asset}}${{ inputs.tarball-suffix }}
+
+  build-asset-shim-v2:
+    runs-on: s390x
+    needs: [build-asset, build-asset-rootfs, remove-rootfs-binary-artifacts]
+    steps:
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.push-to-registry == 'yes' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0 # This is needed in order to keep the commit ids history
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+
+      - name: Build shim-v2
+        id: build
+        run: |
+          ./tests/gha-adjust-to-use-prebuilt-components.sh kata-artifacts "${KATA_ASSET}"
+          make "${KATA_ASSET}-tarball"
+          build_dir=$(readlink -f build)
+          # store-artifact does not work with symlink
+          mkdir -p kata-build && cp "${build_dir}"/kata-static-"${KATA_ASSET}"*.tar.* kata-build/.
+        env:
+          KATA_ASSET: shim-v2
+          TAR_OUTPUT: shim-v2.tar.gz
+          PUSH_TO_REGISTRY: ${{ inputs.push-to-registry }}
+          ARTEFACT_REGISTRY: ghcr.io
+          ARTEFACT_REGISTRY_USERNAME: ${{ github.actor }}
+          ARTEFACT_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+          RELEASE: ${{ inputs.stage == 'release' && 'yes' || 'no' }}
+          MEASURED_ROOTFS: yes
+
+      - name: store-artifact shim-v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-artifacts-s390x-shim-v2${{ inputs.tarball-suffix }}
+          path: kata-build/kata-static-shim-v2.tar.xz
+          retention-days: 15
+          if-no-files-found: error
+
+  create-kata-tarball:
+    runs-on: s390x
+    needs:
+      - build-asset
+      - build-asset-rootfs
+      - build-asset-boot-image-se
+      - build-asset-shim-v2
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+      - name: get-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: kata-artifacts-s390x-*${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+          merge-multiple: true
+      - name: merge-artifacts
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts versions.yaml
+      - name: store-artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+          path: kata-static.tar.xz
+          retention-days: 15
+          if-no-files-found: error

.github/workflows/cargo-deny-runner.yaml

@@ -0,0 +1,30 @@
+name: Cargo Crates Check Runner
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cargo-deny-runner:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout Code
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/checkout@v4
+      - name: Generate Action
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: bash cargo-deny-generator.sh
+        working-directory: ./.github/cargo-deny-composite-action/
+        env:
+          GOPATH: ${{ github.workspace }}/kata-containers
+      - name: Run Action
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: ./.github/cargo-deny-composite-action

.github/workflows/ci-coco-stability.yaml

@@ -0,0 +1,19 @@
+name: Kata Containers CoCo Stability Tests Weekly
+on:
+  schedule:
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci-weekly.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "weekly"
+      tag: ${{ github.sha }}-weekly
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/ci-devel.yaml

@@ -0,0 +1,13 @@
+name: Kata Containers CI (manually triggered)
+on:
+  workflow_dispatch:
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "dev"
+      tag: ${{ github.sha }}-dev
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/ci-nightly-s390x.yaml

@@ -0,0 +1,21 @@
+on:
+  schedule:
+    - cron: '0 5 * * *'
+
+name: Nightly CI for s390x
+jobs:
+  check-internal-test-result:
+    runs-on: s390x
+    strategy:
+      fail-fast: false
+      matrix:
+        test_title:
+          - kata-vfio-ap-e2e-tests
+          - cc-se-e2e-tests
+    steps:
+    - name: Fetch a test result for {{ matrix.test_title }}
+      run: |
+        file_name="${TEST_TITLE}-$(date +%Y-%m-%d).log"
+        "/home/${USER}/script/handle_test_log.sh" download "$file_name"
+      env:
+        TEST_TITLE: ${{ matrix.test_title }}

.github/workflows/ci-nightly.yaml

@@ -0,0 +1,18 @@
+name: Kata Containers Nightly CI
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-containers-ci-on-push:
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      pr-number: "nightly"
+      tag: ${{ github.sha }}-nightly
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit

.github/workflows/ci-on-push.yaml

@@ -0,0 +1,39 @@
+name: Kata Containers CI
+on:
+  pull_request_target:
+    branches:
+      - 'main'
+      - 'stable-*'
+    types:
+      # Adding 'labeled' to the list of activity types that trigger this event
+      # (default: opened, synchronize, reopened) so that we can run this
+      # workflow when the 'ok-to-test' label is added.
+      # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  skipper:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  kata-containers-ci-on-push:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
+    uses: ./.github/workflows/ci.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      pr-number: ${{ github.event.pull_request.number }}
+      tag: ${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+      skip-test: ${{ needs.skipper.outputs.skip_test }}
+    secrets: inherit

.github/workflows/ci-weekly.yaml

@@ -0,0 +1,87 @@
+name: Run the CoCo Kata Containers Stability CI
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  build-and-publish-tee-confidential-unencrypted-image:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v5
+        with:
+          tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }}
+          push: true
+          context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/
+          platforms: linux/amd64
+          file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
+
+  run-kata-coco-stability-tests:
+    needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-kata-coco-stability-tests.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+      tarball-suffix: -${{ inputs.tag }}
+    secrets: inherit

.github/workflows/ci.yaml

@@ -0,0 +1,300 @@
+name: Run the Kata Containers CI
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+      skip-test:
+        required: false
+        type: string
+        default: no
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  build-kata-static-tarball-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  publish-kata-deploy-payload-arm64:
+    needs: build-kata-static-tarball-arm64
+    uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-arm64
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  build-kata-static-tarball-ppc64le:
+    uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  publish-kata-deploy-payload-s390x:
+    needs: build-kata-static-tarball-s390x
+    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-s390x
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-ppc64le:
+    needs: build-kata-static-tarball-ppc64le
+    uses: ./.github/workflows/publish-kata-deploy-payload-ppc64le.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-ppc64le
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  build-and-publish-tee-confidential-unencrypted-image:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v5
+        with:
+          tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }}
+          push: true
+          context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/
+          platforms: linux/amd64, linux/s390x
+          file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
+
+  publish-csi-driver-amd64:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64-${{ inputs.tag }}
+          path: kata-artifacts
+
+      - name: Install tools
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
+      - name: Copy binary into Docker context
+        run: |
+          # Copy to the location where the Dockerfile expects the binary.
+          mkdir -p src/tools/csi-kata-directvolume/bin/
+          cp /opt/kata/bin/csi-kata-directvolume src/tools/csi-kata-directvolume/bin/directvolplugin
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Kata Containers ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker build and push
+        uses: docker/build-push-action@v5
+        with:
+          tags: ghcr.io/kata-containers/csi-kata-directvolume:${{ inputs.pr-number }}
+          push: true
+          context: src/tools/csi-kata-directvolume/
+          platforms: linux/amd64
+          file: src/tools/csi-kata-directvolume/Dockerfile
+
+  run-kata-monitor-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-kata-monitor-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-k8s-tests-on-aks:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-aks.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-k8s-tests-on-amd64:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-amd64
+    uses: ./.github/workflows/run-k8s-tests-on-amd64.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-kata-coco-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs:
+     - publish-kata-deploy-payload-amd64
+     - build-and-publish-tee-confidential-unencrypted-image
+     - publish-csi-driver-amd64
+    uses: ./.github/workflows/run-kata-coco-tests.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-amd64
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-k8s-tests-on-zvsi:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: [publish-kata-deploy-payload-s390x, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-k8s-tests-on-zvsi.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-s390x
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
+  run-k8s-tests-on-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: publish-kata-deploy-payload-ppc64le
+    uses: ./.github/workflows/run-k8s-tests-on-ppc64le.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-ppc64le
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-metrics-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/run-metrics.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-basic-amd64-tests:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-amd64
+    uses: ./.github/workflows/basic-ci-amd64.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-cri-containerd-tests-s390x:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-s390x
+    uses: ./.github/workflows/run-cri-containerd-tests-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  run-cri-containerd-tests-ppc64le:
+    if: ${{ inputs.skip-test != 'yes' }}
+    needs: build-kata-static-tarball-ppc64le
+    uses: ./.github/workflows/run-cri-containerd-tests-ppc64le.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}

.github/workflows/cleanup-resources.yaml

@@ -0,0 +1,31 @@
+name: Cleanup dangling Azure resources
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  cleanup-resources:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log into Azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+
+      - name: Install Python dependencies
+        run: |
+          pip3 install --user --upgrade \
+            azure-identity==1.16.0 \
+            azure-mgmt-resource==23.0.1
+
+      - name: Cleanup resources
+        env:
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          CLEANUP_AFTER_HOURS: 24 # Clean up resources created more than this many hours ago.
+        run: python3 tests/cleanup_resources.py

.github/workflows/commit-message-check.yaml

@@ -6,6 +6,10 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
@@ -14,7 +18,9 @@ env:
 
 jobs:
   commit-message-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    env:
+      PR_AUTHOR: ${{ github.event.pull_request.user.login }}
     name: Commit Message Check
     steps:
     - name: Get PR Commits
@@ -28,7 +34,10 @@ jobs:
         #
         # Revert "<original-subject-line>"
         #
-        filter_out_pattern: '^Revert "'
+        # The format of a re-re-vert commit as follows:
+        #
+        # Reapply "<original-subject-line>"
+        filter_out_pattern: '^Revert "|^Reapply "'
 
     - name: DCO Check
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
@@ -43,7 +52,7 @@ jobs:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
 
     - name: Check Subject Line Length
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
+      if: ${{ (env.PR_AUTHOR != 'dependabot[bot]') && !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
@@ -52,7 +61,7 @@ jobs:
         post_error: ${{ env.error_msg }}
 
     - name: Check Body Line Length
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
+      if: ${{ (env.PR_AUTHOR != 'dependabot[bot]') && !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
@@ -62,8 +71,12 @@ jobs:
         #   to be specified at the start of the regex as the action is passed
         #   the entire commit message.
         #
+        # - This check will pass if the commit message only contains a subject
+        #   line, as other body message properties are enforced elsewhere.
+        #
         # - Body lines *can* be longer than the maximum if they start
-        #   with a non-alphabetic character.
+        #   with a non-alphabetic character or if there is no whitespace in
+        #   the line.
         #
         #   This allows stack traces, log files snippets, emails, long URLs,
         #   etc to be specified. Some of these naturally "work" as they start
@@ -74,23 +87,12 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,149}|[^a-zA-Z\n].*|Signed-off-by:.*|))+$'
-        error: 'Body line too long (max 72)'
+        pattern: '(^[^\n]+$|^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$)'
+        error: 'Body line too long (max 150)'
         post_error: ${{ env.error_msg }}
 
-    - name: Check Fixes
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
-      uses: tim-actions/commit-message-checker-with-regex@v0.3.1
-      with:
-        commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '\s*Fixes\s*:?\s*(#\d+|github\.com\/kata-containers\/[a-z-.]*#\d+)|^\s*release\s*:'
-        flags: 'i'
-        error: 'No "Fixes" found'
-        post_error: ${{ env.error_msg }}
-        one_pass_all_pass: 'true'
-
     - name: Check Subsystem
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
+      if: ${{ (env.PR_AUTHOR != 'dependabot[bot]') && !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && ( success() || failure() ) }}
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}

.github/workflows/darwin-tests.yaml

@@ -6,20 +6,20 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 name: Darwin tests
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [macos-latest]
-    runs-on: ${{ matrix.os }}
+    runs-on: macos-latest
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.22.11
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Build utils
       run: ./ci/darwin-test.sh

.github/workflows/docs-url-alive-check.yaml

@@ -5,40 +5,28 @@ on:
 name: Docs URL Alive Check
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-22.04
+    # don't run this action on forks
+    if: github.repository_owner == 'kata-containers'
     env:
       target_branch: ${{ github.base_ref }}
     steps:
     - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.22.11
       env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
+        GOPATH: ${{ github.workspace }}/kata-containers
     - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+        echo "GOPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
+        echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
     # docs url alive check
     - name: Docs URL Alive Check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check
+        cd "${GOPATH}/src/github.com/${{ github.repository }}" && make docs-url-alive-check

.github/workflows/gatekeeper-skipper.yaml

@@ -0,0 +1,52 @@
+name: Skipper
+
+# This workflow sets various "skip_*" output values that can be used to
+# determine what workflows/jobs are expected to be executed. Sample usage:
+#
+#   skipper:
+#     uses: ./.github/workflows/gatekeeper-skipper.yaml
+#     with:
+#       commit-hash: ${{ github.event.pull_request.head.sha }}
+#       target-branch: ${{ github.event.pull_request.base.ref }}
+#
+#   your-workflow:
+#     needs: skipper
+#     if: ${{ needs.skipper.outputs.skip_build != 'yes' }}
+
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+    outputs:
+      skip_build:
+        value: ${{ jobs.skipper.outputs.skip_build }}
+      skip_test:
+        value: ${{ jobs.skipper.outputs.skip_test }}
+      skip_static:
+        value: ${{ jobs.skipper.outputs.skip_static }}
+
+
+jobs:
+  skipper:
+    runs-on: ubuntu-22.04
+    outputs:
+      skip_build: ${{ steps.skipper.outputs.skip_build }}
+      skip_test: ${{ steps.skipper.outputs.skip_test }}
+      skip_static: ${{ steps.skipper.outputs.skip_static }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+      - id: skipper
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+        run: |
+          python3 tools/testing/gatekeeper/skips.py | tee -a "$GITHUB_OUTPUT"
+        shell: /usr/bin/bash -x {0}

.github/workflows/gatekeeper.yaml

@@ -0,0 +1,44 @@
+name: Gatekeeper
+
+# Gatekeeper uses the "skips.py" to determine which job names/regexps are
+# required for given PR and waits for them to either complete or fail
+# reporting the status.
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  gatekeeper:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+      - id: gatekeeper
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMIT_HASH: ${{ github.event.pull_request.head.sha }}
+          GH_PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          #!/usr/bin/env bash -x
+          mapfile -t lines < <(python3 tools/testing/gatekeeper/skips.py -t)
+          export REQUIRED_JOBS="${lines[0]}"
+          export REQUIRED_REGEXPS="${lines[1]}"
+          export REQUIRED_LABELS="${lines[2]}"
+          echo "REQUIRED_JOBS: $REQUIRED_JOBS"
+          echo "REQUIRED_REGEXPS: $REQUIRED_REGEXPS"
+          echo "REQUIRED_LABELS: $REQUIRED_LABELS"
+          python3 tools/testing/gatekeeper/jobs.py
+          exit $?
+        shell: /usr/bin/bash -x {0}

.github/workflows/kata-deploy-push.yaml

@@ -1,83 +0,0 @@
-name: kata deploy build
-
-on: 
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths:
-      - tools/**
-      - versions.yaml
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - kernel
-          - shim-v2
-          - qemu
-          - cloud-hypervisor
-          - firecracker
-          - rootfs-image
-          - rootfs-initrd
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Build ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r --preserve=all "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-
-      - name: store-artifact ${{ matrix.asset }}
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: build
-      - name: merge-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make merge-builds
-      - name: store-artifacts
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  make-kata-tarball:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: make kata-tarball
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          make kata-tarball
-          sudo make install-tarball

.github/workflows/kata-deploy-test.yaml

@@ -1,147 +0,0 @@
-on:
-  issue_comment:
-    types: [created, edited]
-
-name: test-kata-deploy
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/test_kata_deploy')
-    steps:
-      - name: Check membership
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: get-PR-ref
-        id: get-PR-ref
-        run: |
-            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            echo "reference for PR: " ${ref}
-            echo "##[set-output name=pr-ref;]${ref}"
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          PR_SHA=$(git log --format=format:%H -n1)
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/kata-containers/kata-deploy-ci:$PR_SHA $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$PR_SHA
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${PR_SHA}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-        env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}

.github/workflows/kata-runtime-classes-sync.yaml

@@ -0,0 +1,36 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  kata-deploy-runtime-classes-check:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Ensure the split out runtime classes match the all-in-one file
+      run: |
+        pushd tools/packaging/kata-deploy/runtimeclasses/
+        echo "::group::Combine runtime classes"
+        for runtimeClass in $(find . -type f \( -name "*.yaml" -and -not -name "kata-runtimeClasses.yaml" \) | sort); do
+            echo "Adding ${runtimeClass} to the resultingRuntimeClasses.yaml"
+            cat "${runtimeClass}" >> resultingRuntimeClasses.yaml;
+        done
+        echo "::endgroup::"
+        echo "::group::Displaying the content of resultingRuntimeClasses.yaml"
+        cat resultingRuntimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        echo "::group::Displaying the content of kata-runtimeClasses.yaml"
+        cat kata-runtimeClasses.yaml
+        echo "::endgroup::"
+        echo ""
+        diff resultingRuntimeClasses.yaml kata-runtimeClasses.yaml

.github/workflows/move-issues-to-in-progress.yaml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   move-linked-issues-to-in-progress:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Install hub
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
@@ -31,14 +31,24 @@ jobs:
         run: |
           # Clone into a temporary directory to avoid overwriting
           # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
+          pushd "$(mktemp -d)" &>/dev/null
           git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
           sudo install hub-util.sh /usr/local/bin
           popd &>/dev/null
 
       - name: Checkout code to allow hub to communicate with the project
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
 
       - name: Move issue to "In progress"
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
@@ -52,20 +62,19 @@ jobs:
             grep -v "^\#"  |\
             cut -d';' -f3 || true)
 
-          # PR doesn't have any linked issues
-          # (it should, but maybe a new user forgot to add a "Fixes: #XXX" commit).
+          # PR doesn't have any linked issues, handle it only if it exists
           [ -z "$linked_issue_urls" ] && {
-            echo "::error::No linked issues for PR $pr"
-            exit 1
+            echo "::warning::No linked issues for PR $pr"
+            exit 0
           }
 
           project_name="Issue backlog"
           project_type="org"
           project_column="In progress"
 
-          for issue_url in $(echo "$linked_issue_urls")
+          for issue_url in $linked_issue_urls
           do
-            issue=$(echo "$issue_url"| awk -F\/ '{print $NF}' || true)
+            issue=$(echo "$issue_url"| awk -F/ '{print $NF}' || true)
 
             [ -z "$issue" ] && {
               echo "::error::Cannot determine issue number from $issue_url for PR $pr"

.github/workflows/payload-after-push.yaml

@@ -0,0 +1,107 @@
+name: CI | Publish Kata Containers payload
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  build-assets-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  build-assets-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  build-assets-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+  
+  build-assets-ppc64le:
+    uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      push-to-registry: yes
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-amd64:
+    needs: build-assets-amd64
+    uses: ./.github/workflows/publish-kata-deploy-payload-amd64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-latest-amd64
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-arm64:
+    needs: build-assets-arm64
+    uses: ./.github/workflows/publish-kata-deploy-payload-arm64.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-latest-arm64
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-s390x:
+    needs: build-assets-s390x
+    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-latest-s390x
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-kata-deploy-payload-ppc64le:
+    needs: build-assets-ppc64le
+    uses: ./.github/workflows/publish-kata-deploy-payload-ppc64le.yaml
+    with:
+      commit-hash: ${{ github.sha }}
+      registry: quay.io
+      repo: kata-containers/kata-deploy-ci
+      tag: kata-containers-latest-ppc64le
+      target-branch: ${{ github.ref_name }}
+    secrets: inherit
+
+  publish-manifest:
+    runs-on: ubuntu-22.04
+    needs: [publish-kata-deploy-payload-amd64, publish-kata-deploy-payload-arm64, publish-kata-deploy-payload-s390x, publish-kata-deploy-payload-ppc64le]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Push multi-arch manifest
+        run: |
+          ./tools/packaging/release/release.sh publish-multiarch-manifest
+        env:
+          KATA_DEPLOY_IMAGE_TAGS: "kata-containers-latest"
+          KATA_DEPLOY_REGISTRIES: "quay.io/kata-containers/kata-deploy-ci"

.github/workflows/publish-kata-deploy-payload-amd64.yaml

@@ -0,0 +1,66 @@
+name: CI | Publish kata-deploy payload for amd64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          "$(pwd)"/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-arm64.yaml

@@ -0,0 +1,66 @@
+name: CI | Publish kata-deploy payload for arm64
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: ubuntu-22.04-arm
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-arm64${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          "$(pwd)"/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-ppc64le.yaml

@@ -0,0 +1,76 @@
+name: CI | Publish kata-deploy payload for ppc64le
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: ppc64le
+    steps:
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+          "${HOME}/scripts/prepare_runner.sh"
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Adjust a permission for repo
+        run: |
+          sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-ppc64le${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          "$(pwd)"/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/publish-kata-deploy-payload-s390x.yaml

@@ -0,0 +1,66 @@
+name: CI | Publish kata-deploy payload for s390x
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  kata-payload:
+    runs-on: s390x
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+
+      - name: Login to Kata Containers quay.io
+        if: ${{ inputs.registry == 'quay.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Login to Kata Containers ghcr.io
+        if: ${{ inputs.registry == 'ghcr.io' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: build-and-push-kata-payload
+        id: build-and-push-kata-payload
+        run: |
+          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+          "$(pwd)"/kata-static.tar.xz \
+          ${{ inputs.registry }}/${{ inputs.repo }} ${{ inputs.tag }}

.github/workflows/release-amd64.yaml

@@ -0,0 +1,59 @@
+name: Publish Kata release artifacts for amd64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-amd64:
+    uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml
+    with:
+      push-to-registry: yes
+      stage: release
+    secrets: inherit
+
+  kata-deploy:
+    needs: build-kata-static-tarball-amd64
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64
+
+      - name: build-and-push-kata-deploy-ci-amd64
+        id: build-and-push-kata-deploy-ci-amd64
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF
+          # is "refs/tags/<tag>"
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
+          if [ "${tag}" = "main" ]; then
+              tag=$(./tools/packaging/release/release.sh release-version)
+              tags=("${tag}" "latest")
+          else
+              tags=("${tag}")
+          fi
+          for tag in "${tags[@]}"; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-arm64.yaml

@@ -0,0 +1,59 @@
+name: Publish Kata release artifacts for arm64
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-arm64:
+    uses: ./.github/workflows/build-kata-static-tarball-arm64.yaml
+    with:
+      push-to-registry: yes
+      stage: release
+    secrets: inherit
+
+  kata-deploy:
+    needs: build-kata-static-tarball-arm64
+    runs-on: ubuntu-22.04-arm
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: build-and-push-kata-deploy-ci-arm64
+        id: build-and-push-kata-deploy-ci-arm64
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF
+          # is "refs/tags/<tag>"
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
+          if [ "${tag}" = "main" ]; then
+              tag=$(./tools/packaging/release/release.sh release-version)
+              tags=("${tag}" "latest")
+          else
+              tags=("${tag}")
+          fi
+          for tag in "${tags[@]}"; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-ppc64le.yaml

@@ -0,0 +1,65 @@
+name: Publish Kata release artifacts for ppc64le
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-ppc64le:
+    uses: ./.github/workflows/build-kata-static-tarball-ppc64le.yaml
+    with:
+      push-to-registry: yes
+      stage: release
+    secrets: inherit
+
+  kata-deploy:
+    needs: build-kata-static-tarball-ppc64le
+    runs-on: ppc64le
+    steps:
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+          bash "${HOME}/scripts/prepare_runner.sh"
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-ppc64le
+
+      - name: build-and-push-kata-deploy-ci-ppc64le
+        id: build-and-push-kata-deploy-ci-ppc64le
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF
+          # is "refs/tags/<tag>"
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
+          if [ "${tag}" = "main" ]; then
+              tag=$(./tools/packaging/release/release.sh release-version)
+              tags=("${tag}" "latest")
+          else
+              tags=("${tag}")
+          fi
+          for tag in "${tags[@]}"; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release-s390x.yaml

@@ -0,0 +1,59 @@
+name: Publish Kata release artifacts for s390x
+on:
+  workflow_call:
+    inputs:
+      target-arch:
+        required: true
+        type: string
+
+jobs:
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      push-to-registry: yes
+      stage: release
+    secrets: inherit
+
+  kata-deploy:
+    needs: build-kata-static-tarball-s390x
+    runs-on: s390x
+    steps:
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - uses: actions/checkout@v4
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: build-and-push-kata-deploy-ci-s390x
+        id: build-and-push-kata-deploy-ci-s390x
+        run: |
+          # We need to do such trick here as the format of the $GITHUB_REF
+          # is "refs/tags/<tag>"
+          tag=$(echo "$GITHUB_REF" | cut -d/ -f3-)
+          if [ "${tag}" = "main" ]; then
+              tag=$(./tools/packaging/release/release.sh release-version)
+              tags=("${tag}" "latest")
+          else
+              tags=("${tag}")
+          fi
+          for tag in "${tags[@]}"; do
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "docker.io/katadocker/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+              ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh \
+                  "$(pwd)"/kata-static.tar.xz "quay.io/kata-containers/kata-deploy" \
+                  "${tag}-${{ inputs.target-arch }}"
+          done

.github/workflows/release.yaml

@@ -1,177 +1,206 @@
-name: Publish Kata 2.x release artifacts
+name: Release Kata Containers
 on:
-  push:
-    tags:
-     - '2.*'
+  workflow_dispatch
 
 jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
+  release:
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-      - name: Build ${{ matrix.asset }}
+      - name: Create a new release
         run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
+          ./tools/packaging/release/release.sh create-new-release
         env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
+          GH_TOKEN: ${{ github.token }}
 
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
+  build-and-push-assets-amd64:
+    needs: release
+    uses: ./.github/workflows/release-amd64.yaml
+    with:
+      target-arch: amd64
+    secrets: inherit
 
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
+  build-and-push-assets-arm64:
+    needs: release
+    uses: ./.github/workflows/release-arm64.yaml
+    with:
+      target-arch: arm64
+    secrets: inherit
+
+  build-and-push-assets-s390x:
+    needs: release
+    uses: ./.github/workflows/release-s390x.yaml
+    with:
+      target-arch: s390x
+    secrets: inherit
+
+  build-and-push-assets-ppc64le:
+    needs: release
+    uses: ./.github/workflows/release-ppc64le.yaml
+    with:
+      target-arch: ppc64le
+    secrets: inherit
+
+  publish-multi-arch-images:
+    runs-on: ubuntu-22.04
+    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x, build-and-push-assets-ppc64le]
     steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
+      - name: Login to Kata Containers docker.io
+        uses: docker/login-action@v3
         with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Kata Containers quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
+          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
+
+      - name: Get the image tags
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t katadocker/kata-deploy-ci:$pkg_sha -t quay.io/kata-containers/kata-deploy-ci:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
-          docker push katadocker/kata-deploy-ci:$pkg_sha
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/kata-containers/kata-deploy-ci:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"
-      - name: test-kata-deploy-ci-in-aks
-        uses: ./packaging/kata-deploy/action
-        with:
-          packaging-sha: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
+          release_version=$(./tools/packaging/release/release.sh release-version)
+          echo "KATA_DEPLOY_IMAGE_TAGS=$release_version latest" >> "$GITHUB_ENV"
+
+      - name: Publish multi-arch manifest on docker.io and quay.io
+        run: |
+          ./tools/packaging/release/release.sh publish-multiarch-manifest
         env:
-          PKG_SHA: ${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}}
-          AZ_APPID: ${{ secrets.AZ_APPID }}
-          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
-          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
-          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
-      - name: push-tarball
-        run: |
-          # tag the container image we created and push to DockerHub
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tags=($tag)
-          tags+=($([[ "$tag" =~ "alpha"|"rc" ]] && echo "latest" || echo "stable"))
-          for tag in ${tags[@]}; do \
-            docker tag katadocker/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} katadocker/kata-deploy:${tag} && \
-            docker tag quay.io/kata-containers/kata-deploy-ci:${{steps.build-and-push-kata-deploy-ci.outputs.PKG_SHA}} quay.io/kata-containers/kata-deploy:${tag} && \
-            docker push katadocker/kata-deploy:${tag} && \
-            docker push quay.io/kata-containers/kata-deploy:${tag}; \
-          done
+          KATA_DEPLOY_REGISTRIES: "quay.io/kata-containers/kata-deploy docker.io/katadocker/kata-deploy"
 
-  upload-static-tarball:
-    needs: kata-deploy
-    runs-on: ubuntu-latest
+  upload-multi-arch-static-tarball:
+    needs: [build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x, build-and-push-assets-ppc64le]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - name: download-artifacts
-        uses: actions/download-artifact@v2
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set KATA_STATIC_TARBALL env var
+        run: |
+          tarball=$(pwd)/kata-static.tar.xz
+          echo "KATA_STATIC_TARBALL=${tarball}" >> "$GITHUB_ENV"
+
+      - name: Download amd64 artifacts
+        uses: actions/download-artifact@v4
         with:
-          name: kata-static-tarball
-      - name: install hub
+          name: kata-static-tarball-amd64
+
+      - name: Upload amd64 static tarball to GitHub
         run: |
-          HUB_VER=$(curl -s "https://api.github.com/repos/github/hub/releases/latest" | jq -r .tag_name | sed 's/^v//')
-          wget -q -O- https://github.com/github/hub/releases/download/v$HUB_VER/hub-linux-amd64-$HUB_VER.tgz | \
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && sudo mv hub /usr/local/bin/hub
-      - name: push static tarball to github
+          ./tools/packaging/release/release.sh upload-kata-static-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ARCHITECTURE: amd64
+
+      - name: Download arm64 artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-arm64
+
+      - name: Upload arm64 static tarball to GitHub
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-static-$tag-x86_64.tar.xz"
-          mv kata-static.tar.xz "$GITHUB_WORKSPACE/${tarball}"
-          pushd $GITHUB_WORKSPACE
-          echo "uploading asset '${tarball}' for tag: ${tag}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}"
-          popd
+          ./tools/packaging/release/release.sh upload-kata-static-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ARCHITECTURE: arm64
+
+      - name: Download s390x artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-s390x
+
+      - name: Upload s390x static tarball to GitHub
+        run: |
+          ./tools/packaging/release/release.sh upload-kata-static-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ARCHITECTURE: s390x
+
+      - name: Download ppc64le artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-ppc64le
+
+      - name: Upload ppc64le static tarball to GitHub
+        run: |
+          ./tools/packaging/release/release.sh upload-kata-static-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ARCHITECTURE: ppc64le
+
+  upload-versions-yaml:
+    needs: release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Upload versions.yaml to GitHub
+        run: |
+          ./tools/packaging/release/release.sh upload-versions-yaml-file
+        env:
+          GH_TOKEN: ${{ github.token }}
 
   upload-cargo-vendored-tarball:
-    needs: upload-static-tarball
-    runs-on: ubuntu-latest
+    needs: release
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - name: generate-and-upload-tarball
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Generate and upload vendored code tarball
         run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          tarball="kata-containers-$tag-vendor.tar.gz"
-          pushd $GITHUB_WORKSPACE
-          bash -c "tools/packaging/release/generate_vendor.sh ${tarball}"
-          GITHUB_TOKEN=${{ secrets.GIT_UPLOAD_TOKEN }} hub release edit -m "" -a "${tarball}" "${tag}" 
-          popd
+          ./tools/packaging/release/release.sh upload-vendored-code-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
 
   upload-libseccomp-tarball:
-    needs: upload-cargo-vendored-tarball
-    runs-on: ubuntu-latest
+    needs: release
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - name: download-and-upload-tarball
-        env:
-          GITHUB_TOKEN: ${{ secrets.GIT_UPLOAD_TOKEN }}
-          GOPATH: ${HOME}/go
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download libseccomp tarball and upload it to GitHub
         run: |
-          pushd $GITHUB_WORKSPACE
-          ./ci/install_yq.sh
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          versions_yaml="versions.yaml"
-          version=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.version")
-          repo_url=$(${GOPATH}/bin/yq read ${versions_yaml} "externals.libseccomp.url")
-          download_url="${repo_url}/releases/download/v${version}"
-          tarball="libseccomp-${version}.tar.gz"
-          asc="${tarball}.asc"
-          curl -sSLO "${download_url}/${tarball}"
-          curl -sSLO "${download_url}/${asc}"
-          # "-m" option should be empty to re-use the existing release title
-          # without opening a text editor.
-          # For the details, check https://hub.github.com/hub-release.1.html.
-          hub release edit -m "" -a "${tarball}" "${tag}"
-          hub release edit -m "" -a "${asc}" "${tag}"
-          popd
+          ./tools/packaging/release/release.sh upload-libseccomp-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  upload-helm-chart-tarball:
+    needs: release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install helm
+        uses: azure/setup-helm@v4.2.0
+        id: install
+
+      - name: Generate and upload helm chart tarball
+        run: |
+          ./tools/packaging/release/release.sh upload-helm-chart-tarball
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  publish-release:
+    needs: [ build-and-push-assets-amd64, build-and-push-assets-arm64, build-and-push-assets-s390x, build-and-push-assets-ppc64le, publish-multi-arch-images, upload-multi-arch-static-tarball, upload-versions-yaml, upload-cargo-vendored-tarball, upload-libseccomp-tarball ]
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Publish a release
+        run: |
+          ./tools/packaging/release/release.sh publish-release
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/require-pr-porting-labels.yaml

@@ -1,54 +0,0 @@
-# Copyright (c) 2020 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-name: Ensure PR has required porting labels
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - reopened
-      - labeled
-      - unlabeled
-    branches:
-      - main
-
-jobs:
-  check-pr-porting-labels:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install hub
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          HUB_ARCH="amd64"
-          HUB_VER=$(curl -sL "https://api.github.com/repos/github/hub/releases/latest" |\
-            jq -r .tag_name | sed 's/^v//')
-          curl -sL \
-            "https://github.com/github/hub/releases/download/v${HUB_VER}/hub-linux-${HUB_ARCH}-${HUB_VER}.tgz" |\
-          tar xz --strip-components=2 --wildcards '*/bin/hub' && \
-          sudo install hub /usr/local/bin
-
-      - name: Checkout code to allow hub to communicate with the project
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
-
-      - name: Install porting checker script
-        run: |
-          # Clone into a temporary directory to avoid overwriting
-          # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
-          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
-          sudo install pr-porting-checks.sh /usr/local/bin
-          popd &>/dev/null
-
-      - name: Stop PR being merged unless it has a correct set of porting labels
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }}
-        run: |
-          pr=${{ github.event.number }}
-          repo=${{ github.repository }}
-
-          pr-porting-checks.sh "$pr" "$repo"

.github/workflows/run-cri-containerd-tests-ppc64le.yaml

@@ -0,0 +1,69 @@
+name: CI | Run cri-containerd tests on ppc64le
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance
+      fail-fast: false
+      matrix:
+        containerd_version: ['active']
+        vmm: ['qemu']
+    runs-on: ppc64le
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - name: Adjust a permission for repo
+        run: sudo chown -R "$USER":"$USER" "$GITHUB_WORKSPACE"
+
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+          bash "${HOME}/scripts/prepare_runner.sh" cri-containerd
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-ppc64le${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+      - name: Cleanup actions for the self hosted runner
+        run: bash "${HOME}/scripts/cleanup_runner.sh"

.github/workflows/run-cri-containerd-tests-s390x.yaml

@@ -0,0 +1,56 @@
+name: CI | Run cri-containerd tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance
+      fail-fast: false
+      matrix:
+        containerd_version: ['active']
+        vmm: ['qemu', 'qemu-runtime-rs']
+    runs-on: s390x-large
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -0,0 +1,133 @@
+name: CI | Run kubernetes tests on AKS
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+          - qemu-runtime-rs
+          - stratovirt
+          - cloud-hypervisor
+        instance-type:
+          - small
+          - normal
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+            instance-type: small
+            genpolicy-pull-method: oci-distribution
+            auto-generate-policy: yes
+          - host_os: cbl-mariner
+            vmm: clh
+            instance-type: small
+            genpolicy-pull-method: containerd
+            auto-generate-policy: yes
+          - host_os: cbl-mariner
+            vmm: clh
+            instance-type: normal
+            auto-generate-policy: yes
+    runs-on: ubuntu-22.04
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
+      GENPOLICY_PULL_METHOD: ${{ matrix.genpolicy-pull-method }}
+      AUTO_GENERATE_POLICY: ${{ matrix.auto-generate-policy }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
+      - name: Download Azure CLI
+        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
+      - name: Create AKS cluster
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 15
+          max_attempts: 20
+          retry_on: error
+          retry_wait_seconds: 10
+          command: bash tests/integration/kubernetes/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/integration/kubernetes/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
+
+      - name: Run tests
+        timeout-minutes: 60
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-amd64.yaml

@@ -0,0 +1,109 @@
+name: CI | Run kubernetes tests on amd64
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests-amd64:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh #cloud-hypervisor
+          - dragonball
+          - fc #firecracker
+          - qemu
+          - cloud-hypervisor
+        container_runtime:
+          - containerd
+        snapshotter:
+          - devmapper
+        k8s:
+          - k3s
+        include:
+          - vmm: qemu
+            container_runtime: crio
+            snapshotter: ""
+            k8s: k0s
+    runs-on: ubuntu-22.04
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      KUBERNETES_EXTRA_PARAMS: ${{ matrix.container_runtime != 'crio' && '' || '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"' }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+      K8S_TEST_HOST_TYPE: all
+      CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Configure CRI-O
+        if: matrix.container_runtime == 'crio'
+        run: bash tests/integration/kubernetes/gha-run.sh setup-crio
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        if: matrix.snapshotter != ''
+        run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Collect artifacts ${{ matrix.vmm }}
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh collect-artifacts
+        continue-on-error: true
+
+      - name: Archive artifacts ${{ matrix.vmm }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: k8s-tests-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ inputs.tag }}
+          path: /tmp/artifacts
+          retention-days: 1
+
+      - name: Delete kata-deploy
+        if: always()
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup

.github/workflows/run-k8s-tests-on-ppc64le.yaml

@@ -0,0 +1,83 @@
+name: CI | Run kubernetes tests on Power(ppc64le)
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        k8s:
+          - kubeadm
+    runs-on: k8s-ppc64le
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+      TARGET_ARCH: "ppc64le"
+    steps:
+      - name: Prepare the self-hosted runner
+        timeout-minutes: 15
+        run: |
+          bash "${HOME}/scripts/prepare_runner.sh" kubernetes
+          sudo rm -rf "$GITHUB_WORKSPACE"/*
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install golang
+        run: |
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
+
+      - name: Prepare the runner for k8s cluster creation
+        run: bash "${HOME}/scripts/k8s_cluster_cleanup.sh"
+
+      - name: Create k8s cluster using kubeadm
+        run: bash "${HOME}/scripts/k8s_cluster_create.sh"
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-kubeadm
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete cluster and post cleanup actions
+        run: bash "${HOME}/scripts/k8s_cluster_cleanup.sh"

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -0,0 +1,137 @@
+name: CI | Run kubernetes tests on IBM Cloud Z virtual server instance (zVSI)
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        snapshotter:
+          - overlayfs
+          - devmapper
+          - nydus
+        vmm:
+          - qemu
+          - qemu-runtime-rs
+          - qemu-coco-dev
+        k8s:
+          - kubeadm
+        include:
+          - snapshotter: devmapper
+            pull-type: default
+            using-nfd: true
+            deploy-cmd: configure-snapshotter
+          - snapshotter: nydus
+            pull-type: guest-pull
+            using-nfd: false
+            deploy-cmd: deploy-snapshotter
+        exclude:
+          - snapshotter: overlayfs
+            vmm: qemu
+          - snapshotter: overlayfs
+            vmm: qemu-coco-dev
+          - snapshotter: devmapper
+            vmm: qemu-runtime-rs
+          - snapshotter: devmapper
+            vmm: qemu-coco-dev
+          - snapshotter: nydus
+            vmm: qemu
+          - snapshotter: nydus
+            vmm: qemu-runtime-rs
+    runs-on: s390x-large
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: "ubuntu"
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      PULL_TYPE: ${{ matrix.pull-type }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: ${{ matrix.using-nfd }}
+      TARGET_ARCH: "s390x"
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Set SNAPSHOTTER to empty if overlayfs
+        run: echo "SNAPSHOTTER=" >> "$GITHUB_ENV"
+        if: ${{ matrix.snapshotter == 'overlayfs' }}
+
+      - name: Set KBS and KBS_INGRESS if qemu-coco-dev
+        run: |
+          echo "KBS=true" >> "$GITHUB_ENV"
+          echo "KBS_INGRESS=nodeport" >> "$GITHUB_ENV"
+        if: ${{ matrix.vmm == 'qemu-coco-dev' }}
+
+      # qemu-runtime-rs only works with overlayfs
+      # See: https://github.com/kata-containers/kata-containers/issues/10066
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        run: bash tests/integration/kubernetes/gha-run.sh ${{ matrix.deploy-cmd }}
+        if: ${{ matrix.snapshotter != 'overlayfs' }}
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-zvsi
+
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+        if: ${{ matrix.vmm == 'qemu-coco-dev' }}
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+        if: ${{ matrix.vmm == 'qemu-coco-dev' }}
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+        if: ${{ matrix.vmm == 'qemu-coco-dev' }}
+
+      - name: Run tests
+        timeout-minutes: 60
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-zvsi
+
+      - name: Delete CoCo KBS
+        if: always()
+        run: |
+          if [ "${KBS}" == "true" ]; then
+            bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+          fi

.github/workflows/run-kata-coco-stability-tests.yaml

@@ -0,0 +1,129 @@
+name: CI | Run Kata CoCo k8s Stability Tests
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+      tarball-suffix:
+        required: false
+        type: string
+
+jobs:
+  # Generate jobs for testing CoCo on non-TEE environments
+  run-stability-k8s-tests-coco-nontee:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-coco-dev
+        snapshotter:
+          - nydus
+        pull-type:
+          - guest-pull
+    runs-on: ubuntu-22.04
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      # Some tests rely on that variable to run (or not)
+      KBS: "true"
+      # Set the KBS ingress handler (empty string disables handling)
+      KBS_INGRESS: "aks"
+      KUBERNETES: "vanilla"
+      PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
+      - name: Download Azure CLI
+        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
+      - name: Create AKS cluster
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 15
+          max_attempts: 20
+          retry_on: error
+          retry_wait_seconds: 10
+          command: bash tests/integration/kubernetes/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/integration/kubernetes/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
+
+      - name: Deploy Snapshotter
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Run stability tests
+        timeout-minutes: 300
+        run: bash tests/stability/gha-stability-run.sh run-tests
+
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-kata-coco-tests.yaml

@@ -0,0 +1,303 @@
+name: CI | Run kata coco tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests-on-tdx:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-tdx
+        snapshotter:
+          - nydus
+        pull-type:
+          - guest-pull
+    runs-on: tdx
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "true"
+      KBS: "true"
+      K8S_TEST_HOST_TYPE: "baremetal"
+      KBS_INGRESS: "nodeport"
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      ITA_KEY: ${{ secrets.ITA_KEY }}
+      AUTO_GENERATE_POLICY: "yes"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Snapshotter
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx
+
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
+      - name: Run tests
+        timeout-minutes: 100
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx
+
+      - name: Delete Snapshotter
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
+
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
+  # AMD has deprecated SEV support on Kata and henceforth SNP will be the only feature supported for Kata Containers.
+  run-k8s-tests-sev-snp:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-snp
+        snapshotter:
+          - nydus
+        pull-type:
+          - guest-pull
+    runs-on: sev-snp
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBECONFIG: /home/kata/.kube/config
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+      KBS: "true"
+      KBS_INGRESS: "nodeport"
+      K8S_TEST_HOST_TYPE: "baremetal"
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      AUTO_GENERATE_POLICY: "yes"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy Snapshotter
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-snp
+
+      - name: Uninstall previous `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh uninstall-kbs-client
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
+      - name: Run tests
+        timeout-minutes: 50
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete kata-deploy
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snp
+
+      - name: Delete Snapshotter
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh cleanup-snapshotter
+
+      - name: Delete CoCo KBS
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-coco-kbs
+
+      - name: Delete CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh delete-csi-driver
+
+  # Generate jobs for testing CoCo on non-TEE environments
+  run-k8s-tests-coco-nontee:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu-coco-dev
+        snapshotter:
+          - nydus
+        pull-type:
+          - guest-pull
+    runs-on: ubuntu-22.04
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      # Some tests rely on that variable to run (or not)
+      KBS: "true"
+      # Set the KBS ingress handler (empty string disables handling)
+      KBS_INGRESS: "aks"
+      KUBERNETES: "vanilla"
+      PULL_TYPE: ${{ matrix.pull-type }}
+      AUTHENTICATED_IMAGE_USER: ${{ secrets.AUTHENTICATED_IMAGE_USER }}
+      AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }}
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "false"
+      AUTO_GENERATE_POLICY: "yes"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-artifacts
+
+      - name: Download Azure CLI
+        run: bash tests/integration/kubernetes/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/integration/kubernetes/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
+      - name: Create AKS cluster
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 15
+          max_attempts: 20
+          retry_on: error
+          retry_wait_seconds: 10
+          command: bash tests/integration/kubernetes/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/integration/kubernetes/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/integration/kubernetes/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials
+
+      - name: Deploy Snapshotter
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks
+
+      - name: Deploy CoCo KBS
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs
+
+      - name: Install `kbs-client`
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client
+
+      - name: Deploy CSI driver
+        timeout-minutes: 5
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-csi-driver
+
+      - name: Run tests
+        timeout-minutes: 80
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/integration/kubernetes/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-aks.yaml

@@ -0,0 +1,96 @@
+name: CI | Run kata-deploy tests on AKS
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        host_os:
+          - ubuntu
+        vmm:
+          - clh
+          - dragonball
+          - qemu
+          - qemu-runtime-rs
+        include:
+          - host_os: cbl-mariner
+            vmm: clh
+    runs-on: ubuntu-22.04
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "vanilla"
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Download Azure CLI
+        run: bash tests/functional/kata-deploy/gha-run.sh install-azure-cli
+
+      - name: Log into the Azure account
+        run: bash tests/functional/kata-deploy/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
+          AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+
+      - name: Create AKS cluster
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 15
+          max_attempts: 20
+          retry_on: error
+          retry_wait_seconds: 10
+          command: bash tests/integration/kubernetes/gha-run.sh create-cluster
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Install `kubectl`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-kubectl
+
+      - name: Download credentials for the Kubernetes CLI to use them
+        run: bash tests/functional/kata-deploy/gha-run.sh get-cluster-credentials
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests
+
+      - name: Delete AKS cluster
+        if: always()
+        run: bash tests/functional/kata-deploy/gha-run.sh delete-cluster

.github/workflows/run-kata-deploy-tests-on-garm.yaml

@@ -0,0 +1,69 @@
+name: CI | Run kata-deploy tests on GARM
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-kata-deploy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - clh
+          - qemu
+        k8s:
+          - k0s
+          - k3s
+          - rke2
+    # TODO: There are a couple of vmm/k8s combination failing (https://github.com/kata-containers/kata-containers/issues/9854)
+    # and we will put the entire kata-deploy-tests on GARM on maintenance.
+    # TODO: Transition to free runner (see #9940).
+    if: false
+    runs-on: garm-ubuntu-2004-smaller
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: ${{ matrix.k8s }}
+      USING_NFD: "false"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run:  bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
+
+      - name: Install `bats`
+        run: bash tests/functional/kata-deploy/gha-run.sh install-bats
+
+      - name: Run tests
+        run: bash tests/functional/kata-deploy/gha-run.sh run-tests

.github/workflows/run-kata-monitor-tests.yaml

@@ -0,0 +1,64 @@
+name: CI | Run kata-monitor tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-monitor:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        container_engine:
+          - crio
+          - containerd
+        # TODO: enable when https://github.com/kata-containers/kata-containers/issues/9853 is fixed
+        #include:
+        #  - container_engine: containerd
+        #    containerd_version: lts
+        exclude:
+          # TODO: enable with containerd when https://github.com/kata-containers/kata-containers/issues/9761 is fixed
+          - container_engine: containerd
+            vmm: qemu
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINER_ENGINE: ${{ matrix.container_engine }}
+      #CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/functional/kata-monitor/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/functional/kata-monitor/gha-run.sh install-kata kata-artifacts
+
+      - name: Run kata-monitor tests
+        run: bash tests/functional/kata-monitor/gha-run.sh run

.github/workflows/run-metrics.yaml

@@ -0,0 +1,94 @@
+name: CI | Run test metrics
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  setup-kata:
+    name: Kata Setup
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/metrics/gha-run.sh install-kata kata-artifacts
+
+  run-metrics:
+    needs: setup-kata
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance.
+      fail-fast: false
+      matrix:
+        vmm: ['clh', 'qemu']
+      max-parallel: 1
+    runs-on: metrics
+    env:
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - name: enabling the hypervisor
+        run: bash tests/metrics/gha-run.sh enabling-hypervisor
+
+      - name: run launch times test
+        run: bash tests/metrics/gha-run.sh run-test-launchtimes
+
+      - name: run memory foot print test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage
+
+      - name: run memory usage inside container test
+        run:  bash tests/metrics/gha-run.sh run-test-memory-usage-inside-container
+
+      - name: run blogbench test
+        run:  bash tests/metrics/gha-run.sh run-test-blogbench
+
+      - name: run tensorflow test
+        run:  bash tests/metrics/gha-run.sh run-test-tensorflow
+
+      - name: run fio test
+        run:  bash tests/metrics/gha-run.sh run-test-fio
+
+      - name: run iperf test
+        run:  bash tests/metrics/gha-run.sh run-test-iperf
+
+      - name: run latency test
+        run:  bash tests/metrics/gha-run.sh run-test-latency
+
+      - name: make metrics tarball ${{ matrix.vmm }}
+        run: bash tests/metrics/gha-run.sh make-tarball-results
+
+      - name: archive metrics results ${{ matrix.vmm }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-artifacts-${{ matrix.vmm }}
+          path: results-${{ matrix.vmm }}.tar.gz
+          retention-days: 1
+          if-no-files-found: error

.github/workflows/run-runk-tests.yaml

@@ -0,0 +1,48 @@
+name: CI | Run runk tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-runk:
+    # Skip runk tests as we have no maintainers. TODO: Decide when to remove altogether
+    if: false
+    runs-on: ubuntu-22.04
+    env:
+      CONTAINERD_VERSION: lts
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/runk/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v4
+        with:
+          name: kata-static-tarball-amd64${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/runk/gha-run.sh install-kata kata-artifacts
+
+      - name: Run runk tests
+        run: bash tests/integration/runk/gha-run.sh run

.github/workflows/shellcheck.yaml

@@ -0,0 +1,29 @@
+# https://github.com/marketplace/actions/shellcheck
+name: Check shell scripts
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/checkout@v4
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+

.github/workflows/snap-release.yaml

@@ -1,39 +0,0 @@
-name: Release Kata 2.x in snapcraft store
-on:
-  push:
-    tags:
-      - '2.*'
-jobs:
-  release-snap:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        uses: samuelmeuli/action-snapcraft@v1
-        with:
-          snapcraft_token: ${{ secrets.snapcraft_token }}
-
-      - name: Build snap
-        run: |
-          sudo apt-get install -y git git-extras
-          kata_url="https://github.com/kata-containers/kata-containers"
-          latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)
-          current_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          # Check semantic versioning format (x.y.z) and if the current tag is the latest tag
-          if echo "${current_version}" | grep -q "^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$" && echo -e "$latest_version\n$current_version" | sort -C -V; then
-            # Current version is the latest version, build it
-            snapcraft -d snap --destructive-mode
-          fi
-
-      - name: Upload snap
-        run: |
-          snap_version="$(echo ${GITHUB_REF} | cut -d/ -f3)"
-          snap_file="kata-containers_${snap_version}_amd64.snap"
-          # Upload the snap if it exists
-          if [ -f ${snap_file} ]; then
-            snapcraft upload --release=stable ${snap_file}
-          fi

.github/workflows/snap.yaml

@@ -1,27 +0,0 @@
-name: snap CI
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - edited
-
-jobs:
-  test:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Check out
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install Snapcraft
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: samuelmeuli/action-snapcraft@v1
-
-      - name: Build snap
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          snapcraft -d snap --destructive-mode

.github/workflows/stale.yaml

@@ -0,0 +1,17 @@
+name: 'Automatically close stale PRs'
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-pr-message: 'This PR has been opened without with no activity for 180 days. Comment on the issue otherwise it will be closed in 7 days'
+          days-before-pr-stale: 180
+          days-before-pr-close: 7
+          days-before-issue-stale: -1
+          days-before-issue-close: -1

.github/workflows/static-checks-self-hosted.yaml

@@ -0,0 +1,34 @@
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled # a workflow runs only when the 'ok-to-test' label is added
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+name: Static checks self-hosted
+jobs:
+  skipper:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  build-checks:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        instance:
+          - "ubuntu-22.04-arm"
+          - "s390x"
+          - "ppc64le"
+    uses: ./.github/workflows/build-checks.yaml
+    with:
+      instance: ${{ matrix.instance }}

.github/workflows/static-checks.yaml

@@ -6,91 +6,120 @@ on:
       - reopened
       - synchronize
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 name: Static checks
 jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
+  skipper:
+    uses: ./.github/workflows/gatekeeper-skipper.yaml
+    with:
+      commit-hash: ${{ github.event.pull_request.head.sha }}
+      target-branch: ${{ github.event.pull_request.base.ref }}
+
+  check-kernel-config-version:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
+    runs-on: ubuntu-22.04
     steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    # Check whether the vendored code is up-to-date & working as the first thing
-    - name: Check vendored code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
-    - name: Static Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
-    - name: Run Compiler Checks
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
-    - name: Run Unit Tests
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
-    - name: Run Unit Tests As Root User
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && sudo -E PATH="$PATH" make test
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Ensure the kernel config version has been updated
+        run: |
+          kernel_dir="tools/packaging/kernel/"
+          kernel_version_file="${kernel_dir}kata_config_version"
+          modified_files=$(git diff --name-only origin/"$GITHUB_BASE_REF"..HEAD)
+          if git diff --name-only origin/"$GITHUB_BASE_REF"..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+            echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
+            if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
+              echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
+            else
+              echo "Readme file changed, no need for kernel config version update."
+            fi
+            echo "Check passed"
+          fi
+
+  build-checks:
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
+    uses: ./.github/workflows/build-checks.yaml
+    with:
+      instance: ubuntu-22.04
+
+  build-checks-depending-on-kvm:
+    runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - runtime-rs
+        include:
+          - component: runtime-rs
+            command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test"
+          - component: runtime-rs
+            component-path: src/dragonball
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install system deps
+        run: |
+          sudo apt-get install -y build-essential musl-tools
+      - name: Install yq
+        run: |
+          sudo -E ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install rust
+        run: |
+          export PATH="$PATH:/usr/local/bin"
+          ./tests/install_rust.sh
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          export PATH="$PATH:${HOME}/.cargo/bin"
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  static-checks:
+    runs-on: ubuntu-22.04
+    needs: skipper
+    if: ${{ needs.skipper.outputs.skip_static != 'yes' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        cmd:
+          - "make static-checks"
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: ./src/github.com/${{ github.repository }}
+      - name: Install yq
+        run: |
+          cd "${GOPATH}/src/github.com/${{ github.repository }}"
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        run: |
+          cd "${GOPATH}/src/github.com/${{ github.repository }}"
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
+      - name: Install system dependencies
+        run: |
+          sudo apt-get -y install moreutils hunspell hunspell-en-gb hunspell-en-us pandoc
+      - name: Run check
+        run: |
+          export PATH="${PATH}:${GOPATH}/bin"
+          cd "${GOPATH}/src/github.com/${{ github.repository }}" && ${{ matrix.cmd }}

.gitignore

@@ -4,10 +4,15 @@
 **/*.rej
 **/target
 **/.vscode
+**/.idea
+**/.fleet
+**/*.swp
+**/*.swo
 pkg/logging/Cargo.lock
 src/agent/src/version.rs
 src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
 build
-
+src/tools/log-parser/kata-log-parser
+tools/packaging/static-build/agent/install_libseccomp.sh

CODEOWNERS

@@ -1,4 +1,4 @@
-# Copyright (c) 2019 Intel Corporation
+# Copyright (c) 2019-2023 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -9,4 +9,83 @@
 # Order in this file is important. Only the last match will be
 # used. See https://help.github.com/articles/about-code-owners/
 
-*.md    @kata-containers/documentation
+/CODEOWNERS			@kata-containers/codeowners
+
+VERSION				@kata-containers/release
+
+# The versions database needs careful handling
+versions.yaml			@kata-containers/release @kata-containers/ci @kata-containers/tests
+
+Makefile*			@kata-containers/build
+*.mak				@kata-containers/build
+*.mk				@kata-containers/build
+
+# Documentation related files could also appear anywhere
+# else in the repo.
+*.md				@kata-containers/documentation
+*.drawio			@kata-containers/documentation
+*.jpg				@kata-containers/documentation
+*.png				@kata-containers/documentation
+*.svg				@kata-containers/documentation
+
+*.bash				@kata-containers/shell
+*.sh				@kata-containers/shell
+**/completions/			@kata-containers/shell
+
+Dockerfile*			@kata-containers/docker
+
+/ci/				@kata-containers/ci
+
+*.bats				@kata-containers/tests
+/tests/				@kata-containers/tests
+
+*.rs				@kata-containers/rust
+*.go				@kata-containers/golang
+
+/utils/				@kata-containers/utils
+
+# FIXME: Maybe a new "protocol" team would be better?
+#
+# All protocol changes must be reviewed.
+# Note, we include all subdirs, including the vendor dir, as at present there are no .proto files
+# in the vendor dir. Later we may have to extend this matching rule if that changes.
+/src/libs/protocols/*.proto	@kata-containers/architecture-committee @kata-containers/builder @kata-containers/packaging
+
+# GitHub Actions
+/.github/workflows/		@kata-containers/action-admins @kata-containers/ci
+
+/ci/				@kata-containers/ci @kata-containers/tests
+/docs/				@kata-containers/documentation
+
+/src/agent/			@kata-containers/agent
+
+/src/runtime*/			@kata-containers/runtime
+
+/src/runtime/			@kata-containers/golang
+
+src/runtime-rs/			@kata-containers/rust
+src/libs/			@kata-containers/rust
+
+src/dragonball/			@kata-containers/dragonball
+
+/tools/osbuilder/		@kata-containers/builder
+/tools/packaging/		@kata-containers/packaging
+/tools/packaging/kernel/	@kata-containers/kernel
+/tools/packaging/kata-deploy/	@kata-containers/kata-deploy
+/tools/packaging/qemu/		@kata-containers/qemu
+/tools/packaging/release/	@kata-containers/release
+
+**/vendor/			@kata-containers/vendoring
+
+# Handle arch specific files last so they match more specifically than
+# the kernel packaging files.
+**/*aarch64*			@kata-containers/arch-aarch64
+**/*arm64*			@kata-containers/arch-aarch64
+
+**/*amd64*			@kata-containers/arch-amd64
+**/*x86-64*			@kata-containers/arch-amd64
+**/*x86_64*			@kata-containers/arch-amd64
+
+**/*ppc64*			@kata-containers/arch-ppc64le
+
+**/*s390x*			@kata-containers/arch-s390x

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Intel Corporation
+# Copyright (c) 2020-2023 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -6,25 +6,29 @@
 # List of available components
 COMPONENTS =
 
+COMPONENTS += libs
 COMPONENTS += agent
+COMPONENTS += dragonball
 COMPONENTS += runtime
+COMPONENTS += runtime-rs
 
 # List of available tools
 TOOLS =
 
 TOOLS += agent-ctl
-TOOLS += trace-forwarder
+TOOLS += kata-ctl
+TOOLS += log-parser
 TOOLS += runk
+TOOLS += trace-forwarder
 
-STANDARD_TARGETS = build check clean install test vendor
+STANDARD_TARGETS = build check clean install static-checks-build test vendor
+
+# Variables for the build-and-publish-kata-debug target
+KATA_DEBUG_REGISTRY ?= ""
+KATA_DEBUG_TAG ?= ""
 
 default: all
 
-all: logging-crate-tests build
-
-logging-crate-tests:
-	make -C src/libs/logging
-
 include utils.mk
 include ./tools/packaging/kata-deploy/local-build/Makefile
 
@@ -37,19 +41,19 @@ generate-protocols:
 	make -C src/agent generate-protocols
 
 # Some static checks rely on generated source files of components.
-static-checks: build
-	bash ci/static-checks.sh
+static-checks: static-checks-build
+	bash tests/static-checks.sh github.com/kata-containers/kata-containers
 
 docs-url-alive-check:
 	bash ci/docs-url-alive-check.sh
 
+build-and-publish-kata-debug:
+	bash tools/packaging/kata-debug/kata-debug-build-and-upload-payload.sh ${KATA_DEBUG_REGISTRY} ${KATA_DEBUG_TAG} 
+
 .PHONY: \
 	all \
-	binary-tarball \
+	kata-tarball \
+	install-tarball \
 	default \
-	install-binary-tarball \
-	logging-crate-tests \
 	static-checks \
 	docs-url-alive-check
-
-

README.md

@@ -1,4 +1,6 @@
-<img src="https://www.openstack.org/assets/kata/kata-vertical-on-white.png" width="150">
+<img src="https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/kata/SVG/kata-1.svg" width="900">
+
+[![CI | Publish Kata Containers payload](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/payload-after-push.yaml) [![Kata Containers Nightly CI](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml/badge.svg)](https://github.com/kata-containers/kata-containers/actions/workflows/ci-nightly.yaml)
 
 # Kata Containers
 
@@ -71,6 +73,7 @@ See the [official documentation](docs) including:
 - [Developer guide](docs/Developer-Guide.md)
 - [Design documents](docs/design)
   - [Architecture overview](docs/design/architecture)
+  - [Architecture 3.0 overview](docs/design/architecture_3.0/)
 
 ## Configuration
 
@@ -116,10 +119,11 @@ The table below lists the core parts of the project:
 | Component | Type | Description |
 |-|-|-|
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
+| [runtime-rs](src/runtime-rs) | core | The Rust version runtime. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
+| [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
-| [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
-| [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
+| [tests](tests) | tests | Excludes unit tests which live with the main code. |
 
 ### Additional components
 
@@ -130,18 +134,28 @@ The table below lists the remaining parts of the project:
 | [packaging](tools/packaging) | infrastructure | Scripts and metadata for producing packaged binaries<br/>(components, hypervisors, kernel and rootfs). |
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
+| [kata-debug](tools/packaging/kata-debug/README.md) | infrastructure | Utility tool to gather Kata Containers debug information from Kubernetes clusters. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
+| [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
 | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
-| [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
+| [`ci`](.github/workflows) | CI | Continuous Integration configuration files and scripts. |
+| [`ocp-ci`](ci/openshift-ci/README.md) | CI | Continuous Integration configuration for the OpenShift pipelines. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
+| [`Webhook`](tools/testing/kata-webhook/README.md) | utility | Example of a simple admission controller webhook to annotate pods with the Kata runtime class |
 
 ### Packaging and releases
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
-the [components](#components) section for further details.
+
+## General tests
+
+See the [tests documentation](tests/README.md).
+
+## Metrics tests
+
+See the [metrics documentation](tests/metrics/README.md).
 
 ## Glossary of Terms
 

VERSION

@@ -1 +1 @@
-2.5.0-alpha1
+3.14.0

ci/README.md

@@ -0,0 +1,400 @@
+# Kata Containers CI
+
+> [!WARNING]
+> While this project's CI has several areas for improvement, it is constantly
+> evolving. This document attempts to describe its current state, but due to
+> ongoing changes, you may notice some outdated information here. Feel free to
+> modify/improve this document as you use the CI and notice anything odd. The
+> community appreciates it!
+
+## Introduction
+
+The Kata Containers CI relies on [GitHub Actions][gh-actions], where the actions
+themselves can be found in the `.github/workflows` directory, and they may call
+helper scripts, which are located under the `tests` directory, to actually
+perform the tasks required for each test case.
+
+## The different workflows
+
+There are a few different sets of workflows that are running as part of our CI,
+and here we're going to cover the ones that are less likely to get rotten.  With
+this said, it's fair to advise that if the reader finds something that got
+rotten, opening an issue to the project pointing to the problem is a nice way to
+help, and providing a fix for the issue is a very encouraging way to help.
+
+### Jobs that run automatically when a PR is raised
+
+These are a bunch of tests that will automatically run as soon as a PR is
+opened, they're mostly running on "cost free" runners, and they do some
+pre-checks to evaluate that your PR may be okay to start getting reviewed.
+
+Mind, though, that the community expects the contributors to, at least, build
+their code before submitting a PR, which the community sees as a very fair
+request.
+
+Without getting into the weeds with details on this, those jobs are the ones
+responsible for ensuring that:
+
+- The commit message is in the expected format
+- There's no missing Developer's Certificate of Origin
+- Static checks are passing
+
+### Jobs that require a maintainer's approval to run
+
+There are some tests, and our so-called "CI".  These require a
+maintainer's approval to run as parts of those jobs will be running on "paid
+runners", which are currently using Azure infrastructure.
+
+Once a maintainer of the project gives "the green light" (currently by adding an
+`ok-to-test` label to the PR, soon to be changed to commenting "/test" as part
+of a PR review), the following tests will be executed:
+
+- Build all the components (runs on free cost runners, or bare-metal depending on the architecture)
+- Create a tarball with all the components (runs on free cost runners, or bare-metal depending on the architecture)
+- Create a kata-deploy payload with the tarball generated in the previous step (runs on free costs runner, or bare-metal depending on the architecture)
+- Run the following tests:
+  - Tests depending on the generated tarball
+    - Metrics (runs on bare-metal)
+    - `docker` (runs on cost free runners)
+    - `nerdctl` (runs on cost free runners)
+    - `kata-monitor` (runs on cost free runners)
+    - `cri-containerd` (runs on cost free runners)
+    - `nydus` (runs on cost free runners)
+    - `vfio` (runs on cost free runners)
+  - Tests depending on the generated kata-deploy payload
+    - kata-deploy (runs on cost free runners)
+      - Tests are performed using different "Kubernetes flavors", such as k0s, k3s, rke2, and Azure Kubernetes Service (AKS).
+    - Kubernetes (runs in Azure small and medium instances depending on what's required by each test, and on TEE bare-metal machines)
+      - Tests are performed with different runtime engines, such as CRI-O and containerd.
+      - Tests are performed with different snapshotters for containerd, namely OverlayFS and devmapper.
+      - Tests are performed with all the supported hypervisors, which are Cloud Hypervisor, Dragonball, Firecracker, and QEMU.
+
+For all the tests relying on Azure instances, real money is being spent, so the
+community asks for the maintainers to be mindful about those, and avoid abusing
+them to merely debug issues.
+
+## The different runners
+
+In the previous section we've mentioned using different runners, now in this section we'll go through each type of runner used.
+
+- Cost free runners:  Those are the runners provided by GitHub itself, and
+  those are fairly small machines with virtualization capabilities enabled.
+- Azure small instances: Those are runners which have virtualization
+  capabilities enabled, 2 CPUs, and 8GB of RAM.  These runners have a "-smaller"
+  suffix to their name.
+- Azure normal instances: Those are runners which have virtualization
+  capabilities enabled, 4 CPUs, and 16GB of RAM.  These runners are usually
+  `garm` ones with no "-smaller" suffix.
+- Bare-metal runners: Those are runners provided by community contributors,
+  and they may vary in architecture, size and virtualization capabilities.
+  Builder runners don't actually require any virtualization capabilities, while
+  runners which will be actually performing the tests must have virtualization
+  capabilities and a reasonable amount for CPU and RAM available (at least
+  matching the Azure normal instances).
+
+## Adding new tests
+
+Before someone decides to add a new test, we strongly recommend them to go
+through [GitHub Actions Documentation][gh-actions],
+which will provide you a very sensible background on how to read and understand
+current tests we have, and also become familiar with how to write a new test.
+
+On the Kata Containers land, there are basically two sets of tests: "standalone"
+and "part of something bigger".
+
+The "standalone" tests, for example the commit message check, won't be covered
+here as they're better covered by the GitHub Actions documentation pasted above.
+
+The "part of something bigger" is the more complicated one and not so
+straightforward to add, so we'll be focusing our efforts on describing the
+addition of those.
+
+> [!NOTE]
+> TODO: Currently, this document refers to "tests" when it actually means the
+> jobs (or workflows) of GitHub. In an ideal world, except in some specific cases,
+> new tests should be added without the need to add new workflows. In the
+> not-too-distant future (hopefully), we will improve the workflows to support
+> this.
+
+### Adding a new test that's "part of something bigger"
+
+The first important thing here is to align expectations, and we must say that
+the community strongly prefers receiving tests that already come with:
+
+- Instructions how to run them
+- A proven run where it's passing
+
+There are several ways to achieve those two requirements, and an example of that
+can be seen in PR #8115.
+
+With the expectations aligned, adding a test consists in:
+
+- Adding a new yaml file for your test, and ensure it's called from the
+  "bigger" yaml. See the [Kata Monitor test example][monitor-ex01].
+
+- Adding the helper scripts needed for your test to run. Again, use the [Kata Monitor script as example][monitor-ex02].
+
+Following those examples, the community advice during the review, and even
+asking the community directly on Slack are the best ways to get your test
+accepted.
+
+## Required tests
+
+In our CI we have two categories of jobs - required and non-required:
+- Required jobs need to all pass for a PR to be merged normally and
+should cover all the core features on Kata Containers that we want to
+ensure don't have regressions.
+- The non-required jobs are for unstable tests, or for features that
+are experimental and not-fully supported. We'd like those tests to also
+pass on all PRs ideally, but don't block merging if they don't as it's
+not necessarily an indication of the PR code causing regressions.
+
+### Transitioning between required and non-required status
+
+Required jobs that fail block merging of PRs, so we want to ensure that
+jobs are stable and maintained before we make them required.
+
+The [Kata Containers CI Dashboard](https://kata-containers.github.io/)
+is a useful resource to check when collecting evidence of job stability.
+At time of writing it reports the last ten days of Kata CI nightly test
+results for each job. This isn't perfect as it doesn't currently capture
+results on PRs, but is a good guideline for stability.
+
+> [!NOTE]
+> Below are general guidelines about jobs being marked as
+> required/non-required, but they are subject to change and the Kata
+> Architecture Committee may overrule these guidelines at their
+> discretion.
+
+#### Initial marking as required
+
+For new jobs, or jobs that haven't been marked as required recently,
+the criteria to be initially marked as required is ten days
+of passing tests, with no relevant PR failures reported in that time.
+Required jobs also need one or more nominated maintainers that are
+responsible for the stability of their jobs.
+
+> [!NOTE]
+> We don't currently have a good place to record the job maintainers, but
+> once we have this, the intention is to show it on the CI Dashboard so
+> people can find the contact easily.
+
+#### Expectation of required job maintainers
+
+Due to the nature of the Kata Containers community having contributors
+spread around the world, required jobs being blocked due to infrastructure,
+or test issues can have a big impact on work. As such, the expectation is
+that when a problem with a required job is noticed/reported, the maintainers
+have one working day to acknowledge the issue, perform an initial
+investigation and then either fix it, or get it marked as non-required
+whilst the investigation and/or fix it done.
+
+### Re-marking of required status
+
+Once a job has been removed from the required list, it requires two
+consecutive successful nightly test runs before being made required
+again.
+
+## Running tests
+
+### Running the tests as part of the CI
+
+If you're a maintainer of the project, you'll be able to kick in the tests by
+yourself.  With the current approach, you just need to add the `ok-to-test`
+label and the tests will automatically start.  We're moving, though, to use a
+`/test` command as part of a GitHub review comment, which will simplify this
+process.
+
+If you're not a maintainer, please, send a message on Slack or wait till one of
+the maintainers reviews your PR.  Maintainers will then kick in the tests on
+your behalf.
+
+In case a test fails and there's the suspicion it happens due to flakiness in
+the test itself, please, create an issue for us, and then re-run (or asks
+maintainers to re-run) the tests following these steps:
+
+- Locate which tests is failing
+- Click in "details"
+- In the top right corner, click in "Re-run jobs"
+- And then in "Re-run failed jobs"
+- And finally click in the green "Re-run jobs" button
+
+> [!NOTE]
+> TODO: We need figures here
+
+### Running the tests locally
+
+In this section, aligning expectations is also something very important, as one
+will not be able to run the tests exactly in the same way the tests are running
+in the CI, as one most likely won't have access to an Azure subscription.
+However, we're trying our best here to provide you with instructions on how to
+run the tests in an environment that's "close enough" and will help you to debug
+issues you find with the current tests, or even provide a proof-of-concept to
+the new test you're trying to add.
+
+The basic steps, which we will cover in details down below are:
+
+ 1. Create a VM matching the configuration of the target runner
+ 2. Generate the artifacts you'll need for the test, or download them from a
+    current failed run
+ 3. Follow the steps provided in the action itself to run the tests.
+
+Although the general overview looks easy, we know that some tricks need to be
+shared, and we'll go through the general process of debugging one non-Kubernetes
+and one Kubernetes specific test for educational purposes.
+
+One important thing to note is that "Create a VM" can be done in innumerable
+different ways, using the tools of your choice.  For the sake of simplicity on
+this guide, we'll be using `kcli`, which we strongly recommend in case you're a
+non-experienced user, and happen to be developing on a Linux box.
+
+For both non-Kubernetes and Kubernetes cases, we'll be using PR #8070 as an
+example, which at the time this document is being written serves us very well
+the purpose, as you can see that we have `nerdctl` and Kubernetes tests failing.
+
+## Debugging tests
+
+### Debugging a non Kubernetes test
+
+As shown above, the `nerdctl` test is failing.
+
+As a developer you can go ahead to the details of the job, and expand the job
+that's failing in order to gather more information.
+
+But when that doesn't help, we need to set up our own environment to debug
+what's going on.
+
+Taking a look at the `nerdctl` test, which is located here, you can easily see
+that it runs-on a `garm-ubuntu-2304-smaller` virtual machine.
+
+The important parts to understand are `ubuntu-2304`, which is the OS where the
+test is running on; and "smaller", which means we're running it on a machine
+with 2 CPUs and 8GB of RAM.
+
+With this information, we can go ahead and create a similar VM locally using `kcli`.
+
+```bash
+$ sudo kcli create vm -i ubuntu2304 -P disks=[60] -P numcpus=2 -P memory=8192 -P cpumodel=host-passthrough debug-nerdctl-pr8070
+```
+
+In order to run the tests, you'll need the "kata-tarball" artifacts, which you
+can build your own using "make kata-tarball" (see below), or simply get them
+from the PR where the tests failed.  To download them, click on the "Summary"
+button that's on the top left corner, and then scroll down till you see the
+artifacts, as shown below.
+
+Unfortunately GitHub doesn't give us a link that we can download those from
+inside the VM, but we can download them on our local box, and then `scp` the
+tarball to the newly created VM that will be used for debugging purposes.
+
+> [!NOTE]
+> Those artifacts are only available (for 15 days) when all jobs are finished.
+
+Once you have the `kata-static.tar.xz` in your VM, you can login to the VM with
+`kcli ssh debug-nerdctl-pr8070`, go ahead and then clone your development branch
+
+```bash
+$ git clone --branch feat_add-fc-runtime-rs https://github.com/nubificus/kata-containers
+```
+
+Add the upstream as a remote, set up your git, and rebase your branch atop of the upstream main one
+
+```bash
+$ git remote add upstream https://github.com/kata-containers/kata-containers
+$ git remote update
+$ git config --global user.email "you@example.com"
+$ git config --global user.name "Your Name"
+$ git rebase upstream/main
+```
+
+Now copy the `kata-static.tar.xz` into your `kata-containers/kata-artifacts` directory
+
+```bash
+$ mkdir kata-artifacts
+$ cp ../kata-static.tar.xz kata-artifacts/
+```
+
+> [!NOTE]
+> If you downloaded the .zip from GitHub you need to uncompress first to see `kata-static.tar.xz`
+
+And finally run the tests following what's in the yaml file for the test you're
+debugging.
+
+In our case, the `run-nerdctl-tests-on-garm.yaml`.
+
+When looking at the file you'll notice that some environment variables are set,
+such as `KATA_HYPERVISOR`, and should be aware that, for this particular example,
+the important steps to follow are:
+
+Install the dependencies
+Install kata
+Run the tests
+
+Let's now run the steps mentioned above exporting the expected environment variables
+
+```bash
+$ export KATA_HYPERVISOR=dragonball
+$ bash ./tests/integration/nerdctl/gha-run.sh install-dependencies
+$ bash ./tests/integration/nerdctl/gha-run.sh install-kata
+$ bash tests/integration/nerdctl/gha-run.sh run
+```
+
+And with this you should've been able to reproduce exactly the same issue found
+in the CI, and from now on you can build your own code, use your own binaries,
+and have fun debugging and hacking!
+
+### Debugging a Kubernetes test
+
+Steps for debugging the Kubernetes tests are very similar to the ones for
+debugging non-Kubernetes tests, with the caveat that what you'll need, this
+time, is not the `kata-static.tar.xz` tarball, but rather a payload to be used
+with kata-deploy.
+
+In order to generate your own kata-deploy image you can generate your own
+`kata-static.tar.xz` and then take advantage of the following script.  Be aware
+that the image generated and uploaded must be accessible by the VM where you'll
+be performing your tests.
+
+In case you want to take advantage of the payload that was already generated
+when you faced the CI failure, which is considerably easier, take a look at the
+failed job, then click in "Deploy Kata" and expand the "Final kata-deploy.yaml
+that is used in the test" section.  From there you can see exactly what you'll
+have to use when deploying kata-deploy in your local cluster.
+
+> [!NOTE]
+> TODO: WAINER TO FINISH THIS PART BASED ON HIS PR TO RUN A LOCAL CI
+
+## Adding new runners
+
+Any admin of the project is able to add or remove GitHub runners, and those are
+the folks you should rely on.
+
+If you need a new runner added, please, tag @ac in the Kata Containers slack,
+and someone from that group will be able to help you.
+
+If you're part of that group and you're looking for information on how to help
+someone, this is simple, and must be done in private. Basically what you have to
+do is:
+
+- Go to the kata-containers/kata-containers repo
+- Click on the Settings button, located in the top right corner
+- On the left panel, under "Code and automation", click on "Actions"
+- Click on "Runners"
+
+If you want to add a new self-hosted runner:
+
+- In the top right corner there's a green button called "New self-hosted runner"
+
+If you want to remove a current self-hosted runner:
+
+- For each runner there's a "..." menu, where you can just click and the
+  "Remove runner" option will show up
+
+## Known limitations
+
+As the GitHub actions are structured right now we cannot: Test the addition of a
+GitHub action that's not triggered by a pull_request event as part of the PR.
+
+[gh-actions]: https://docs.github.com/en/actions
+[monitor-ex01]: https://github.com/kata-containers/kata-containers/commit/a3fb067f1bccde0cbd3fd4d5de12dfb3d8c28b60
+[monitor-ex02]: https://github.com/kata-containers/kata-containers/commit/489caf1ad0fae27cfd00ba3c9ed40e3d512fa492

ci/darwin-test.sh

@@ -11,10 +11,10 @@ runtimedir=$cidir/../src/runtime
 
 build_working_packages() {
 	# working packages:
-	device_api=$runtimedir/virtcontainers/device/api
-	device_config=$runtimedir/virtcontainers/device/config
-	device_drivers=$runtimedir/virtcontainers/device/drivers
-	device_manager=$runtimedir/virtcontainers/device/manager
+	device_api=$runtimedir/pkg/device/api
+	device_config=$runtimedir/pkg/device/config
+	device_drivers=$runtimedir/pkg/device/drivers
+	device_manager=$runtimedir/pkg/device/manager
 	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
 	utils_pkg_dir=$runtimedir/virtcontainers/utils
 

ci/docs-url-alive-check.sh

@@ -7,6 +7,6 @@
 set -e
 
 cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+source "${cidir}/../tests/common.bash"
 
 run_docs_url_alive_check

ci/gh-util.sh

@@ -0,0 +1,182 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Intel Corporation
+# Copyright (c) 2024 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o errtrace
+set -o nounset
+set -o pipefail
+
+[ -n "${DEBUG:-}" ] && set -o xtrace
+
+script_name=${0##*/}
+
+#---------------------------------------------------------------------
+
+die()
+{
+    echo >&2 "$*"
+    exit 1
+}
+
+usage()
+{
+    cat <<EOF
+Usage: $script_name [OPTIONS] [command] [arguments]
+
+Description: Utility to expand the abilities of the GitHub CLI tool, gh.
+
+Command descriptions:
+
+  list-issues-for-pr     List issues linked to a PR.
+  list-labels-for-issue  List labels, in json format for an issue
+
+Commands and arguments:
+
+  list-issues-for-pr <pr>
+  list-labels-for-issue <issue>
+
+Options:
+
+ -h                 Show this help statement.
+ -r <owner/repo>    Optional <org/repo> specification. Default: 'kata-containers/kata-containers'
+
+Examples:
+
+- List issues for a Pull Request 123 in kata-containers/kata-containers repo
+
+  $ $script_name list-issues-for-pr 123
+EOF
+}
+
+list_issues_for_pr()
+{
+    local pr="${1:-}"
+    local repo="${2:-kata-containers/kata-containers}"
+
+    [ -z "$pr" ] && die "need PR"
+
+    local commits=$(gh pr view ${pr} --repo ${repo} --json commits --jq .commits[].messageBody)
+
+    [ -z "$commits" ] && die "cannot determine commits for PR $pr"
+
+    # Extract the issue number(s) from the commits.
+    #
+    # This needs to be careful to take account of lines like this:
+    #
+    # fixes 99
+    # fixes: 77
+    # fixes #123.
+    # Fixes: #1, #234, #5678.
+    #
+    # Note the exclusion of lines starting with whitespace which is
+    # specifically to ignore vendored git log comments, which are whitespace
+    # indented and in the format:
+    #
+    #     "<git-commit> <git-commit-msg>"
+    #
+    local issues=$(echo "$commits" |\
+        grep -v -E "^( |	)" |\
+        grep -i -E "fixes:* *(#*[0-9][0-9]*)" |\
+        tr ' ' '\n' |\
+        grep "[0-9][0-9]*" |\
+        sed 's/[.,\#]//g' |\
+        sort -nu || true)
+
+    [ -z "$issues" ] && die "cannot determine issues for PR $pr"
+
+    echo "# Issues linked to PR"
+    echo "#"
+    echo "# Fields: issue_number"
+
+    local issue
+    echo "$issues"|while read issue
+    do
+        printf "%s\n" "$issue"
+    done
+}
+
+list_labels_for_issue()
+{
+    local issue="${1:-}"
+
+    [ -z "$issue" ] && die "need issue number"
+
+    local labels=$(gh issue view ${issue} --repo kata-containers/kata-containers --json labels)
+
+    [ -z "$labels" ] && die "cannot determine labels for issue $issue"
+
+    printf "$labels"
+}
+
+setup()
+{
+    for cmd in gh jq
+    do
+        command -v "$cmd" &>/dev/null || die "need command: $cmd"
+    done
+}
+
+handle_args()
+{
+    setup
+
+    local show_all="false"
+    local opt
+
+    while getopts "ahr:" opt "$@"
+    do
+        case "$opt" in
+            a) show_all="true" ;;
+            h) usage && exit 0 ;;
+            r) repo="${OPTARG}" ;;
+        esac
+    done
+
+    shift $(($OPTIND - 1))
+
+    local repo="${repo:-kata-containers/kata-containers}"
+    local cmd="${1:-}"
+
+    case "$cmd" in
+        list-issues-for-pr) ;;
+        list-labels-for-issue) ;;
+
+        "") usage && exit 0 ;;
+        *) die "invalid command: '$cmd'" ;;
+    esac
+
+    # Consume the command name
+    shift
+
+    local issue=""
+    local pr=""
+
+    case "$cmd" in
+        list-issues-for-pr)
+            pr="${1:-}"
+
+            list_issues_for_pr "$pr" "${repo}"
+            ;;
+
+        list-labels-for-issue)
+            issue="${1:-}"
+
+            list_labels_for_issue "$issue"
+            ;;
+
+        *) die "impossible situation: cmd: '$cmd'" ;;
+    esac
+
+    exit 0
+}
+
+main()
+{
+    handle_args "$@"
+}
+
+main "$@"

ci/go-test.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2020 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-run_go_test

ci/install_go.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-clone_tests_repo
-
-new_goroot=/usr/local/go
-
-pushd "${tests_repo_dir}"
-# Force overwrite the current version of golang
-[ -z "${GOROOT}" ] || rm -rf "${GOROOT}"
-.ci/install_go.sh -p -f -d "$(dirname ${new_goroot})"
-[ -z "${GOROOT}" ] || sudo ln -sf "${new_goroot}" "${GOROOT}"
-go version
-popd

ci/install_libseccomp.sh

@@ -7,12 +7,10 @@
 
 set -o errexit
 
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+script_name="$(basename "${BASH_SOURCE[0]}")"
 
-clone_tests_repo
-
-source "${tests_repo_dir}/.ci/lib.sh"
+source "${script_dir}/../tests/common.bash"
 
 # The following variables if set on the environment will change the behavior
 # of gperf and libseccomp configure scripts, that may lead this script to
@@ -23,88 +21,91 @@ arch=${ARCH:-$(uname -m)}
 workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 
 # Variables for libseccomp
-# Currently, specify the libseccomp version directly without using `versions.yaml`
-# because the current Snap workflow is incomplete.
-# After solving the issue, replace this code by using the `versions.yaml`.
-# libseccomp_version=$(get_version "externals.libseccomp.version")
-# libseccomp_url=$(get_version "externals.libseccomp.url")
-libseccomp_version="2.5.1"
-libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_version="${LIBSECCOMP_VERSION:-""}"
+if [ -z "${libseccomp_version}" ]; then
+	libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version")
+fi
+libseccomp_url="${LIBSECCOMP_URL:-""}"
+if [ -z "${libseccomp_url}" ]; then
+	libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url")
+fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
 cflags="-O2"
 
 # Variables for gperf
-# Currently, specify the gperf version directly without using `versions.yaml`
-# because the current Snap workflow is incomplete.
-# After solving the issue, replace this code by using the `versions.yaml`.
-# gperf_version=$(get_version "externals.gperf.version")
-# gperf_url=$(get_version "externals.gperf.url")
-gperf_version="3.1"
-gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_version="${GPERF_VERSION:-""}"
+if [ -z "${gperf_version}" ]; then
+	gperf_version=$(get_from_kata_deps ".externals.gperf.version")
+fi
+gperf_url="${GPERF_URL:-""}"
+if [ -z "${gperf_url}" ]; then
+	gperf_url=$(get_from_kata_deps ".externals.gperf.url")
+fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"
 
 # We need to build the libseccomp library from sources to create a static library for the musl libc.
 # However, ppc64le and s390x have no musl targets in Rust. Hence, we do not set cflags for the musl libc.
 if ([ "${arch}" != "ppc64le" ] && [ "${arch}" != "s390x" ]); then
-    # Set FORTIFY_SOURCE=1 because the musl-libc does not have some functions about FORTIFY_SOURCE=2
-    cflags="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -O2"
+	# Set FORTIFY_SOURCE=1 because the musl-libc does not have some functions about FORTIFY_SOURCE=2
+	cflags="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -O2"
 fi
 
 die() {
-    msg="$*"
-    echo "[Error] ${msg}" >&2
-    exit 1
+	msg="$*"
+	echo "[Error] ${msg}" >&2
+	exit 1
 }
 
 finish() {
-    rm -rf "${workdir}"
+	rm -rf "${workdir}"
 }
 
 trap finish EXIT
 
 build_and_install_gperf() {
-    echo "Build and install gperf version ${gperf_version}"
-    mkdir -p "${gperf_install_dir}"
-    curl -sLO "${gperf_tarball_url}"
-    tar -xf "${gperf_tarball}"
-    pushd "gperf-${gperf_version}"
-    # Unset $CC for configure, we will always use native for gperf
-    CC= ./configure --prefix="${gperf_install_dir}"
-    make
-    make install
-    export PATH=$PATH:"${gperf_install_dir}"/bin
-    popd
-    echo "Gperf installed successfully"
+	echo "Build and install gperf version ${gperf_version}"
+	mkdir -p "${gperf_install_dir}"
+	curl -sLO "${gperf_tarball_url}"
+	tar -xf "${gperf_tarball}"
+	pushd "gperf-${gperf_version}"
+	# Unset $CC for configure, we will always use native for gperf
+	CC= ./configure --prefix="${gperf_install_dir}"
+	make
+	make install
+	export PATH=$PATH:"${gperf_install_dir}"/bin
+	popd
+	echo "Gperf installed successfully"
 }
 
 build_and_install_libseccomp() {
-    echo "Build and install libseccomp version ${libseccomp_version}"
-    mkdir -p "${libseccomp_install_dir}"
-    curl -sLO "${libseccomp_tarball_url}"
-    tar -xf "${libseccomp_tarball}"
-    pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
-    make
-    make install
-    popd
-    echo "Libseccomp installed successfully"
+	echo "Build and install libseccomp version ${libseccomp_version}"
+	mkdir -p "${libseccomp_install_dir}"
+	curl -sLO "${libseccomp_tarball_url}"
+	tar -xf "${libseccomp_tarball}"
+	pushd "libseccomp-${libseccomp_version}"
+	[ "${arch}" == $(uname -m) ] && cc_name="" || cc_name="${arch}-linux-gnu-gcc"
+	CC=${cc_name} ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
+	make
+	make install
+	popd
+	echo "Libseccomp installed successfully"
 }
 
 main() {
-    local libseccomp_install_dir="${1:-}"
-    local gperf_install_dir="${2:-}"
+	local libseccomp_install_dir="${1:-}"
+	local gperf_install_dir="${2:-}"
 
-    if [ -z "${libseccomp_install_dir}" ] || [ -z "${gperf_install_dir}" ]; then
-        die "Usage: ${0} <libseccomp-install-dir> <gperf-install-dir>"
-    fi
+	if [ -z "${libseccomp_install_dir}" ] || [ -z "${gperf_install_dir}" ]; then
+		die "Usage: ${0} <libseccomp-install-dir> <gperf-install-dir>"
+	fi
 
-    pushd "$workdir"
-    # gperf is required for building the libseccomp.
-    build_and_install_gperf
-    build_and_install_libseccomp
-    popd
+	pushd "$workdir"
+	# gperf is required for building the libseccomp.
+	build_and_install_gperf
+	build_and_install_libseccomp
+	popd
 }
 
 main "$@"

ci/install_rust.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) 2019 Ant Financial
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-clone_tests_repo
-
-pushd ${tests_repo_dir}
-.ci/install_rust.sh ${1:-}
-popd

ci/install_vc.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-vcdir="${cidir}/../src/runtime/virtcontainers/"
-source "${cidir}/lib.sh"
-export CI_JOB="${CI_JOB:-default}"
-
-clone_tests_repo
-
-if [ "${CI_JOB}" != "PODMAN" ]; then
-	echo "Install virtcontainers"
-	make -C "${vcdir}" && chronic sudo make -C "${vcdir}" install
-fi

ci/install_yq.sh

@@ -5,6 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+[ -n "$DEBUG" ] && set -o xtrace
+
 # If we fail for any reason a message will be displayed
 die() {
 	msg="$*"
@@ -12,21 +14,48 @@ die() {
 	exit 1
 }
 
+function verify_yq_exists() {
+	local yq_path=$1
+	local yq_version=$2
+	local expected="yq (https://github.com/mikefarah/yq/) version $yq_version"
+	if [ -x  "${yq_path}" ] && [ "$($yq_path --version)"X == "$expected"X ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 # Install the yq yaml query package from the mikefarah github repo
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
 	local yq_pkg="github.com/mikefarah/yq"
-	local yq_version=3.4.1
+	local yq_version=v4.44.5
+	local precmd=""
+	local yq_path=""
 	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 
-	if [ "${INSTALL_IN_GOPATH}"  == "true" ];then
+	if [ "${INSTALL_IN_GOPATH}" == "true" ]; then
 		GOPATH=${GOPATH:-${HOME}/go}
 		mkdir -p "${GOPATH}/bin"
-		local yq_path="${GOPATH}/bin/yq"
+		yq_path="${GOPATH}/bin/yq"
 	else
 		yq_path="/usr/local/bin/yq"
 	fi
-	[ -x  "${yq_path}" ] && [ "`${yq_path} --version`"X == "yq version ${yq_version}"X ] && return
+	if verify_yq_exists "$yq_path" "$yq_version"; then
+		echo "yq is already installed in correct version"
+		return
+	fi
+	if [ "${yq_path}" == "/usr/local/bin/yq" ]; then
+		# Check if we need sudo to install yq
+		if [ ! -w "/usr/local/bin" ]; then
+			# Check if we have sudo privileges
+			if ! sudo -n true 2>/dev/null; then
+				die "Please provide sudo privileges to install yq"
+			else
+				precmd="sudo"
+			fi
+		fi
+	fi
 
 	read -r -a sysInfo <<< "$(uname -sm)"
 
@@ -43,6 +72,19 @@ function install_yq() {
 	"aarch64")
 		goarch=arm64
 		;;
+	"arm64")
+		# If we're on an apple silicon machine, just assign amd64. 
+		# The version of yq we use doesn't have a darwin arm build, 
+		# but Rosetta can come to the rescue here.
+		if [ $goos == "Darwin" ]; then 
+			goarch=amd64
+		else 
+			goarch=arm64
+		fi
+		;;
+	"riscv64")
+		goarch=riscv64
+		;;
 	"ppc64le")
 		goarch=ppc64le
 		;;
@@ -64,10 +106,10 @@ function install_yq() {
 	fi
 
 	## NOTE: ${var,,} => gives lowercase value of var
-	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos,,}_${goarch}"
-	curl -o "${yq_path}" -LSsf "${yq_url}"
+	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
+	${precmd} curl -o "${yq_path}" -LSsf "${yq_url}"
 	[ $? -ne 0 ] && die "Download ${yq_url} failed"
-	chmod +x "${yq_path}"
+	${precmd} chmod +x "${yq_path}"
 
 	if ! command -v "${yq_path}" >/dev/null; then
 		die "Cannot not get ${yq_path} executable"

ci/lib.sh

@@ -1,55 +0,0 @@
-#
-# Copyright (c) 2018 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -o nounset
-
-export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
-export tests_repo_dir="$GOPATH/src/$tests_repo"
-export branch="${target_branch:-main}"
-
-# Clones the tests repository and checkout to the branch pointed out by
-# the global $branch variable.
-# If the clone exists and `CI` is exported then it does nothing. Otherwise
-# it will clone the repository or `git pull` the latest code.
-#
-clone_tests_repo()
-{
-	if [ -d "$tests_repo_dir" ]; then
-		[ -n "${CI:-}" ] && return
-		pushd "${tests_repo_dir}"
-		git checkout "${branch}"
-		git pull
-		popd
-	else
-		git clone -q "https://${tests_repo}" "$tests_repo_dir"
-		pushd "${tests_repo_dir}"
-		git checkout "${branch}"
-		popd
-	fi
-}
-
-run_static_checks()
-{
-	clone_tests_repo
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
-}
-
-run_go_test()
-{
-	clone_tests_repo
-	bash "$tests_repo_dir/.ci/go-test.sh"
-}
-
-run_docs_url_alive_check()
-{
-	clone_tests_repo
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers"
-}

ci/openshift-ci/README.md

@@ -0,0 +1,149 @@
+OpenShift CI
+============
+
+This directory contains scripts used by
+[the OpenShift CI](https://github.com/openshift/release/tree/master/ci-operator/config/kata-containers/kata-containers)
+pipelines to monitor selected functional tests on OpenShift.
+There are 2 pipelines, history and logs can be accessed here:
+
+* [main - currently supported OCP](https://prow.ci.openshift.org/job-history/gs/origin-ci-test/logs/periodic-ci-kata-containers-kata-containers-main-e2e-tests)
+* [next - currently under development OCP](https://prow.ci.openshift.org/job-history/gs/origin-ci-test/logs/periodic-ci-kata-containers-kata-containers-main-next-e2e-tests)
+
+
+Running openshift-tests on OCP with kata-containers manually
+============================================================
+
+To run openshift-tests (or other suites) with kata-containers one can use
+the kata-webhook. To deploy everything you can mimic the CI pipeline by:
+
+```bash
+#!/bin/bash -e
+# Setup your kubectl and check it's accessible by
+kubectl nodes
+# Deploy kata (set KATA_DEPLOY_IMAGE to override the default kata-deploy-ci:latest image)
+./test.sh
+# Deploy the webhook
+KATA_RUNTIME=kata-qemu cluster/deploy_webhook.sh
+```
+
+This should ensure kata-containers as well as kata-webhook are installed and
+working. Before running the openshift-tests it's (currently) recommended to
+ignore some security features by:
+
+```bash
+#!/bin/bash -e
+oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
+oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts
+oc label --overwrite ns default pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+```
+
+Now you should be ready to run the openshift-tests. Our CI only uses a subset
+of tests, to get the current ``TEST_SKIPS`` see
+[the pipeline config](https://github.com/openshift/release/tree/master/ci-operator/config/kata-containers/kata-containers).
+Following steps require the [openshift tests](https://github.com/openshift/origin)
+being cloned and built in the current directory:
+
+```bash
+#!/bin/bash -e
+# Define tests to be skipped (see the pipeline config for the current version)
+TEST_SKIPS="\[sig-node\] Security Context should support seccomp runtime/default\|\[sig-node\] Variable Expansion should allow substituting values in a volume subpath\|\[k8s.io\] Probing container should be restarted with a docker exec liveness probe with timeout\|\[sig-node\] Pods Extended Pod Container lifecycle evicted pods should be terminal\|\[sig-node\] PodOSRejection \[NodeConformance\] Kubelet should reject pod when the node OS doesn't match pod's OS\|\[sig-network\].*for evicted pods\|\[sig-network\].*HAProxy router should override the route\|\[sig-network\].*HAProxy router should serve a route\|\[sig-network\].*HAProxy router should serve the correct\|\[sig-network\].*HAProxy router should run\|\[sig-network\].*when FIPS.*the HAProxy router\|\[sig-network\].*bond\|\[sig-network\].*all sysctl on whitelist\|\[sig-network\].*sysctls should not affect\|\[sig-network\] pods should successfully create sandboxes by adding pod to network"
+# Get the list of tests to be executed
+TESTS="$(./openshift-tests run --dry-run --provider "${TEST_PROVIDER}" "${TEST_SUITE}")"
+# Store the list of tests in /tmp/tsts file
+echo "${TESTS}" | grep -v "$TEST_SKIPS" > /tmp/tsts
+# Remove previously-existing temporarily files as well as previous results
+OUT=RESULTS/tmp
+rm -Rf /tmp/*test* /tmp/e2e-*
+rm -R $OUT
+mkdir -p $OUT
+# Run the tests ignoring the monitor health checks
+./openshift-tests run --provider azure -o "$OUT/job.log" --junit-dir "$OUT" --file /tmp/tsts --max-parallel-tests 5 --cluster-stability Disruptive --run '^\[sig-node\].*|^\[sig-network\]'
+```
+
+[!NOTE]
+Note we are ignoring the cluster stability checks because our public cloud is
+not that stable and running with VMs instead of containers results in minor
+stability issues. Some of the old monitor stability tests do not reflect
+the ``--cluster-stability`` setting, one should simply ignore these. If you
+get a message like ``invariant was violated`` or ``error: failed due to a
+MonitorTest failure``, it's usually an indication that only those kind of
+tests failed but the real tests passed. See
+[wrapped-openshift-tests.sh](https://github.com/openshift/release/blob/master/ci-operator/config/kata-containers/kata-containers/wrapped-openshift-tests.sh)
+for details how our pipeline deals with that.
+
+[!TIP]
+To compare multiple results locally one can use
+[junit2html](https://github.com/inorton/junit2html) tool.
+
+
+Best-effort kata-containers cleanup
+===================================
+
+If you need to cleanup the cluster after testing, you can use the
+``cleanup.sh`` script from the current directory. It tries to delete all
+resources created by ``test.sh`` as well as ``cluster/deploy_webhook.sh``
+ignoring all failures. The primary purpose of this script is to allow
+soft-cleanup after deployment to test different versions without
+re-provisioning everything.
+
+[!WARNING]
+Do not rely on this script in production, return codes are not checked!**
+
+
+Bisecting e2e tests failures
+============================
+
+Let's say the OCP pipeline passed running with
+``quay.io/kata-containers/kata-deploy-ci:kata-containers-d7afd31fd40e37a675b25c53618904ab57e74ccd-amd64``
+but failed running with
+``quay.io/kata-containers/kata-deploy-ci:kata-containers-9f512c016e75599a4a921bd84ea47559fe610057-amd64``
+and you'd like to know which PR caused the regression. You can either run with
+all the 60 tags between or you can utilize the [bisecter](https://github.com/ldoktor/bisecter)
+to optimize the number of steps in between.
+
+Before running the bisection you need a reproducer script. Sample one called
+``sample-test-reproducer.sh`` is provided in this directory but you might
+want to copy and modify it, especially:
+
+* ``OCP_DIR`` - directory where your openshift/release is located (can be exported)
+* ``E2E_TEST`` - openshift-test(s) to be executed (can be exported)
+* behaviour of SETUP (returning 125 skips the current image tag, returning
+  >=128 interrupts the execution, everything else reports the tag as failure
+* what should be executed (perhaps running the setup is enough for you or
+  you might want to be looking for specific failures...)
+* use ``timeout`` to interrupt execution in case you know things should be faster
+
+Executing that script with the GOOD commit should pass
+``./sample-test-reproducer.sh quay.io/kata-containers/kata-deploy-ci:kata-containers-d7afd31fd40e37a675b25c53618904ab57e74ccd-amd64``
+and fail when executed with the BAD commit
+``./sample-test-reproducer.sh quay.io/kata-containers/kata-deploy-ci:kata-containers-9f512c016e75599a4a921bd84ea47559fe610057-amd64``.
+
+To get the list of all tags in between those two PRs you can use the
+``bisect-range.sh`` script
+
+```bash
+./bisect-range.sh d7afd31fd40e37a675b25c53618904ab57e74ccd 9f512c016e75599a4a921bd84ea47559fe610057
+```
+
+[!NOTE]
+The tagged images are only built per PR, not for individual commits. See
+[kata-deploy-ci](https://quay.io/kata-containers/kata-deploy-ci) to see the
+available images.
+
+To find out which PR caused this regression, you can either manually try the
+individual commits or you can simply execute:
+
+```bash
+bisecter start "$(./bisect-range.sh d7afd31fd40 9f512c016)"
+OCP_DIR=/path/to/openshift/release bisecter run ./sample-test-reproducer.sh
+```
+
+[!NOTE]
+If you use ``KATA_WITH_SYSTEM_QEMU=yes`` you might want to deploy once with
+it and skip it for the cleanup. That way you might (in most cases) test
+all images with a single MCP update instead of per-image MCP update.
+
+[!TIP]
+You can check the bisection progress during/after execution by running
+``bisecter log`` from the current directory. Before starting a new
+bisection you need to execute ``bisecter reset``.

ci/openshift-ci/bisect-range.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+if [ "$#" -gt 2 ] || [ "$#" -lt 1 ] ; then
+	echo "Usage: $0 GOOD [BAD]"
+	echo "Prints list of available kata-deploy-ci tags between GOOD and BAD commits (by default BAD is the latest available tag)"
+	exit 255
+fi
+GOOD="$1"
+[ -n "$2" ] && BAD="$2"
+ARCH=amd64
+REPO="quay.io/kata-containers/kata-deploy-ci"
+
+TAGS=$(skopeo list-tags "docker://$REPO")
+# Only amd64
+TAGS=$(echo "$TAGS" | jq '.Tags' | jq "map(select(endswith(\"$ARCH\")))" | jq -r '.[]')
+# Sort by git
+SORTED=""
+[ -n "$BAD" ] && LOG_ARGS="$GOOD~1..$BAD" || LOG_ARGS="$GOOD~1.."
+for TAG in $(git log --merges --pretty=format:%H --reverse $LOG_ARGS); do
+	[[ "$TAGS" =~ "$TAG" ]] && SORTED+="
+kata-containers-$TAG-$ARCH"
+done
+# Comma separated tags with repo
+echo "$SORTED" | tail -n +2 | sed -e "s@^@$REPO:@" | paste -s -d, -

ci/openshift-ci/cleanup.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script tries to removes most of the resources added by `test.sh` script
+# from the cluster.
+
+scripts_dir=$(dirname $0)
+deployments_dir=${scripts_dir}/cluster/deployments
+configs_dir=${scripts_dir}/configs
+
+source ${scripts_dir}/lib.sh
+
+# Set to 'yes' if you want to configure SELinux to permissive on the cluster
+# workers.
+#
+SELINUX_PERMISSIVE=${SELINUX_PERMISSIVE:-no}
+
+# Enable workaround for OCP 4.13 https://github.com/kata-containers/kata-containers/pull/9206
+#
+WORKAROUND_9206_CRIO=${WORKAROUND_9206_CRIO:-no}
+
+# Ignore errors as we want best-effort-approach here
+trap - ERR
+
+# Delete webhook resources
+oc delete -f "${scripts_dir}/../../tools/testing/kata-webhook/deploy"
+oc delete -f "${scripts_dir}/cluster/deployments/configmap_kata-webhook.yaml.in"
+
+# Delete potential smoke-test resources
+oc delete -f "${scripts_dir}/smoke/service.yaml"
+oc delete -f "${scripts_dir}/smoke/service_kubernetes.yaml"
+oc delete -f "${scripts_dir}/smoke/http-server.yaml"
+
+# Delete test.sh resources
+oc delete -f "${deployments_dir}/relabel_selinux.yaml"
+if [[ "$WORKAROUND_9206_CRIO" == "yes" ]]; then
+	oc delete -f "${deployments_dir}/workaround-9206-crio-ds.yaml"
+	oc delete -f "${deployments_dir}/workaround-9206-crio.yaml"
+fi
+[ ${SELINUX_PERMISSIVE} == "yes" ] && oc delete -f "${deployments_dir}/machineconfig_selinux.yaml.in"
+
+# Delete kata-containers
+pushd "$katacontainers_repo_dir/tools/packaging/kata-deploy"
+oc delete -f kata-deploy/base/kata-deploy.yaml
+oc -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+oc apply -f kata-cleanup/base/kata-cleanup.yaml
+echo "Wait for all related pods to be gone"
+( repeats=1; for i in $(seq 1 600); do
+  oc get pods -l name="kubelet-kata-cleanup" --no-headers=true -n kube-system 2>&1 | grep "No resources found" -q && ((repeats++)) || repeats=1
+  [ "$repeats" -gt 5 ] && echo kata-cleanup finished && break
+  sleep 1
+done) || { echo "There are still some kata-cleanup related pods after 600 iterations"; oc get all -n kube-system; exit -1; }
+oc delete -f kata-cleanup/base/kata-cleanup.yaml
+oc delete -f kata-rbac/base/kata-rbac.yaml
+oc delete -f runtimeclasses/kata-runtimeClasses.yaml
+

ci/openshift-ci/cluster/configs/selinux.conf

@@ -0,0 +1,6 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+SELINUX=permissive
+SELINUXTYPE=targeted

ci/openshift-ci/cluster/deploy_webhook.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script builds the kata-webhook and deploys it in the test cluster.
+#
+# You should export the KATA_RUNTIME variable with the runtimeclass name
+# configured in your cluster in case it is not the default "kata-ci".
+#
+set -e
+set -o nounset
+set -o pipefail
+
+script_dir="$(realpath $(dirname $0))"
+webhook_dir="${script_dir}/../../../tools/testing/kata-webhook"
+source "${script_dir}/../lib.sh"
+KATA_RUNTIME=${KATA_RUNTIME:-kata-ci}
+
+pushd "${webhook_dir}" >/dev/null
+# Build and deploy the webhook
+#
+info "Builds the kata-webhook"
+./create-certs.sh
+info "Deploys the kata-webhook"
+oc apply -f deploy/
+
+info "Override our KATA_RUNTIME ConfigMap"
+RUNTIME_CLASS="${KATA_RUNTIME}" \
+	envsubst < "${script_dir}/deployments/configmap_kata-webhook.yaml.in" \
+	| oc apply -f -
+
+# Check the webhook was deployed and is working.
+RUNTIME_CLASS="${KATA_RUNTIME}" ./webhook-check.sh
+popd >/dev/null

ci/openshift-ci/cluster/deployments/configmap_installer_kernel.yaml

@@ -0,0 +1,13 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Instruct the daemonset installer to configure Kata Containers to use the
+# host kernel.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ci.kata.installer.kernel
+data:
+  host_kernel: "yes"

ci/openshift-ci/cluster/deployments/configmap_installer_qemu.yaml

@@ -0,0 +1,14 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Instruct the daemonset installer to configure Kata Containers to use the
+# system QEMU.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ci.kata.installer.qemu
+data:
+  qemu_path: /usr/libexec/qemu-kvm
+  host_kernel: "yes"

ci/openshift-ci/cluster/deployments/configmap_kata-webhook.yaml.in

@@ -0,0 +1,12 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Apply customizations to the kata-webhook.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kata-webhook
+data:
+  runtime_class: ${RUNTIME_CLASS}

ci/openshift-ci/cluster/deployments/machineconfig_sandboxedcontainers_extension.yaml

@@ -0,0 +1,9 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 50-enable-sandboxed-containers-extension
+spec:
+  extensions:
+  - sandboxed-containers

ci/openshift-ci/cluster/deployments/machineconfig_selinux.yaml.in

@@ -0,0 +1,23 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Configure SELinux on worker nodes.
+---
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 51-kata-selinux
+spec:
+  config:
+    ignition:
+      version: 2.2.0
+    storage:
+      files:
+      - contents:
+              source: data:text/plain;charset=utf-8;base64,${SELINUX_CONF_BASE64}
+        filesystem: root
+        mode: 0644
+        path: /etc/selinux/config

ci/openshift-ci/cluster/deployments/relabel_selinux.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: relabel-selinux-daemonset
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: restorecon
+  template:
+    metadata:
+      labels:
+        app: restorecon
+    spec:
+      serviceAccountName: kata-deploy-sa
+      hostPID: true
+      containers:
+        - name: relabel-selinux-container
+          image: alpine
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c", "
+            set -e;
+            echo Starting the relabel;
+            nsenter --target 1 --mount bash -xc '
+                command -v semanage &>/dev/null || { echo Does not look like a SELINUX cluster, skipping; exit 0; };
+                for ENTRY in \
+                    \"/(.*/)?opt/kata/bin(/.*)?\" \
+                    \"/(.*/)?opt/kata/runtime-rs/bin(/.*)?\" \
+                    \"/(.*/)?opt/kata/share/kata-.*(/.*)?(/.*)?\" \
+                    \"/(.*/)?opt/kata/share/ovmf(/.*)?\" \
+                    \"/(.*/)?opt/kata/share/tdvf(/.*)?\" \
+                    \"/(.*/)?opt/kata/libexec(/.*)?\";
+                do
+                    semanage fcontext -a -t qemu_exec_t \"$ENTRY\" || semanage fcontext -m -t qemu_exec_t \"$ENTRY\" || { echo \"Error in semanage command\"; exit 1; }
+                done;
+                restorecon -v -R /opt/kata || { echo \"Error in restorecon command\"; exit 1; }
+            ';
+            echo NSENTER_FINISHED_WITH: $?;
+            sleep infinity"]

ci/openshift-ci/cluster/deployments/workaround-9206-crio-ds.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: workaround-9206-crio-ds
+spec:
+  selector:
+    matchLabels:
+      app: workaround-9206-crio-ds
+  template:
+    metadata:
+      labels:
+        app: workaround-9206-crio-ds
+    spec:
+      containers:
+      - name: workaround-9206-crio-ds
+        image: alpine
+        volumeMounts:
+        - name: host-dir
+          mountPath: /tmp/config
+        securityContext:
+          runAsUser: 0
+          privileged: true
+        command: ["/bin/sh", "-c", "while [ ! -f '/tmp/config/10-workaround-9206-crio' ]; do sleep 1; done; echo 'Config file present'; sleep infinity"]
+      volumes:
+      - name: host-dir
+        hostPath:
+          path: /etc/crio/crio.conf.d/

ci/openshift-ci/cluster/deployments/workaround-9206-crio.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 10-workaround-9206-crio
+spec:
+  config:
+    ignition:
+      version: 2.2.0
+    storage:
+      files:
+      - contents:
+              source: data:text/plain;charset=utf-8;base64,W2NyaW9dCnN0b3JhZ2Vfb3B0aW9uID0gWwoJIm92ZXJsYXkuc2tpcF9tb3VudF9ob21lPXRydWUiLApdCg==
+        filesystem: root
+        mode: 0644
+        path: /etc/crio/crio.conf.d/10-workaround-9206-crio

ci/openshift-ci/cluster/install_kata.sh

@@ -0,0 +1,245 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script installs the built kata-containers in the test cluster,
+# and configure a runtime.
+
+scripts_dir=$(dirname $0)
+deployments_dir=${scripts_dir}/deployments
+configs_dir=${scripts_dir}/configs
+
+source ${scripts_dir}/../lib.sh
+
+# Set to 'yes' if you want to configure SELinux to permissive on the cluster
+# workers.
+#
+SELINUX_PERMISSIVE=${SELINUX_PERMISSIVE:-no}
+
+# Set to 'yes' if you want to configure Kata Containers to use the system's
+# QEMU (from the RHCOS extension).
+#
+KATA_WITH_SYSTEM_QEMU=${KATA_WITH_SYSTEM_QEMU:-no}
+
+# Set to 'yes' if you want to configure Kata Containers to use the host kernel.
+#
+KATA_WITH_HOST_KERNEL=${KATA_WITH_HOST_KERNEL:-no}
+
+# kata-deploy image to be used to deploy the kata (by default use CI image
+# that is built for each pull request)
+#
+KATA_DEPLOY_IMAGE=${KATA_DEPLOY_IMAGE:-quay.io/kata-containers/kata-deploy-ci:kata-containers-latest}
+
+# Enable workaround for OCP 4.13 https://github.com/kata-containers/kata-containers/pull/9206
+#
+WORKAROUND_9206_CRIO=${WORKAROUND_9206_CRIO:-no}
+
+# Leverage kata-deploy to install Kata Containers in the cluster.
+#
+apply_kata_deploy() {
+	local deploy_file="tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	pushd "$katacontainers_repo_dir"
+	sed -ri "s#(\s+image:) .*#\1 ${KATA_DEPLOY_IMAGE}#" "$deploy_file"
+
+	info "Applying kata-deploy"
+	oc apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
+	oc label --overwrite ns kube-system pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+	oc apply -f "$deploy_file"
+	oc -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+
+	info "Adding the kata runtime classes"
+	oc apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+	popd
+}
+
+
+# Wait all worker nodes reboot.
+#
+# Params:
+#   $1 - timeout in seconds (default to 900).
+#
+wait_for_reboot() {
+	local delta="${1:-900}"
+	local sleep_time=60
+	declare -A BOOTIDS
+	local workers=($(oc get nodes | \
+		awk '{if ($3 == "worker") { print $1 } }'))
+	# Get the boot ID to compared it changed over time.
+	for node in ${workers[@]}; do
+		BOOTIDS[$node]=$(oc get -o jsonpath='{.status.nodeInfo.bootID}'\
+			node/$node)
+		echo "Wait $node reboot"
+	done
+
+	echo "Set timeout to $delta seconds"
+	timer_start=$(date +%s)
+	while [ ${#workers[@]} -gt 0 ]; do
+		sleep $sleep_time
+		now=$(date +%s)
+		if [ $(($timer_start + $delta)) -lt $now ]; then
+			echo "Timeout: not all workers rebooted"
+			return 1
+		fi
+		echo "Checking after $(($now - $timer_start)) seconds"
+		for i in ${!workers[@]}; do
+			current_id=$(oc get \
+				-o jsonpath='{.status.nodeInfo.bootID}' \
+				node/${workers[i]})
+			if [ "$current_id" != ${BOOTIDS[${workers[i]}]} ]; then
+				echo "${workers[i]} rebooted"
+				unset workers[i]
+			fi
+		done
+	done
+}
+
+wait_mcp_update() {
+	local delta="${1:-3600}"
+	local sleep_time=30
+	# The machineconfigpool is fine when all the workers updated and are ready,
+	# and none are degraded.
+	local ready_count=0
+	local degraded_count=0
+	local machine_count=$(oc get mcp worker -o jsonpath='{.status.machineCount}')
+
+	if [[ -z "$machine_count" && "$machine_count" -lt 1 ]]; then
+		warn "Unabled to obtain the machine count"
+		return 1
+	fi
+
+	echo "Set timeout to $delta seconds"
+	local deadline=$(($(date +%s) + $delta))
+	# The ready count might not have changed yet, so wait a little.
+	while [[ "$ready_count" != "$machine_count" && \
+		"$degraded_count" == 0 ]]; do
+		# Let's check it hit the timeout (or not).
+		local now=$(date +%s)
+		if [ $deadline -lt $now ]; then
+			echo "Timeout: not all workers updated" >&2
+			return 1
+		fi
+		sleep $sleep_time
+		ready_count=$(oc get mcp worker \
+			-o jsonpath='{.status.readyMachineCount}')
+		degraded_count=$(oc get mcp worker \
+			-o jsonpath='{.status.degradedMachineCount}')
+		echo "check machineconfigpool - ready_count: $ready_count degraded_count: $degraded_count"
+	done
+	[ $degraded_count -eq 0 ]
+}
+
+# Enable the RHCOS extension for the Sandboxed Containers.
+#
+enable_sandboxedcontainers_extension() {
+	info "Enabling the RHCOS extension for Sandboxed Containers"
+	local deployment_file="${deployments_dir}/machineconfig_sandboxedcontainers_extension.yaml"
+	oc apply -f ${deployment_file}
+	oc get -f ${deployment_file} || \
+		die "Sandboxed Containers extension machineconfig not found"
+	wait_mcp_update || die "Failed to update the machineconfigpool"
+}
+
+# Print useful information for debugging.
+#
+# Params:
+#   $1 - the pod name
+debug_pod() {
+	local pod="$1"
+	info "Debug pod: ${pod}"
+	oc describe pods "$pod"
+        oc logs "$pod"
+}
+
+# Wait for all pods of the app label to contain expected message
+#
+# Params:
+#   $1 - app labela
+#   $2 - expected pods count (>=1)
+#   $3 - message to be present in the logs
+#   $4 - timeout (60)
+#   $5 - namespace (the current one)
+wait_for_app_pods_message() {
+	local app="$1"
+	local pod_count="$2"
+	local message="$3"
+	local timeout="$4"
+	local namespace="$5"
+	[ -z "$pod_count" ] && pod_count=1
+	[ -z "$timeout" ] && timeout=60
+	[ -n "$namespace" ] && namespace=" -n $namespace "
+	local pod
+	local pods
+	local i
+	SECONDS=0
+	while :; do
+		pods=($(oc get pods -l app="$app" --no-headers=true $namespace | awk '{print $1}'))
+		[ "${#pods}" -ge "$pod_count" ] && break
+		if [ "$SECONDS" -gt "$timeout" ]; then
+			echo "Unable to find ${pod_count} pods for '-l app=\"$app\"' in ${SECONDS}s (${pods[@]})"
+			return -1
+		fi
+	done
+	for pod in "${pods[@]}"; do
+		while :; do
+			local log=$(oc logs $namespace "$pod")
+			echo "$log" | grep "$message" -q && echo "Found $(echo "$log" | grep "$message") in $pod's log ($SECONDS)" && break;
+			if [ "$SECONDS" -gt "$timeout" ]; then
+				echo -n "Message '$message' not present in '${pod}' pod of the '-l app=\"$app\"' "
+				echo "pods after ${SECONDS}s (${pods[@]})"
+				echo "Pod $pod's output so far:"
+				echo "$log"
+				return -1
+			fi
+			sleep 1;
+		done
+	done
+}
+
+oc config set-context --current --namespace=default
+
+worker_nodes=$(oc get nodes |  awk '{if ($3 == "worker") { print $1 } }')
+num_nodes=$(echo $worker_nodes | wc -w)
+[ $num_nodes -ne 0 ] || \
+	die "No worker nodes detected. Something is wrong with the cluster"
+
+if [ "${KATA_WITH_SYSTEM_QEMU}" == "yes" ]; then
+	# QEMU is deployed on the workers via RCHOS extension.
+	enable_sandboxedcontainers_extension
+	oc apply -f ${deployments_dir}/configmap_installer_qemu.yaml
+fi
+
+if [ "${KATA_WITH_HOST_KERNEL}" == "yes" ]; then
+	oc apply -f ${deployments_dir}/configmap_installer_kernel.yaml
+fi
+
+apply_kata_deploy
+
+# Set SELinux to permissive mode
+if [ ${SELINUX_PERMISSIVE} == "yes" ]; then
+	info "Configuring SELinux"
+	if [ -z "$SELINUX_CONF_BASE64" ]; then
+		export SELINUX_CONF_BASE64=$(echo \
+			$(cat $configs_dir/selinux.conf|base64) | \
+			sed -e 's/\s//g')
+	fi
+	envsubst < ${deployments_dir}/machineconfig_selinux.yaml.in | \
+		oc apply -f -
+	oc get machineconfig/51-kata-selinux || \
+		die "SELinux machineconfig not found"
+	# The new SELinux configuration will trigger another reboot.
+	wait_for_reboot
+fi
+
+if [[ "$WORKAROUND_9206_CRIO" == "yes" ]]; then
+	info "Applying workaround to enable skip_mount_home in crio on OCP 4.13"
+	oc apply -f "${deployments_dir}/workaround-9206-crio.yaml"
+	oc apply -f "${deployments_dir}/workaround-9206-crio-ds.yaml"
+	wait_for_app_pods_message workaround-9206-crio-ds "$num_nodes" "Config file present" 1200 || echo "Failed to apply the workaround, proceeding anyway..."
+fi
+
+# FIXME: Remove when https://github.com/kata-containers/kata-containers/pull/8417 is resolved
+# Selinux context is currently not handled by kata-deploy
+oc apply -f ${deployments_dir}/relabel_selinux.yaml
+wait_for_app_pods_message restorecon "$num_nodes" "NSENTER_FINISHED_WITH:" 120 "kube-system" || echo "Failed to treat selinux, proceeding anyway..."

ci/openshift-ci/images/Dockerfile.buildroot

@@ -4,7 +4,7 @@
 #
 # This is the build root image for Kata Containers on OpenShift CI.
 #
-FROM quay.io/centos/centos:stream8
+FROM quay.io/centos/centos:stream9
 
 RUN yum -y update && \
     yum -y install \

ci/openshift-ci/lib.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2023 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Ensure GOPATH set
+if command -v go > /dev/null; then
+    export GOPATH=${GOPATH:-$(go env GOPATH)}
+else
+    # if go isn't installed, set default location for GOPATH
+    export GOPATH="${GOPATH:-$HOME/go}"
+fi
+
+lib_dir=$(dirname "${BASH_SOURCE[0]}")
+source "$lib_dir/../../tests/common.bash"
+
+export katacontainers_repo=${katacontainers_repo:="github.com/kata-containers/kata-containers"}
+export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"

ci/openshift-ci/run_smoke_test.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Run a smoke test.
+#
+
+script_dir=$(dirname $0)
+source ${script_dir}/lib.sh
+
+pod='http-server'
+
+# Create a pod.
+#
+info "Creating the ${pod} pod"
+[ -z "$KATA_RUNTIME" ] && die "Please set the KATA_RUNTIME first"
+envsubst < "${script_dir}/smoke/${pod}.yaml.in" | \
+	oc apply -f - || \
+	die "failed to create ${pod} pod"
+
+# Check it eventually goes to 'running'
+#
+wait_time=600
+sleep_time=5
+cmd="oc get pod/${pod} -o jsonpath='{.status.containerStatuses[0].state}' | \
+	grep running > /dev/null"
+info "Wait until the pod gets running"
+waitForProcess $wait_time $sleep_time "$cmd" || timed_out=$?
+if [ -n "$timed_out" ]; then
+	oc describe pod/${pod}
+	oc delete pod/${pod}
+	die "${pod} not running"
+fi
+info "${pod} is running"
+
+# Add a file with the hello message
+#
+hello_file=/tmp/hello
+hello_msg='Hello World'
+oc exec ${pod} -- sh -c "echo $hello_msg > $hello_file"
+
+info "Creating the service and route"
+if oc apply -f ${script_dir}/smoke/service.yaml; then
+    # Likely on OCP, use service
+    is_ocp=1
+    host=$(oc get route/http-server-route -o jsonpath={.spec.host})
+    port=80
+else
+    # Likely on plain kubernetes, test using another container
+    is_ocp=0
+    info "Failed to create service, likely not on OCP, trying via NodePort"
+    oc apply -f "${script_dir}/smoke/service_kubernetes.yaml"
+    # For some reason kcli's cluster lists external IP as internal IP, try both
+    host=$(oc get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="ExternalIP")].address}')
+    [ -z "$host"] && host=$(oc get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
+    port=$(oc get service/http-server-service -o jsonpath='{.spec.ports[0].nodePort}')
+fi
+
+info "Wait for the HTTP server to respond"
+tempfile=$(mktemp)
+check_cmd="curl -vvv '${host}:${port}${hello_file}' 2>&1 | tee -a '$tempfile' | grep -q '$hello_msg'"
+if waitForProcess 60 1 "${check_cmd}"; then
+    test_status=0
+    info "HTTP server is working"
+else
+    test_status=1
+    echo "::error:: HTTP server not working"
+    echo "::group::Output of the \"curl -vvv '${host}:${port}${hello_file}'\""
+    cat "${tempfile}"
+    echo "::endgroup::"
+    echo "::group::Describe kube-system namespace"
+    oc describe -n kube-system all
+    echo "::endgroup::"
+    echo "::group::Descibe current namespace"
+    oc describe all
+    echo "::endgroup::"
+    info "HTTP server is unreachable"
+fi
+rm -f "$tempfile"
+
+# Delete the resources.
+#
+info "Deleting the service/route"
+if [ "$is_ocp" -eq 0 ]; then
+    oc delete -f ${script_dir}/smoke/service_kubernetes.yaml
+else
+    oc delete -f ${script_dir}/smoke/service.yaml
+fi
+info "Deleting the ${pod} pod"
+oc delete pod/${pod} || test_status=$?
+
+exit $test_status

ci/openshift-ci/sample-test-reproducer.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# A sample script to deploy, configure, run E2E_TEST and soft-cleanup
+# afterwards OCP cluster using kata-containers primarily created for use
+# with https://github.com/ldoktor/bisecter
+
+[ "$#" -ne 1 ] && echo "Provide image as the first and only argument" && exit 255
+export KATA_DEPLOY_IMAGE="$1"
+OCP_DIR="${OCP_DIR:-/path/to/your/openshift/release/}"
+E2E_TEST="${E2E_TEST:-'"[sig-node] Container Runtime blackbox test on terminated container should report termination message as empty when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set [NodeConformance] [Conformance] [Suite:openshift/conformance/parallel/minimal] [Suite:k8s]"'}"
+KATA_CI_DIR="${KATA_CI_DIR:-$(pwd)}"
+export KATA_RUNTIME="${KATA_RUNTIME:-kata-qemu}"
+
+## SETUP
+# Deploy kata
+SETUP=0
+pushd "$KATA_CI_DIR" || { echo "Failed to cd to '$KATA_CI_DIR'"; exit 255; }
+./test.sh || SETUP=125
+cluster/deploy_webhook.sh || SETUP=125
+if [ $SETUP != 0 ]; then
+    ./cleanup.sh
+    exit "$SETUP"
+fi
+popd || true
+# Disable security
+oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
+oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts
+oc label --overwrite ns default pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/warn=baseline pod-security.kubernetes.io/audit=baseline
+
+## TEST EXECUTION
+# Run the testing
+pushd "$OCP_DIR" || { echo "Failed to cd to '$OCP_DIR'"; exit 255; }
+echo "$E2E_TEST" > /tmp/tsts
+# Remove previously-existing temporarily files as well as previous results
+OUT=RESULTS/tmp
+rm -Rf /tmp/*test* /tmp/e2e-*
+rm -R $OUT
+mkdir -p $OUT
+# Run the tests ignoring the monitor health checks
+./openshift-tests run --provider azure -o "$OUT/job.log" --junit-dir "$OUT" --file /tmp/tsts --max-parallel-tests 5 --cluster-stability Disruptive
+RET=$?
+popd || true
+
+## CLEANUP
+./cleanup.sh
+exit "$RET"
+

ci/openshift-ci/smoke/http-server.yaml.in

@@ -0,0 +1,30 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Define the pod for a http server app.
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: http-server
+  labels:
+    app: http-server-app
+spec:
+  containers:
+    - name: http-server
+      image: docker.io/library/python:3
+      ports:
+        - containerPort: 8080
+      command: ["python3"]
+      args: [ "-m", "http.server", "8080"]
+      securityContext:
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop:
+            - ALL
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+  runtimeClassName: ${KATA_RUNTIME}

ci/openshift-ci/smoke/service.yaml

@@ -0,0 +1,28 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Create the service on port 80 for the http-server app.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-server-service
+spec:
+  selector:
+    app: http-server-app
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+# Create the route to the app's service '/'.
+---
+apiVersion: route.openshift.io/v1
+kind: Route
+metadata:
+  name: http-server-route
+spec:
+  path: "/"
+  to:
+    kind: Service
+    name: http-server-service

ci/openshift-ci/smoke/service_kubernetes.yaml

@@ -0,0 +1,18 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Create the service on port 80 for the http-server app.
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: http-server-service
+spec:
+  selector:
+    app: http-server-app
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+  type: NodePort

ci/openshift-ci/test.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# The kata shim to be used
+export KATA_RUNTIME=${KATA_RUNTIME:-kata-qemu}
+
+script_dir=$(dirname $0)
+source ${script_dir}/lib.sh
+
+suite=$1
+if [ -z "$1" ]; then
+	suite='smoke'
+fi
+
+# Make oc and kubectl visible
+export PATH=/tmp/shared:$PATH
+
+oc version || die "Test cluster is unreachable"
+
+info "Install and configure kata into the test cluster"
+export SELINUX_PERMISSIVE="no"
+${script_dir}/cluster/install_kata.sh || die "Failed to install kata-containers"
+
+info "Run test suite: $suite"
+test_status='PASS'
+${script_dir}/run_${suite}_test.sh || test_status='FAIL'
+info "Test suite: $suite: $test_status"
+[ "$test_status" == "PASS" ]

ci/run.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 Ant Financial
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -e
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-export CI_JOB="${CI_JOB:-}"
-
-clone_tests_repo
-
-pushd ${tests_repo_dir}
-.ci/run.sh
-# temporary fix, see https://github.com/kata-containers/tests/issues/3878
-if [ "$(uname -m)" != "s390x" ] && [ "$CI_JOB" == "CRI_CONTAINERD_K8S_MINIMAL" ]; then
-	tracing/test-agent-shutdown.sh
-fi
-popd