From 516d62d717742a9cde7bae9e29f7148e55c19139 Mon Sep 17 00:00:00 2001 From: marvelshan Date: Tue, 2 Jun 2026 06:54:27 +0000 Subject: [PATCH] ci: add Firecracker testing (k8s with blockfile snapshotter) Firecracker currently has no testing CI, only build CI. Add fc hypervisor to the test matrices for stability, CRI containerd, nerdctl, and Kubernetes tests. For the Kubernetes test matrix, configure the blockfile snapshotter instead of devmapper as devmapper requires 10G+ loop devices which is too space costly. The blockfile snapshotter uses a 500MB ext4 scratch file. Firecracker requires shared_fs=none in the kata configuration when using block-based rootfs; this is set by the _configure_fc_shared_fs helper which searches all three possible FC config paths and warns if none is found. The other test matrices (stability, CRI containerd, nerdctl) run FC with the default overlayfs snapshotter, which validates basic FC functionality (boot, agent, container lifecycle) without requiring snapshotter pre-configuration in those workflows. Refactor configure_snapshotter to extract the shared containerd configuration flow (config path resolution, tomlq install/uninstall, arch mapping, config update, service restart) into a reusable _configure_containerd_snapshotter helper so that adding new snapshotters does not require duplicating the entire function. Related #7996, #7872 Signed-off-by: marvelshan --- .github/workflows/basic-ci-amd64.yaml | 3 +- .github/workflows/ci.yaml | 2 + .../run-k8s-tests-on-free-runner.yaml | 6 + tests/integration/kubernetes/gha-run.sh | 184 +++++++++++------- tools/testing/gatekeeper/required-tests.yaml | 6 + 5 files changed, 132 insertions(+), 69 deletions(-) diff --git a/.github/workflows/basic-ci-amd64.yaml b/.github/workflows/basic-ci-amd64.yaml index 72348b2454..02f66387d4 100644 --- a/.github/workflows/basic-ci-amd64.yaml +++ b/.github/workflows/basic-ci-amd64.yaml @@ -88,7 +88,7 @@ jobs: fail-fast: false matrix: containerd_version: ['lts', 'active'] - vmm: ['clh', 'clh-runtime-rs', 'dragonball', 'qemu', 'qemu-runtime-rs'] + vmm: ['clh', 'clh-runtime-rs', 'dragonball', 'qemu', 'qemu-runtime-rs', 'fc'] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-run-containerd-stability-amd64-${{ toJSON(matrix) }} cancel-in-progress: true @@ -245,6 +245,7 @@ jobs: - qemu - clh-runtime-rs - qemu-runtime-rs + - fc concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-nerdctl-amd64-${{ toJSON(matrix) }} cancel-in-progress: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ac1a3234c6..bdd1dbdd57 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -403,11 +403,13 @@ jobs: { containerd_version: lts, vmm: qemu }, { containerd_version: lts, vmm: clh-runtime-rs }, { containerd_version: lts, vmm: qemu-runtime-rs }, + { containerd_version: lts, vmm: fc }, { containerd_version: active, vmm: clh }, { containerd_version: active, vmm: dragonball }, { containerd_version: active, vmm: qemu }, { containerd_version: active, vmm: clh-runtime-rs }, { containerd_version: active, vmm: qemu-runtime-rs }, + { containerd_version: active, vmm: fc }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-cri-amd64-${{ toJSON(matrix) }} diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml index 9845bb308b..e6f73fb379 100644 --- a/.github/workflows/run-k8s-tests-on-free-runner.yaml +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -50,6 +50,7 @@ jobs: { vmm: qemu-runtime-rs, containerd_version: active }, { vmm: clh-runtime-rs, containerd_version: lts }, { vmm: clh-runtime-rs, containerd_version: active }, + { vmm: fc, containerd_version: active, snapshotter: blockfile }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-free-runner-${{ toJSON(matrix) }} @@ -68,6 +69,7 @@ jobs: K8S_TEST_HOST_TYPE: baremetal-no-attestation CONTAINER_ENGINE: containerd CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter || '' }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -115,6 +117,10 @@ jobs: - name: Install `bats` run: bash tests/integration/kubernetes/gha-run.sh install-bats + - name: Configure ${{ matrix.environment.snapshotter }} snapshotter + if: ${{ matrix.environment.snapshotter != '' }} + run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter + - name: Deploy Kata timeout-minutes: 20 run: bash tests/integration/kubernetes/gha-run.sh deploy-kata diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d271e10b16..a266235545 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -38,7 +38,53 @@ export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}" export TARGET_ARCH="${TARGET_ARCH:-x86_64}" export RUNS_ON_AKS="${RUNS_ON_AKS:-false}" -function configure_devmapper() { +function _configure_containerd_snapshotter() { + local snapshotter_name="${1}" + local tomlq_filter="${2}" + + local containerd_config_file + case "${KUBERNETES}" in + k3s) + containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" + sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}" + ;; + kubeadm|vanilla) + containerd_config_file="/etc/containerd/config.toml" + ;; + *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; + esac + + install_tomlq + + local containerd_arch + containerd_arch="$(uname -m)" + case "${containerd_arch}" in + x86_64) containerd_arch="amd64" ;; + aarch64|arm64) containerd_arch="arm64" ;; + esac + + echo "Updating containerd config with tomlq for ${snapshotter_name} snapshotter..." + local config_tmp_file + config_tmp_file="$(sudo mktemp)" + # shellcheck disable=SC2016 + sudo cat "${containerd_config_file}" | tomlq -t --arg platform "linux/${containerd_arch}" "${tomlq_filter}" | sudo tee "${config_tmp_file}" > /dev/null + sudo mv "${config_tmp_file}" "${containerd_config_file}" + + uninstall_tomlq + + case "${KUBERNETES}" in + k3s) sudo systemctl restart k3s ;; + vanilla|kubeadm) sudo systemctl restart containerd ;; + *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; + esac + + sleep 60s + sudo cat "${containerd_config_file}" + + info "${snapshotter_name} snapshotter configured" +} + +function _setup_devmapper_device() { sudo mkdir -p /var/lib/containerd/devmapper sudo truncate --size 10G /var/lib/containerd/devmapper/data-disk.img sudo truncate --size 10G /var/lib/containerd/devmapper/meta-disk.img @@ -62,75 +108,12 @@ EOF sudo systemctl daemon-reload sudo systemctl enable --now containerd-devmapper - # Time to setup the thin pool for consumption. - # The table arguments are such. - # start block in the virtual device - # length of the segment (block device size in bytes / Sector size (512) - # metadata device - # block data device - # data_block_size Currently set it 512 (128KB) - # low_water_mark. Copied this from containerd snapshotter test setup - # no. of feature arguments - # Skip zeroing blocks for new volumes. sudo dmsetup create contd-thin-pool \ --table "0 20971520 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing" +} - case "${KUBERNETES}" in - k3s) - containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" - sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}" - ;; - kubeadm) - containerd_config_file="/etc/containerd/config.toml" - ;; - *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; - esac - - # We need to use tomlq to update the containerd config with the devmapper configuration, - # as it's a more complex update that involves adding new entries and modifying existing ones - # for two different containerd versions. - install_tomlq - - containerd_arch="$(uname -m)" - case "${containerd_arch}" in - x86_64) containerd_arch="amd64" ;; - aarch64|arm64) containerd_arch="arm64" ;; - esac - - echo "Updating containerd config with tomlq..." - config_tmp_file="$(sudo mktemp)" - # shellcheck disable=SC2016 - sudo cat "${containerd_config_file}" | tomlq -t --arg platform "linux/${containerd_arch}" ' - .plugins["io.containerd.snapshotter.v1.devmapper"].pool_name = "contd-thin-pool" - | .plugins["io.containerd.snapshotter.v1.devmapper"].base_image_size = "4096MB" - | .plugins["io.containerd.transfer.v1.local"].unpack_config = - [((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "devmapper"})] - | if .version == 3 then - .plugins["io.containerd.cri.v1.images"].snapshotter = "devmapper" - else - .plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "devmapper" - end - ' | sudo tee "${config_tmp_file}" > /dev/null - sudo mv "${config_tmp_file}" "${containerd_config_file}" - - # We only need tomlq for this configuration. - # yq, installed by install_tomlq, might cause an issue with go-based yq used by CI. - # So we uninstall tomlq to remove the yq from PATH and avoid any potential conflict. - uninstall_tomlq - - case "${KUBERNETES}" in - k3s) - sudo systemctl restart k3s ;; - kubeadm) - sudo systemctl restart containerd ;; - *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; - esac - - sleep 60s - sudo cat "${containerd_config_file}" - - if [[ "${KUBERNETES}" = 'k3s' ]] - then +function _verify_devmapper() { + if [[ "${KUBERNETES}" = 'k3s' ]]; then local ctr_dm_status local result @@ -149,11 +132,76 @@ EOF sudo dmsetup status -v } +function _setup_blockfile_device() { + sudo mkdir -p /opt/containerd + sudo dd if=/dev/zero of=/opt/containerd/blockfile bs=1M count=500 + sudo mkfs.ext4 /opt/containerd/blockfile +} + +function _configure_fc_shared_fs() { + if is_firecracker_hypervisor; then + local fc_config="" + for candidate in \ + "/opt/kata/share/defaults/kata-containers/runtimes/fc/configuration-fc.toml" \ + "/opt/kata/share/defaults/kata-containers/configuration-fc.toml" \ + "/etc/kata-containers/configuration-fc.toml"; do + if [[ -f "${candidate}" ]]; then + fc_config="${candidate}" + break + fi + done + if [[ -z "${fc_config}" ]]; then + warn "FC kata config not found, shared_fs unchanged" + return 0 + fi + info "Setting shared_fs=none in ${fc_config}" + sudo sed -i 's/^#\?shared_fs.*/shared_fs = "none"/' "${fc_config}" + fi +} + +# shellcheck disable=SC2016 +readonly devmapper_tomlq_filter=' + .plugins["io.containerd.snapshotter.v1.devmapper"].pool_name = "contd-thin-pool" + | .plugins["io.containerd.snapshotter.v1.devmapper"].base_image_size = "4096MB" + | .plugins["io.containerd.transfer.v1.local"].unpack_config = + [((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "devmapper"})] + | if .version == 3 then + .plugins["io.containerd.cri.v1.images"].snapshotter = "devmapper" + else + .plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "devmapper" + end +' + +# shellcheck disable=SC2016 +readonly blockfile_tomlq_filter=' + .plugins["io.containerd.snapshotter.v1.blockfile"].root_path = "/var/lib/containerd-blockfile" + | .plugins["io.containerd.snapshotter.v1.blockfile"].scratch_file = "/opt/containerd/blockfile" + | .plugins["io.containerd.snapshotter.v1.blockfile"].fs_type = "ext4" + | .plugins["io.containerd.snapshotter.v1.blockfile"].mount_options = [] + | .plugins["io.containerd.snapshotter.v1.blockfile"].recreate_scratch = true + | .plugins["io.containerd.transfer.v1.local"].unpack_config = + [((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "blockfile"})] + | if .version == 3 then + .plugins["io.containerd.cri.v1.images"].snapshotter = "blockfile" + else + .plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "blockfile" + end +' + function configure_snapshotter() { echo "::group::Configuring ${SNAPSHOTTER}" case "${SNAPSHOTTER}" in - devmapper) configure_devmapper ;; + devmapper) + _setup_devmapper_device + _configure_containerd_snapshotter "devmapper" "${devmapper_tomlq_filter}" + _verify_devmapper + ;; + blockfile) + _setup_blockfile_device + _configure_containerd_snapshotter "blockfile" "${blockfile_tomlq_filter}" + _configure_fc_shared_fs + ;; *) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;; esac diff --git a/tools/testing/gatekeeper/required-tests.yaml b/tools/testing/gatekeeper/required-tests.yaml index cc03e78629..d9b1a76010 100644 --- a/tools/testing/gatekeeper/required-tests.yaml +++ b/tools/testing/gatekeeper/required-tests.yaml @@ -49,16 +49,19 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, qemu) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, qemu-runtime-rs) + - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, fc) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, clh) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, clh-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, qemu) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, qemu-runtime-rs) + - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, fc) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-kata-agent-apis - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (clh-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (qemu) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (qemu-runtime-rs) + - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (fc) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, clh) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, qemu) @@ -70,11 +73,13 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, dragonball) / run-cri-containerd-amd64 (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, qemu-runtime-rs) / run-cri-containerd-amd64 (active, qemu-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, qemu) / run-cri-containerd-amd64 (active, qemu) + - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, fc) / run-cri-containerd-amd64 (active, fc) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, clh) / run-cri-containerd-amd64 (lts, clh) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, clh-runtime-rs) / run-cri-containerd-amd64 (lts, clh-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, dragonball) / run-cri-containerd-amd64 (lts, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, qemu-runtime-rs) / run-cri-containerd-amd64 (lts, qemu-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, qemu) / run-cri-containerd-amd64 (lts, qemu) + - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, fc) / run-cri-containerd-amd64 (lts, fc) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-s390x (active, qemu) / run-cri-containerd-s390x (active, qemu) #- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu, kubeadm) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh-azure, normal) @@ -92,6 +97,7 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (qemu-runtime-rs, active) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (clh-runtime-rs, lts) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (clh-runtime-rs, active) + - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (fc, active, blockfile) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (devmapper, qemu, kubeadm) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (nydus, qemu-coco-dev, kubeadm) - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-on-tee (sev-snp, qemu-snp)