diff --git a/.github/workflows/basic-ci-amd64.yaml b/.github/workflows/basic-ci-amd64.yaml index 79c938fb1e..687e99f5d9 100644 --- a/.github/workflows/basic-ci-amd64.yaml +++ b/.github/workflows/basic-ci-amd64.yaml @@ -88,7 +88,7 @@ jobs: fail-fast: false matrix: containerd_version: ['lts', 'active'] - vmm: ['clh', 'cloud-hypervisor', 'dragonball', 'qemu', 'qemu-runtime-rs'] + vmm: ['clh', 'cloud-hypervisor', 'dragonball', 'qemu', 'qemu-runtime-rs', 'fc-rs'] runs-on: ubuntu-22.04 env: CONTAINERD_VERSION: ${{ matrix.containerd_version }} @@ -283,6 +283,7 @@ jobs: - qemu - cloud-hypervisor - qemu-runtime-rs + - fc-rs runs-on: ubuntu-22.04 env: KATA_HYPERVISOR: ${{ matrix.vmm }} diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml index 23b738b7b6..c212c80e1f 100644 --- a/.github/workflows/run-k8s-tests-on-free-runner.yaml +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -46,6 +46,8 @@ jobs: { vmm: qemu-runtime-rs, containerd_version: active }, { vmm: cloud-hypervisor, containerd_version: lts }, { vmm: cloud-hypervisor, containerd_version: active }, + { vmm: fc-rs, containerd_version: lts, snapshotter: devmapper }, + { vmm: fc-rs, containerd_version: active, snapshotter: devmapper }, ] runs-on: ubuntu-24.04 permissions: @@ -61,6 +63,7 @@ jobs: K8S_TEST_HOST_TYPE: baremetal-no-attestation CONTAINER_ENGINE: containerd CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter || '' }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -105,6 +108,10 @@ jobs: - name: Deploy k8s (kubeadm) run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s + - name: Configure snapshotter + if: matrix.environment.snapshotter != '' + run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter + - name: Install `bats` run: bash tests/integration/kubernetes/gha-run.sh install-bats diff --git a/docs/design/virtualization.md b/docs/design/virtualization.md index 684e912b55..809209746c 100644 --- a/docs/design/virtualization.md +++ b/docs/design/virtualization.md @@ -275,7 +275,7 @@ Each hypervisor has a dedicated configuration file: |------------|----------------|-----------------| | QEMU |`configuration-qemu-runtime-rs.toml` |`configuration-qemu.toml` | | Cloud Hypervisor | `configuration-cloud-hypervisor.toml` | `configuration-clh.toml` | -| Firecracker | `configuration-rs-fc.toml` | `configuration-fc.toml` | +| Firecracker | `configuration-fc-rs.toml` | `configuration-fc.toml` | | Dragonball | `configuration-dragonball.toml` (default) | `No` | > **Note:** Configuration files are typically installed in `/opt/kata/share/defaults/kata-containers/` or `/opt/kata/share/defaults/kata-containers/runtime-rs/` or `/usr/share/defaults/kata-containers/`. diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 2da9e29e82..21df4846d9 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -419,7 +419,7 @@ endif ifneq (,$(FCCMD)) KNOWN_HYPERVISORS += $(HYPERVISOR_FC) - CONFIG_FILE_FC = configuration-rs-fc.toml + CONFIG_FILE_FC = configuration-fc-rs.toml CONFIG_FC = config/$(CONFIG_FILE_FC) CONFIG_FC_IN = $(CONFIG_FC).in CONFIG_PATH_FC = $(abspath $(CONFDIR)/$(CONFIG_FILE_FC)) diff --git a/src/runtime-rs/README.md b/src/runtime-rs/README.md index fe6f8e10dc..15af90bf38 100644 --- a/src/runtime-rs/README.md +++ b/src/runtime-rs/README.md @@ -157,7 +157,7 @@ Configuration files in `config/`: | `configuration-dragonball.toml.in` | Dragonball | Built-in VMM | | `configuration-qemu-runtime-rs.toml.in` | QEMU | Default external | | `configuration-cloud-hypervisor.toml.in` | Cloud Hypervisor | Modern VMM | -| `configuration-rs-fc.toml.in` | Firecracker | Lightweight microVM | +| `configuration-fc-rs.toml.in` | Firecracker | Lightweight microVM | | `configuration-remote.toml.in` | Remote | Remote hypervisor | | `configuration-qemu-tdx-runtime-rs.toml.in` | QEMU + TDX | Intel TDX confidential computing | | `configuration-qemu-snp-runtime-rs.toml.in` | QEMU + SEV-SNP | AMD SEV-SNP confidential computing | diff --git a/src/runtime-rs/config/configuration-rs-fc.toml.in b/src/runtime-rs/config/configuration-fc-rs.toml.in similarity index 100% rename from src/runtime-rs/config/configuration-rs-fc.toml.in rename to src/runtime-rs/config/configuration-fc-rs.toml.in diff --git a/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs b/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs index 7459814b28..4b44220295 100644 --- a/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs +++ b/src/runtime-rs/crates/hypervisor/src/firecracker/fc_api.rs @@ -3,6 +3,8 @@ // //SPDX-License-Identifier: Apache-2.0 +use std::convert::TryFrom; + use crate::{ firecracker::{ inner_hypervisor::{FC_AGENT_SOCKET_NAME, ROOT}, @@ -121,7 +123,8 @@ impl FcInner { let body_config: String = json!({ "mem_size_mib": self.config.memory_info.default_memory, - "vcpu_count": self.config.cpu_info.default_vcpus.ceil() as u8, + "vcpu_count": u8::try_from(self.config.cpu_info.default_vcpus.ceil() as u64) + .context("vcpu_count overflows u8")?, }) .to_string(); let body_kernel: String = json!({ @@ -289,7 +292,7 @@ impl FcInner { // A transport error (FC not ready yet) — retry. Err(FcRequestError::Transport(e)) => { debug!(sl(), "FC not reachable yet, retrying: {:?}", e); - std::thread::sleep(std::time::Duration::from_millis(10)); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; continue; } // An HTTP-level error from FC — fail immediately with the diff --git a/src/runtime-rs/crates/hypervisor/src/firecracker/inner.rs b/src/runtime-rs/crates/hypervisor/src/firecracker/inner.rs index 9cf888cc80..7ed6c26e98 100644 --- a/src/runtime-rs/crates/hypervisor/src/firecracker/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/firecracker/inner.rs @@ -130,9 +130,11 @@ impl FcInner { if !jailed { if let Some(netns_path) = &netns { debug!(sl(), "set netns for vmm master {:?}", &netns_path); - let netns_fd = std::fs::File::open(netns_path); - let _ = setns(netns_fd?.as_raw_fd(), CloneFlags::CLONE_NEWNET) - .context("set netns failed"); + let netns_fd = std::fs::File::open(netns_path)?; + setns(netns_fd.as_raw_fd(), CloneFlags::CLONE_NEWNET) + .map_err(|e| std::io::Error::other( + format!("setns into {:?} failed: {}", netns_path, e), + ))?; } } if let Some(label) = selinux_label.as_ref() { diff --git a/src/runtime-rs/crates/hypervisor/src/firecracker/inner_device.rs b/src/runtime-rs/crates/hypervisor/src/firecracker/inner_device.rs index b7b9ed7bf5..7c113c2cd6 100644 --- a/src/runtime-rs/crates/hypervisor/src/firecracker/inner_device.rs +++ b/src/runtime-rs/crates/hypervisor/src/firecracker/inner_device.rs @@ -33,7 +33,7 @@ impl FcInner { .context("add block device"), DeviceType::Network(network) => { // Buffer network devices and send them to FC just before InstanceStart - // in start_vm(). Firecracker rejects PUT /network-interfaces after the + // in boot_vm(). Firecracker rejects PUT /network-interfaces after the // VM has started, so we must ensure they arrive before InstanceStart. // This mirrors the Go runtime's batch-configuration approach. // diff --git a/tests/common.bash b/tests/common.bash index 2257c9bcd2..5a40c33ad1 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -517,7 +517,7 @@ function enabling_hypervisor() { declare -r CONTAINERD_SHIM_KATA="/usr/local/bin/containerd-shim-kata-${KATA_HYPERVISOR}-v2" case "${KATA_HYPERVISOR}" in - dragonball|cloud-hypervisor|qemu-runtime-rs|qemu-se-runtime-rs) + dragonball|cloud-hypervisor|qemu-runtime-rs|qemu-se-runtime-rs|fc-rs) sudo ln -sf "${KATA_DIR}/runtime-rs/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}" declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers/runtime-rs" ;; @@ -536,6 +536,65 @@ function enabling_hypervisor() { } +# Sets up a devmapper thin-pool and reconfigures standalone containerd to use +# it as the default snapshotter. Required for block-device based hypervisors +# (e.g. Firecracker / fc-rs) that cannot use the overlayfs snapshotter. +# Expects containerd to already be installed and /etc/containerd/config.toml +# to exist (e.g. after `containerd config default | sudo tee ...`). +function configure_devmapper_for_containerd() { + info "Configuring devmapper snapshotter for standalone containerd" + + sudo mkdir -p /var/lib/containerd/devmapper + sudo truncate --size 10G /var/lib/containerd/devmapper/data-disk.img + sudo truncate --size 1G /var/lib/containerd/devmapper/meta-disk.img + + # Allocate loop devices dynamically to avoid conflicts with pre-existing ones. + local loop_data loop_meta + loop_data=$(sudo losetup --find --show /var/lib/containerd/devmapper/data-disk.img) + loop_meta=$(sudo losetup --find --show /var/lib/containerd/devmapper/meta-disk.img) + info "devmapper: data=${loop_data} meta=${loop_meta}" + + local data_sectors + data_sectors=$(sudo blockdev --getsz "${loop_data}") + sudo dmsetup create contd-thin-pool \ + --table "0 ${data_sectors} thin-pool ${loop_meta} ${loop_data} 512 32768 1 skip_block_zeroing" + + # Write a complete containerd v2 config combining the kata runtime with the + # devmapper snapshotter. We replace whatever was there before (both the v2 + # config from overwrite_containerd_config and the v3 default from + # 'containerd config default') so the result is always correct regardless of + # format or prior content. + sudo tee /etc/containerd/config.toml <<'TOML_EOF' +version = 2 + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "devmapper" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" + +[plugins."io.containerd.snapshotter.v1.devmapper"] + pool_name = "contd-thin-pool" + root_path = "/var/lib/containerd/devmapper" + base_image_size = "4096MB" + discard_blocks = true +TOML_EOF + + sudo systemctl restart containerd + + # Verify the plugin came up healthy + local dm_status + dm_status=$(sudo ctr plugins ls | awk '$2 ~ /^devmapper$/ { print $4 }' || true) + [ "${dm_status}" = "ok" ] || \ + die "containerd devmapper snapshotter not healthy (status: '${dm_status}')" + + info "devmapper snapshotter configured and healthy" +} + function check_containerd_config_for_kata() { # check containerd config declare -r line1="default_runtime_name = \"kata\"" diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 6055005d64..b9a1c8c615 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -80,7 +80,7 @@ EOF containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}" ;; - kubeadm) + kubeadm|vanilla) containerd_config_file="/etc/containerd/config.toml" ;; *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; @@ -120,7 +120,7 @@ EOF case "${KUBERNETES}" in k3s) sudo systemctl restart k3s ;; - kubeadm) + kubeadm|vanilla) sudo systemctl restart containerd ;; *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac diff --git a/tests/integration/nerdctl/gha-run.sh b/tests/integration/nerdctl/gha-run.sh index 91fbff1f52..444debd08f 100755 --- a/tests/integration/nerdctl/gha-run.sh +++ b/tests/integration/nerdctl/gha-run.sh @@ -51,7 +51,7 @@ function install_dependencies() { # Create the default containerd configuration sudo mkdir -p /etc/containerd - containerd config default > sudo tee /etc/containerd/config.toml + containerd config default | sudo tee /etc/containerd/config.toml sudo systemctl restart containerd } @@ -106,27 +106,46 @@ function run() { info "Running nerdctl smoke test tests using ${KATA_HYPERVISOR} hypervisor" + # fc-rs uses devmapper block devices; nerdctl must be told to use the + # devmapper snapshotter explicitly so images are unpacked into the pool. + local snapshotter_flag="" + if [ "${KATA_HYPERVISOR}" = "fc-rs" ]; then + snapshotter_flag="--snapshotter devmapper" + fi + info "Running nerdctl with Kata Containers (${KATA_HYPERVISOR})" - sudo nerdctl run --rm --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 --entrypoint nping instrumentisto/nmap --tcp-connect -c 2 -p 80 www.github.com + sudo nerdctl run --rm ${snapshotter_flag} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 --entrypoint nping instrumentisto/nmap --tcp-connect -c 2 -p 80 www.github.com info "Running nerdctl with Kata Containers (${KATA_HYPERVISOR}) and multiple bridge nwtorks" - sudo nerdctl run --rm --net ${net1} --net ${net2} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 alpine ip a + sudo nerdctl run --rm ${snapshotter_flag} --net ${net1} --net ${net2} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 alpine ip a info "Running nerdctl with Kata Containers (${KATA_HYPERVISOR}) and ipvlan network" - sudo nerdctl run --rm --net ${ipvlan_net_name} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 alpine ip a | grep "eth0" + sudo nerdctl run --rm ${snapshotter_flag} --net ${ipvlan_net_name} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 alpine ip a | grep "eth0" info "Running nerdctl with Kata Containers (${KATA_HYPERVISOR}) and macvlan network" - sudo nerdctl run --rm --net ${macvlan_net_name} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 alpine ip a | grep "eth0" + sudo nerdctl run --rm ${snapshotter_flag} --net ${macvlan_net_name} --runtime io.containerd.kata-${KATA_HYPERVISOR}.v2 alpine ip a | grep "eth0" info "Removing networks" sudo nerdctl network rm ${macvlan_net_name} ${ipvlan_net_name} } +function install_kata_for_nerdctl() { + install_kata + + # Firecracker (fc-rs) uses block devices and requires the devmapper + # snapshotter; other hypervisors work fine with the default overlayfs. + # Must run AFTER install_kata since install_kata may overwrite the + # containerd config via overwrite_containerd_config. + if [ "${KATA_HYPERVISOR:-}" = "fc-rs" ]; then + configure_devmapper_for_containerd + fi +} + function main() { action="${1:-}" case "${action}" in install-dependencies) install_dependencies ;; - install-kata) install_kata ;; + install-kata) install_kata_for_nerdctl ;; run) run ;; collect-artifacts) collect_artifacts ;; *) >&2 die "Invalid argument" ;; diff --git a/tests/stability/gha-run.sh b/tests/stability/gha-run.sh index 4e9872d54d..87147f210b 100755 --- a/tests/stability/gha-run.sh +++ b/tests/stability/gha-run.sh @@ -48,11 +48,21 @@ function run() { # bash "${stability_dir}/agent_stability_test.sh" } +function install_kata_for_stability() { + install_kata + + # Firecracker (fc-rs) uses block devices and requires the devmapper + # snapshotter; other hypervisors work fine with the default overlayfs. + if [ "${KATA_HYPERVISOR:-}" = "fc-rs" ]; then + configure_devmapper_for_containerd + fi +} + function main() { action="${1:-}" case "${action}" in install-dependencies) install_dependencies ;; - install-kata) install_kata ;; + install-kata) install_kata_for_stability ;; enabling-hypervisor) enabling_hypervisor ;; run) run ;; *) >&2 die "Invalid argument" ;;